IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 10, 2010, 6:28:51 PM (16 years ago)
Author:
watersc1
Message:

Skycell Summary and stack Association stuff should be finished. I'll merge and do final tests on monday.

Location:
branches/czw_branch/20100519
Files:
9 edited
2 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20100519

  • branches/czw_branch/20100519/ippScripts/scripts/addstar_run.pl

    r27718 r28304  
    3737}
    3838
    39 my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update,
     39my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
    4040     $no_op, $redirect, $save_temps);
    4141GetOptions(
    … …  
    4848    'reduction=s'       => \$reduction, # Reduction class
    4949    'dvodb|w=s'         => \$dvodb,  # output DVO database
     50    'minidvodb_name|w=s'=> \$minidvodb_name,  # miniDVO database name
     51    'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group
     52    'minidvodb'         => \$minidvodb,  # use minidvodb?
    5053    'image-only'        => \$image_only,   # Print to stdout
    5154    'verbose'           => \$verbose,   # Print to stdout
    … …  
    5861pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    5962pod2usage(
    60           -msg => "Required options: --exp_tag --add_id --camera --outroot",
     63          -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot",
    6164          -exitval => 3,
    6265          ) unless
    … …  
    6568    defined $outroot and
    6669    defined $camroot and
     70    defined $dvodb and
    6771    defined $camera;
    68 
     72if ($minidvodb && !defined($minidvodb_group)) {
     73                my_die( "missing minidvodb_group", $add_id, 3 );
     74            }
    6975my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7076
    … …  
    98104if (defined $dvodb) {
    99105    $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO
    100     $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal );
    101 }
     106    $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR);
     107}
     108
     109
     110
    102111
    103112my $dtime_addstar = 0;
    104113
     114if (defined $dvodbReal) {
     115        if ($minidvodb) {
     116           
     117            if (!defined($minidvodb_name)) {
     118                #take the active one, if it's not defined on the command line
     119                #reverts would have this already set, for example.
     120                my $command = "addtool -listminidvodbrun ";
     121                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
     122                $command .= " -state 'active' -limit 1";
     123                $command .= " -dbname $dbname" if defined $dbname;
     124                print $command;
     125                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     126                    run(command => $command, verbose => $verbose);
     127                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
     128                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     129                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
     130
     131                my $components = parse_md_list($metadata) or
     132                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
     133                my $comp = $$components[0];
     134                $minidvodb_name = $comp->{minidvodb_name};
     135       
     136                if (!defined($minidvodb_name)) {
     137                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
     138                }
     139               
     140            }
     141            # tack on the minidvodb part to the db.
     142            $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
     143           
     144        }
    105145unless ($no_op) {
    106     if (defined $dvodbReal) {
     146            print $dvodbReal;
    107147
    108148        ## addstar can either save the full set of detections, or just
    … …  
    142182$fpaCommand .= " -path_base $outroot";
    143183$fpaCommand .= " -dtime_addstar $dtime_addstar";
     184$fpaCommand .= " -dvodb_path $dvodbReal" if defined $dvodbReal;
     185$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
    144186$fpaCommand .= " -dbname $dbname" if defined $dbname;
    145 
     187print $fpaCommand;
    146188# Add the result into the database
    147189unless ($no_update) {
    … …  
    171213        $command .= " -addprocessedexp";
    172214        $command .= " -fault $exit_code";
     215        $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal;
    173216        $command .= " -path_base $outroot" if defined $outroot;
    174217        $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
     218        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
    175219        $command .= " -dbname $dbname" if defined $dbname;
    176220        system ($command);

  • branches/czw_branch/20100519/ippScripts/scripts/automate_stacks.pl

    r27938 r28304  
    2020my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
    2121my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
     22my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    2223my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    2324my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
    … …  
    5960my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
    6061my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
     62my ( $check_detrends, $queue_detrends);
    6163
    6264GetOptions(
    … …  
    8183    'check_stacks'         => \$check_stacks,
    8284    'queue_stacks'         => \$queue_stacks,
     85    'check_detrends'          => \$check_detrends,
     86    'queue_detrends'          => \$queue_detrends,
    8387    'check_diffs'          => \$check_diffs,
    8488    'queue_diffs'          => \$queue_diffs,
    … …  
    105109           --check_stacks         Confirm that stacks can be built.
    106110           --queue_stacks         Issue stacktool commands to queue stacks.
     111           --check_detrends       Confirm that detrend verify runs can be built.
     112           --queue_detrends       Issue dettool commands to queue detrend verify runs.
    107113           --check_diffs          Confirm that diffs can be done.
    108114           --queue_diffs          Issue difftool commands to queue diffs.
    … …  
    115121          ) unless
    116122    defined $check_registration or defined $define_burntool or defined $queue_burntool or
    117     defined $queue_chips or defined $queue_stacks or
    118     defined $check_chips or defined $check_stacks or
     123    defined $queue_chips or defined $queue_stacks or $queue_detrends or
     124    defined $check_chips or defined $check_stacks or $check_detrends or
    119125    defined $test_mode or defined $clean_old or defined $check_mode;
    120126
    … …  
    128134my %stackable_list = ();
    129135my %reduction_class = ();
     136my @detrend_list = ();
     137my %dettype_list = ();
     138my %exptype_list = ();
     139my %refID_list = ();
     140my %refIter_list = ();
     141my %detfilter_list = ();
     142my %detmax_list = ();
    130143my %clean_commands = ();
    131144my %clean_retention = ();
    … …  
    158171        }
    159172    }
    160 #    if (${ $entry }{name} eq 'RETENTION_TIME') {
    161 #        $retention_time = ${ $entry }{value};
    162 #    }
    163173    elsif (${ $entry }{name} eq 'FILTERS') {
    164174        push @filter_list, ${ $entry }{value};
    … …  
    195205        }
    196206    }
    197 }
    198 
    199 
    200 
     207    elsif (${ $entry }{name} eq 'DETRENDS') {
     208        my @detrend_data = @{ ${ $entry }{value} };
     209        my $this_detrend = '';
     210        foreach my $dentry (@detrend_data) {
     211            if (${ $dentry }{name} eq 'NAME') {
     212                $this_detrend = ${ $dentry }{value};
     213                push @detrend_list, $this_detrend;
     214            }
     215            elsif (${ $dentry }{name} eq 'DETTYPE') {
     216                $dettype_list{$this_detrend} = ${ $dentry }{value};
     217            }
     218            elsif (${ $dentry }{name} eq 'EXPTYPE') {
     219                $exptype_list{$this_detrend} = ${ $dentry }{value};
     220            }
     221            elsif (${ $dentry }{name} eq 'REF_ID') {
     222                $refID_list{$this_detrend} = ${ $dentry }{value};
     223            }
     224            elsif (${ $dentry }{name} eq 'REF_ITER') {
     225                $refIter_list{$this_detrend} = ${ $dentry }{value};
     226            }
     227            elsif (${ $dentry }{name} eq 'FILTER') {
     228                $detfilter_list{$this_detrend} = ${ $dentry }{value};
     229            }
     230            elsif (${ $dentry }{name} eq 'MAX') {
     231                $detmax_list{$this_detrend} = ${ $dentry }{value};
     232            }
     233        }
     234    }           
     235}
    201236
    202237unless(defined($date)) {
    … …  
    256291    return_metadata($date);
    257292    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
     293}
     294if (defined($check_detrends) || defined($test_mode)) {
     295    $metadata_out{nsState} = 'CHECKDETRENDS';
     296    &execute_detrends($date,"pretend");
     297    return_metadata($date);
     298    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
     299}
     300if (defined($queue_detrends)) {
     301    $metadata_out{nsState} = 'QUEUEDETRENDS';
     302    &execute_detrends($date,"pretend");
     303    return_metadata($date);
     304    exit(0);
    258305}
    259306if (defined($define_burntool) || defined($test_mode)) {
    … …  
    498545
    499546    my $cmd = "$chiptool";
    500     $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
     547    $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp ";
    501548    $cmd .= " -set_label $label ";
    502549    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
    … …  
    566613}
    567614
     615
     616
    568617sub chip_queue {
    569618    my $date = shift;
    … …  
    612661
    613662#
     663# Detrend verification
     664################################################################################
     665
     666sub construct_dettool_cmd {
     667    my $date = shift;
     668    my $target = shift;
     669
     670    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
     671   
     672    my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
     673    $date =~ s/-//g;
     674
     675    my $cmd = "$dettool";
     676    $cmd .= " -pretend ";
     677    $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
     678    $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";
     679    $cmd .= " $select ";
     680    $cmd .= " -inst $camera ";
     681    $cmd .= " -select_exp_type $exp_type ";
     682    $cmd .= " -select_filter $filter " if defined($filter);
     683    $cmd .= " -workdir $workdir ";
     684    $cmd .= " -label $label ";
     685    if ($maxN > 0) {
     686        $cmd .= " -random_subset -random_limit $maxN ";
     687    }
     688    if ($debug == 1) {
     689        $cmd .= ' -pretend ';
     690    }
     691    print STDERR "$cmd\n";
     692    return($cmd);
     693}   
     694
     695sub verify_uniqueness_detverify {
     696    my $date = shift;
     697    my $target = shift;
     698
     699    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
     700   
     701    my $db = init_gpc_db();
     702    my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter";
     703    my $data_ref = $db->selectall_arrayref( $sth );
     704
     705    return($#{ $data_ref } + 1);
     706}
     707
     708sub pre_detrend_queue {
     709    my $date = shift;
     710    my $target = shift;
     711   
     712    my $command = construct_dettool_cmd($date,$target) . ' -pretend ';
     713    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     714        run ( command => $command, verbose => $verbose);
     715    unless ($success) {
     716        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     717        &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
     718    }
     719   
     720    my @input_exposures = split /\n/, (join '', @$stdout_buf);
     721    return($#input_exposures + 1);
     722}
     723
     724sub detrend_queue {
     725    my $date = shift;
     726    my $target = shift;
     727   
     728    my $command = construct_dettool_cmd($date,$target);
     729    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     730        run ( command => $command, verbose => $verbose);
     731    unless ($success) {
     732        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     733        &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     734    }
     735    $metadata_out{nsState} = 'DETREND_QUEUED';
     736    return(0);
     737}
     738
     739sub execute_detrends {
     740    my $date = shift;
     741    my $pretend = shift;
     742    my $exposures = 0;
     743    foreach my $target (@detrend_list) {
     744        my ($Nexposures) = pre_detrend_queue($date,$target);
     745        if ($Nexposures == 0) {
     746            print STDERR "execute_detrends: Target $target on $date had no exposures.\n";
     747            next;
     748        }
     749        $exposures++;
     750        unless(defined($pretend)) {
     751            detrend_queue($date,$target);
     752        }
     753    }
     754    if ($exposures == 0) {
     755        $metadata_out{nsState} = 'DETREND_DROP';
     756    }
     757}
     758
     759#
    614760# Stacking
    615761################################################################################
    … …  
    627773    my $cmd = "$stacktool";
    628774#    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
    629     $cmd .= ' -simple -dbname gpc1 -definebyquery ';
     775    $cmd .= " -simple -dbname $dbname -definebyquery ";
    630776    $cmd .= " -set_label $label -select_label $label ";
    631777    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
    … …  
    779925    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
    780926    my $args = $command;
    781     $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     927    $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
    782928    if ($debug == 1) {
    783929        $args .= ' -pretend ';
    … …  
    826972
    827973    my $label = "${target}.nightlyscience";
    828     my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
     974    my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}";
    829975    my $obs_mode = $obsmode_list{$target};
    830976    my $object   = $object_list{$target};
    … …  
    837983}
    838984
     985sub get_dettool_parameters {
     986    my $date = shift;
     987    my $target = shift;
     988    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
     989    my $trunc_date = $date; $trunc_date =~ s/-//g;
     990
     991    my $exp_type = $exptype_list{$target};
     992    my $det_type = $dettype_list{$target};
     993    my $ref_det_id = $refID_list{$target};
     994    my $ref_iter = $refIter_list{$target};
     995    my $det_filter = $detfilter_list{$target};
     996    my $internal_filter;
     997    if (defined($det_filter)) {
     998        $internal_filter = $det_filter; $internal_filter =~ s/\..*//;
     999        $internal_filter = '.' . $internal_filter;
     1000    }
     1001    else {
     1002        $internal_filter = '';
     1003    }
     1004    my $maxN = $detmax_list{$target};   
     1005   
     1006    my $lc_type = lc($exp_type);
     1007    my $label = "${lc_type}${internal_filter}.$trunc_date";
     1008    my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}";
     1009    return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN);
     1010}
    8391011
    8401012sub init_gpc_db {

  • branches/czw_branch/20100519/ippScripts/scripts/diff_skycell.pl

    r28043 r28304  
    197197my $recipe_ppSub = $ipprc->reduction($reduction, 'DIFF_PPSUB'); # Recipe to use for ppSub
    198198my $recipe_psphot  = $ipprc->reduction($reduction, 'DIFF_PSPHOT'); # Recipe to use for psphot
     199my $recipe_ppstats = 'DIFFSTATS';
    199200unless ($recipe_ppSub and $recipe_psphot) {
    200201    &my_die("Couldn't find selected reduction for DIFF_PPSUB and DIFF_PSPHOT: $reduction\n", $diff_id, $skycell_id, $PS_EXIT_CONFIG_ERROR);
    … …  
    257258    $traceDest .= '.update';
    258259    $outputStats .= '.update';
     260
     261    $recipe_ppstats = 'WARPSTATS';
    259262}
    260263
    … …  
    292295    $command .= " -recipe PPSUB $recipe_ppSub";
    293296    $command .= " -recipe PSPHOT $recipe_psphot";
    294     $command .= " -recipe PPSTATS DIFFSTATS";
     297    $command .= " -recipe PPSTATS $recipe_ppstats";
    295298    $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";
    296299    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";

  • branches/czw_branch/20100519/ippScripts/scripts/lossy_compress_imfile.pl

    r27718 r28304  
    6161    defined $state;
    6262
    63 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_CONFIG_ERROR );
     63my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $exp_name, $class_id, $uri, $state, $PS_EXIT_CONFIG_ERROR );
    6464if (defined($logfile)) {
    65     $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_SYS_ERROR );
     65    $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $exp_name, $class_id, $uri, $state, $PS_EXIT_SYS_ERROR );
    6666}
    6767
    … …  
    8787if ($state eq 'goto_compressed') {
    8888    # Find the actual filename for this run:
    89     &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR)
     89    &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR)
    9090        unless ($ipprc->file_exists($uri));
    9191    my $uriReal = $ipprc->file_resolve( $uri );
    … …  
    9393    # Create a compressed version:
    9494    my $compUri = $uri . ".fz";
    95     &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR)
     95    &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR)
    9696        if ($ipprc->file_exists($compUri));
    9797    my $compReal= $ipprc->file_resolve( $compUri, 'create');
    … …  
    109109        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    110110        &my_die("Unable to uncompress file: $uri -> $compUri: $error_code",
    111                 $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR);
     111                $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR);
    112112    }
    113113    my $compress_command = "$fpack -h -s 8 -S $tempReal >  $compReal";
    … …  
    118118        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    119119        &my_die("Unable to compress file: $uri -> $compUri: $error_code",
    120                 $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR);
     120                $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR);
    121121    }
    122122    my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state compressed";
    123123    $database_command .= " -dbname $dbname" if defined $dbname;
    124124
    125     &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$PS_EXIT_SYS_ERROR)
     125    &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$state, $PS_EXIT_SYS_ERROR)
    126126        unless ($ipprc->file_exists($compUri));
    127127
    … …  
    135135            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    136136            &my_die("Unable to update database file: $uri -> $compUri: $error_code",
    137                     $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR);
     137                    $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR);
    138138        }
    139139    }
    … …  
    149149if ($state eq 'goto_lossy') {
    150150    # Confirm we have both files
    151     &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR)
     151    &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR)
    152152        unless($ipprc->file_exists($uri));
    153153    my $uriReal = $ipprc->file_resolve( $uri );
    … …  
    156156    unless($ipprc->file_exists($compUri)) {
    157157        &my_die("Couldn't find compressed version of the file: $compUri\n",
    158                 $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR);
     158                $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR);
    159159
    160160        # If that die is removed, this will compress things as well.
    … …  
    185185        $neb->swap($uri,$compUri) or
    186186            &my_die("Nebulous swap failed between $uri and $compUri",
    187                     $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR);
     187                    $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR);
    188188    }
    189189    # Update database
    … …  
    204204            $neb->swap($uri,$compUri) or
    205205                &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri",
    206                         $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR);
     206                        $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR);
    207207            &my_die("Unable to update database file: $uri -> $compUri: $error_code",
    208                     $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR);
     208                    $exp_id,$exp_name,$class_id,$uri, $state,$PS_EXIT_SYS_ERROR);
    209209        }
    210210        # remove original version
    … …  
    224224    my $class_id = shift; # Class identifier
    225225    my $uri = shift; # uri for the file
     226    my $state = shift;
    226227    my $exit_code = shift; # Exit code
    227228    # outputImage and path_base are globals
     229    my $fail_state = $state;
     230    $fail_state =~ s/goto_/error_/;
     231    my $error_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state $fail_state ";
     232    $error_command .= " -dbname $dbname" if defined $dbname;
     233    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
     234        run(command => $error_command, verbose => $verbose);
     235    unless($success) {
     236        warn("Failed at setting the error state too.");
     237    }
    228238
    229239    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;

  • branches/czw_branch/20100519/ippScripts/scripts/magic_process.pl

    r27987 r28304  
    151151
    152152            my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file
    153             &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
     153            &my_die("Unable to find kernel file $kernel", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
    154154
    155155            my ($image, $mask);   # Image and mask
    … …  
    191191
    192192            {
    193                 &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
     193                &my_die("Unable to find image and mask: $image $mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
    194194
    195195                my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel";
    … …  
    204204                $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
    205205                $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
    206                 &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
     206                &my_die("Unable to find created template: $template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
    207207            }
    208208        }

  • branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl

    r28245 r28304  
    3838pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
    3939pod2usage(
    40     -msg => "Required options: --stage --stage_id --outroot\nHelpful options: --camera --dbname",
     40    -msg => "Required options: --stage --stage_id --path_base\nHelpful options: --camera --dbname",
    4141    -exitval => 3,
    4242    ) unless
    … …  
    125125        }
    126126        # Update database:
    127         $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -outroot $outroot";
     127        $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot";
    128128        if (defined($dbname)) {
    129129            $command .= " -dbname $dbname";
    … …  
    136136        }
    137137    }
     138    if (scalar (keys %tangents) == 0) {
     139        my $projection_cell = 'NULL';
     140        $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot";
     141        if (defined($dbname)) {
     142            $command .= " -dbname $dbname";
     143        }
     144       
     145        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     146        unless ($success) {
     147            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     148            &my_die("unable to update diff summary: $error_code", $stage_id, $error_code);
     149        }
     150    }   
    138151} #end warp stage
    139152if ($stage eq 'diff') {
    … …  
    165178        my $state      = $imfile->{data_state};
    166179        my $projection_cell = $skycell_id;
    167         if ($quality == 8007 or $state ne 'full') {
     180        if ($quality == 8007 or $quality = 1010 or $state ne 'full') {
    168181            next;
    169182        }
    … …  
    209222        }
    210223        # Update database:
    211         $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -outroot $outroot";
     224        $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot";
    212225        if (defined($dbname)) {
    213226            $command .= " -dbname $dbname";
    … …  
    220233        }
    221234    }
     235    if (scalar (keys %tangents) == 0) {
     236        my $projection_cell = 'NULL';
     237        $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot";
     238        if (defined($dbname)) {
     239            $command .= " -dbname $dbname";
     240        }
     241       
     242        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     243        unless ($success) {
     244            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     245            &my_die("unable to update diff summary: $error_code", $stage_id, $error_code);
     246        }
     247    }   
    222248} #end diff stage
    223249if ($stage eq 'stack') {
    224250    my $imfiles;
    225     my $command = "$stacktool -stackskyfile -stack_id $stage_id";
     251    my $command = "$stacktool -sassskyfile -sass_id $stage_id";
    226252    if (defined($dbname)) {
    227253        $command .= " -dbname $dbname";
    … …  
    248274        my $quality    = $imfile->{quality};
    249275        my $state      = $imfile->{data_state};
     276        my $fault      = $imfile->{fault};
    250277        my $projection_cell = $skycell_id;
    251         if ($quality == 8007 or $state ne 'full') {
     278        if ($quality == 8007 or $state ne 'full' or $fault != 0) {
    252279            next;
    253280        }
    … …  
    293320        }
    294321        # Update database:
    295         $command = "$stacktool -addsummary -stack_id $stage_id -projection_cell $projection_cell -outroot $outroot";
     322        $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot";
    296323        if (defined($dbname)) {
    297324            $command .= " -dbname $dbname";
    … …  
    304331        }
    305332    }
     333    if (scalar (keys %tangents) == 0) {
     334        my $projection_cell = 'NULL';
     335        $command = "$stacktool -addsummary -stack_id $stage_id -projection_cell $projection_cell -path_base $outroot";
     336        if (defined($dbname)) {
     337            $command .= " -dbname $dbname";
     338        }
     339       
     340        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     341        unless ($success) {
     342            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     343            &my_die("unable to update diff summary: $error_code", $stage_id, $error_code);
     344        }
     345    }   
     346
    306347} #end stack stage
    307348

  • branches/czw_branch/20100519/ippScripts/scripts/stack_skycell.pl

    r28043 r28304  
    149149my $recipe_ppSub = $ipprc->reduction($reduction, 'STACK_PPSUB'); # Recipe to use for ppSub
    150150my $recipe_psphot  = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot
     151my $recipe_ppstats = 'STACKSTATS';
     152if ($run_state eq 'update') {
     153    $recipe_ppstats = 'WARPSTATS';
     154}
    151155unless ($recipe_ppStack and $recipe_ppSub and $recipe_psphot) {
    152156    &my_die("Couldn't find selected reduction for STACK_PPSTACK, STACK_PPSUB and STACK_PSPHOT: $reduction\n", $stack_id, $PS_EXIT_CONFIG_ERROR);
    … …  
    232236    $command .= " -recipe PPSUB $recipe_ppSub";
    233237    $command .= " -recipe PSPHOT $recipe_psphot";
    234     $command .= " -recipe PPSTATS STACKSTATS" if $do_stats;;
     238    $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;;
    235239    $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";
    236240    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";

  • branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl

    r28164 r28304  
    3838}
    3939
    40 my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps);
     40# XXX test:
     41print "run staticsky.pl @ARGV\n";
     42# exit 0;
     43
     44my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps);
    4145GetOptions(
    4246    'sky_id=s'          => \$sky_id, # Diff identifier
    … …  
    4448    'dbname|d=s'        => \$dbname, # Database name
    4549    'threads=s'         => \$threads,   # Number of threads to use
    46     'run-state=s'       => \$run_state,   # state for run: 'new' or 'update'
    4750    'outroot=s'         => \$outroot, # Output root name
    4851    'reduction=s'       => \$reduction, # Reduction class
    … …  
    5659pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    5760pod2usage(
    58     -msg => "Required options: --sky_id --outroot --camera (--run_state)",
     61    -msg => "Required options: --sky_id --outroot --camera",
    5962    -exitval => 3,
    6063          ) unless
    … …  
    6265    defined $outroot and
    6366    defined $camera;
    64 #   and defined $run_state;
    6567
    6668my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    6769
    6870my $logDest = $ipprc->filename("LOG.EXP", $outroot);
    69 # $logDest .= ".update" if $run_state eq "update";
    7071$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect;
    7172
    … …  
    8586    unless ($success) {
    8687        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    87         &my_die("Unable to perform difftool -inputskyfile: $error_code", $sky_id, $error_code);
     88        &my_die("Unable to perform staticskytool -inputs: $error_code", $sky_id, $error_code);
    8889    }
    8990
    … …  
    152153print "recipe_psphot: $recipe_psphot\n";
    153154
    154 my $cmdflags;
    155 
    156 # Perform subtraction
     155# my $cmdflags;
     156
     157# Perform stack photometry analysis
    157158{
    158159    my $command = "$psphotStack $outroot";
    … …  
    167168
    168169    unless ($no_op) {
    169         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    170             run(command => $command, verbose => $verbose);
    171         unless ($success) {
    172             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    173             &my_die("Unable to perform ppSub: $error_code", $sky_id, $error_code);
    174         }
     170        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     171        unless ($success) {
     172            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     173            &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code);
     174        }
    175175
    176176        # my $outputStatsReal = $ipprc->file_resolve($outputStats);
    … …  
    190190        # chomp $cmdflags;
    191191
    192         my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
    193 
     192        # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     193
     194        my $quality = 0;
    194195        if (!$quality) {
    195196
    … …  
    197198            # we have one set of output files per input file set
    198199            for (my $i = 0; $i < @$files; $i++) {
    199                 my $outputName     = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot);
    200                 my $outputMask     = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot);
    201                 my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot);
    202                 my $outputSources  = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot);
     200                my $outputName     = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i);
     201                my $outputMask     = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i);
     202                my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i);
     203                my $outputSources  = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i);
    203204
    204205                &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
    … …  
    211212            #my $outputStats    = $ipprc->filename("SKYCELL.STATS", $outroot);
    212213            #my $traceDest      = $ipprc->filename("TRACE.EXP", $outroot);
    213 
    214             # if ($run_state eq 'update') {
    215             #   $traceDest .= '.update';
    216             #   $outputStats .= '.update';
    217             # }
    218214
    219215            my $chisqName     = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot);
    … …  
    235231    {
    236232        my $command = "$staticskytool -sky_id $sky_id";
    237         # if ($run_state eq 'new') {
    238233        $command .= " -addresult -path_base $outroot";
    239234        $command .= " -num_inputs $nInputs";
    240         $command .= " $cmdflags";
     235        # $command .= " $cmdflags";
    241236        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    242237        $command .= " -hostname $host" if defined $host;
    243         # } else {
    244         #     $command .= " -tofullskyfile";
    245         # }
    246238        $command .= " -dbname $dbname" if defined $dbname;
    247239
    … …  
    250242        unless ($success) {
    251243            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    252             my $err_message = $run_state eq "update" ?
    253                 "Unable to perform difftool -adddiffskyfile" :
    254                 "Unable to perform difftool -tofullskyfile";
    255             &my_die("$err_message: $error_code", $sky_id, $error_code);
     244            my $err_message = "Unable to perform staticskytool -addresult";
     245            warn($err_message);
     246            exit $error_code;
    256247        }
    257248    }
    … …  
    270261    if (defined $sky_id and not $no_update) {
    271262        my $command = "$staticskytool -sky_id $sky_id -fault $exit_code";
    272         if ($run_state eq 'new') {
    273             $command .= " -adddiffskyfile";
    274             $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    275             $command .= " -hostname $host" if defined $host;
    276             $command .= " -path_base $outroot" if defined $outroot;
    277             $command .= " -dbname $dbname" if defined $dbname;
    278         } else {
    279             $command .= " -updatediffskyfile";
    280         }
     263        $command .= " -addresult";
     264        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     265        $command .= " -hostname $host" if defined $host;
     266        $command .= " -path_base $outroot" if defined $outroot;
     267        $command .= " -dbname $dbname" if defined $dbname;
    281268        run(command => $command, verbose => $verbose);
    282269    }
Note: See TracChangeset for help on using the changeset viewer.