IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:41:49 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/tap_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/tap_branches

  • branches/tap_branches/ippScripts/scripts/ipp_cleanup.pl

    r25837 r27838  
    1414use File::Spec;
    1515use PS::IPP::Config 1.01 qw( :standard );
    16 use PS::IPP::Metadata::List qw( parse_md_list );
    1716use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    1817use Pod::Usage qw( pod2usage );
    … …  
    6160
    6261my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
    63                "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
    64                "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
    65                "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
    66 unless ($stages{$stage}) {
     62               "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0,
     63               "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 );
     64unless (exists($stages{$stage})) {
    6765    die "unknown stage $stage for ipp_cleanup.pl\n";
    6866}
     67unless (($stages{$stage})) {
     68    die "unimplemented stage $stage for ipp_cleanup.pl\n";
     69}
     70
    6971
    7072my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    … …  
    9395    # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run.
    9496    if (@$stdout_buf == 0)  {
    95         my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
    96         $command .= " -dbname $dbname" if defined $dbname;
    97 
    98         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    99             run(command => $command, verbose => $verbose);
    100         unless ($success) {
    101             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    102             &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    103         }
    104         exit 0;
    105     }
    106 
    107     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     97        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
     98        $command .= " -dbname $dbname" if defined $dbname;
     99
     100        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     101            run(command => $command, verbose => $verbose);
     102        unless ($success) {
     103            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     104            &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     105        }
     106        exit 0;
     107    }
     108
     109    # extract the metadata for the files into a hash list
     110    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
    108111        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
    109 
    110     # extract the metadata for the files into a hash list
    111     $imfiles = parse_md_list($metadata) or
    112         &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
    113112
    114113    # loop over all of the imfiles, determine the path_base and class_id for each
    … …  
    117116        my $path_base = $imfile->{path_base};
    118117        my $status = 1;
     118        $status = 0 unless defined $path_base and $path_base ne "NULL";
    119119
    120120        # don't clean up unless the data needed to update is available
    121121        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    122         # goto_scrubbed now requires the config file to not exist.
    123         if ($mode eq "goto_cleaned") {
    124             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    125 
    126             unless ($ipprc->file_exists($config_file)) {
    127                 print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    128                     . " because config file ($config_file) is missing\n";
    129                 $status = 0;
    130             }
    131         }
    132         elsif ($mode eq "goto_scrubbed") {
    133             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    134 
    135             if ($ipprc->file_exists($config_file)) {
    136                 print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
    137                     . " because config file ($config_file) is present\n";
    138                 $status = 0;
    139             }
    140         }
     122        # goto_scrubbed now requires the config file to not exist.
     123        if ($status) {
     124            if ($mode eq "goto_cleaned") {
     125                my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     126
     127                unless ($ipprc->file_exists($config_file)) {
     128                    print STDERR "skipping cleanup for chipRun $stage_id $class_id "
     129                        . " because config file ($config_file) is missing\n";
     130                    $status = 0;
     131                }
     132            }
     133            elsif ($mode eq "goto_scrubbed") {
     134                my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     135
     136                if ($ipprc->file_exists($config_file)) {
     137                    print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
     138                        . " because config file ($config_file) is present\n";
     139                    $status = 0;
     140                }
     141            }
     142        }
    141143
    142144        if ($status) {
    … …  
    172174                $command .= " -topurgedimfile";
    173175            }
    174             elsif ($mode eq "goto_cleaned") {
     176            elsif ($mode eq "goto_cleaned") {
    175177                $command .= " -tocleanedimfile";
    176178            }
    177             elsif ($mode eq "goto_scrubbed") {
    178                 $command .= " -toscrubbedimfile";
    179             }
     179            elsif ($mode eq "goto_scrubbed") {
     180                $command .= " -toscrubbedimfile";
     181            }
    180182
    181183            $command .= " -dbname $dbname" if defined $dbname;
    … …  
    188190            }
    189191
    190             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    191             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    192             $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
    193             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    194                 run(command => $command, verbose => $verbose);
    195             unless ($success) {
    196                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    197                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    198             }
     192            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     193            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     194            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     195            $command .= " -dbname $dbname" if defined $dbname;
     196            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     197                run(command => $command, verbose => $verbose);
     198            unless ($success) {
     199                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     200                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     201            }
    199202
    200203        } else {
    201204
    202             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    203             my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
    204             $command .= " -dbname $dbname" if defined $dbname;
     205            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     206            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
     207            $command .= " -dbname $dbname" if defined $dbname;
    205208
    206209            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    210213                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    211214            }
    212            
    213             # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
    214             $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
    215             $command .= " -dbname $dbname" if defined $dbname;
    216 
    217             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    218                 run(command => $command, verbose => $verbose);
    219             unless ($success) {
    220                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    221                 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    222             }
     215
     216            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
     217            $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
     218            $command .= " -dbname $dbname" if defined $dbname;
     219
     220            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     221                run(command => $command, verbose => $verbose);
     222            unless ($success) {
     223                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     224                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     225            }
    223226
    224227        }
    … …  
    242245        &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
    243246    }
    244     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     247    $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
    245248        &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR);
    246 
    247     $exps = parse_md_list($metadata) or
    248         &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR);
    249249
    250250    my $n_exps = @$exps;
    … …  
    260260    if ($mode eq "goto_cleaned") {
    261261        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
    262        
    263         unless ($ipprc->file_exists($config_file)) {
     262
     263        unless ($ipprc->file_exists($config_file)) {
    264264            print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n";
    265265            $status = 0;
    … …  
    267267    }
    268268    elsif ($mode eq "goto_scrubbed") {
    269         my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
    270 
    271         if ($ipprc->file_exists($config_file)) {
    272             print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n";
    273             $status = 0;
    274         }
     269        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     270
     271        if ($ipprc->file_exists($config_file)) {
     272            print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n";
     273            $status = 0;
     274        }
    275275    }
    276276    if ($status) {
    … …  
    292292        if ($mode eq "goto_cleaned") {
    293293            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
    294         }
     294        }
    295295        if ($mode eq "goto_scrubbed") {
    296296            $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed";
    297         }
     297        }
    298298        if ($mode eq "goto_purged") {
    299299            $command = "$camtool -updaterun -cam_id $stage_id -set_state purged";
    … …  
    307307        }
    308308
    309         # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    310         my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    311         $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
    312         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    313             run(command => $command, verbose => $verbose);
    314         unless ($success) {
    315             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    316             &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    317         }
     309        # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     310        my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     311        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     312        $command .= " -dbname $dbname" if defined $dbname;
     313        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     314            run(command => $command, verbose => $verbose);
     315        unless ($success) {
     316            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     317            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     318        }
    318319
    319320    } else {
    320         # since 'camera' has only a single imfile, we can just update the run
     321        # since 'camera' has only a single imfile, we can just update the run
    321322        my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state";
    322323        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    351352        &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    352353    }
    353    
     354
    354355    if (@$stdout_buf == 0) {
    355         # No skycells were found for some reason.
    356         # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
    357         my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state";
    358         $command .= " -dbname $dbname" if defined $dbname;
    359        
    360         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    361             run(command => $command, verbose => $verbose);
    362         unless ($success) {
    363             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    364             &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    365         }
    366        
    367         exit(0);
    368     }
    369     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     356        # No skycells were found for some reason.
     357        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
     358        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state";
     359        $command .= " -dbname $dbname" if defined $dbname;
     360
     361        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     362            run(command => $command, verbose => $verbose);
     363        unless ($success) {
     364            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     365            &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     366        }
     367
     368        exit(0);
     369    }
     370    $skyfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
    370371        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
    371 
    372     $skyfiles = parse_md_list($metadata) or
    373         &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
    374372
    375373    my @files = ();
    … …  
    379377
    380378        my $status = 1;
    381         if ($mode eq "goto_cleaned") {
    382             my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
    383 
    384             unless ($ipprc->file_exists($config_file)) {
    385                 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
    386                     " because config file is missing\n";
    387                 $status = 0;
    388             }
    389         }
    390         elsif ($mode eq "goto_scrubbed") {
    391             my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
    392 
    393             if ($ipprc->file_exists($config_file)) {
    394                 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" .
    395                     " because config file is present\n";
    396                 $status = 0;
    397             }
    398         }
    399         if ($status) {
    400             if ($skyfile->{quality} != 8007) {
    401                 my @files = ();
    402                
    403                 # delete the temporary image datafiles
    404                 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
    405                 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
    406                 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
     379        $status = 0 unless defined $path_base and $path_base ne "NULL";
     380
     381        if ($status) {
     382            if ($mode eq "goto_cleaned") {
     383                my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
     384
     385                unless ($ipprc->file_exists($config_file)) {
     386                    print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
     387                        " because config file is missing\n";
     388                    $status = 0;
     389                }
     390            }
     391            elsif ($mode eq "goto_scrubbed") {
     392                my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
     393
     394                if ($ipprc->file_exists($config_file)) {
     395                    print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" .
     396                        " because config file is present\n";
     397                    $status = 0;
     398                }
     399            }
     400        }
     401
     402        if ($status) {
     403            if ($skyfile->{quality} != 8007) {
     404                my @files = ();
     405
     406                # delete the temporary image datafiles
     407                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
     408                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
     409                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
    407410#            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
    408                 if ($mode eq "goto_purged") {
    409                     # additional files to remove for 'purge' mode
    410                     addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id );
    411                     addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
    412                     addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
    413                     # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
    414                    
    415                     # XXX: do we want to delete these?
    416                     # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
    417                     # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
    418                 }
     411                if ($mode eq "goto_purged") {
     412                    # additional files to remove for 'purge' mode
     413                    addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id );
     414                    addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
     415                    addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
     416                    # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
     417
     418                    # XXX: do we want to delete these?
     419                    # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
     420                    # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
     421                }
    419422            # actual command to delete the files
    420                 $status = &delete_files (\@files);
    421             }
     423                $status = &delete_files (\@files);
     424            }
    422425        }
    423426
    … …  
    426429            if ($mode eq "goto_purged") {
    427430                $command .= " -topurgedskyfile";
    428             } 
    429             elsif ($mode eq "goto_cleaned") {
     431            }
     432            elsif ($mode eq "goto_cleaned") {
    430433                $command .= " -tocleanedskyfile";
    431434            }
    432             elsif ($mode eq "goto_scrubbed") {
    433                 $command .= " -toscrubbedskyfile";
    434             }
     435            elsif ($mode eq "goto_scrubbed") {
     436                $command .= " -toscrubbedskyfile";
     437            }
    435438            $command .= " -dbname $dbname" if defined $dbname;
    436439
    … …  
    442445            }
    443446
    444             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    445             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    446             $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
    447             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    448                 run(command => $command, verbose => $verbose);
    449             unless ($success) {
    450                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    451                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    452             }
     447            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     448            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     449            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     450            $command .= " -dbname $dbname" if defined $dbname;
     451            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     452                run(command => $command, verbose => $verbose);
     453            unless ($success) {
     454                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     455                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     456            }
    453457
    454458         } else {
    455             my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    456             $command .= " -dbname $dbname" if defined $dbname;
     459            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
     460            $command .= " -dbname $dbname" if defined $dbname;
    457461
    458462            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    463467            }
    464468
    465             #            exit $PS_EXIT_UNKNOWN_ERROR;
     469            #            exit $PS_EXIT_UNKNOWN_ERROR;
    466470        }
    467471    }
    … …  
    481485    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    482486    unless ($success) {
    483         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    484         &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     487        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     488        &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    485489    }
    486490
    487491    if (@$stdout_buf == 0) {
    488         # No skycells were found for some reason.
    489         # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
    490         my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state";
    491         $command .= " -dbname $dbname" if defined $dbname;
    492        
    493         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    494             run(command => $command, verbose => $verbose);
    495         unless ($success) {
    496             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    497             &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    498         }
    499        
    500         exit(0);
    501     }
    502 
    503     my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
    504         &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
    505 
    506     $skyfiles = parse_md_list($metadata) or
    507         &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
    508    
     492        # No skycells were found for some reason.
     493        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
     494        my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state";
     495        $command .= " -dbname $dbname" if defined $dbname;
     496
     497        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     498            run(command => $command, verbose => $verbose);
     499        unless ($success) {
     500            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     501            &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     502        }
     503
     504        exit(0);
     505    }
     506
     507    $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or
     508        &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
     509
    509510    my @files = ();
    510511    foreach my $skyfile (@{ $skyfiles }) {
    511         my $path_base = $skyfile->{path_base};
    512         my $skycell_id = $skyfile->{skycell_id};
    513 
    514         my $status = 1;
    515         if ((!exists($skyfile->{path_base}))||
    516             (!defined($path_base))) {
    517             $status = 0;
    518         }
    519         if ($status) {
    520             if ($mode eq "goto_cleaned") {
    521                 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
    522                
    523                 unless ($ipprc->file_exists($config_file)) {
    524                     print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" .
    525                         " because config file is missing\n";
    526                     $status = 0;
    527                 }
    528             }
    529             elsif ($mode eq "goto_scrubbed") {
    530                 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
    531                
    532                 if ($ipprc->file_exists($config_file)) {
    533                     print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" .
    534                         " because config file is present\n";
    535                     $status = 0;
    536                 }
    537             }
    538         }
    539 
    540         if ($status) {
    541             my @files = ();
    542             # delete the temporary image datafiles
    543             addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id);
    544             addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
    545             addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id);
    546 
    547             if ($mode eq "goto_purged") {
    548                 # additional files to remove for 'purge' mode
    549                 addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id);
    550                 addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id);
    551                 addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id);
    552                 # Commented out to match warp files.
    553                 #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id);
    554                 #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id);
    555             }
    556 
    557             $status = &delete_files(\@files);
    558         }
    559 
    560         if ($status) {
    561             my $command = "$stacktool -stack_id $stage_id";
    562             if ($mode eq "goto_purged") {
    563                 $command .= " -updaterun -set_state purged";
    564             }
    565             elsif ($mode eq "goto_cleaned") {
    566                 $command .= " -updaterun -set_state cleaned";
    567             }
    568             elsif ($mode eq "goto_scrubbed") {
    569                 $command .= " -updaterun -set_state scrubbed";
    570             }
    571             $command .= " -dbname $dbname" if defined $dbname;
    572            
    573             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    574                 run(command => $command, verbose => $verbose);
    575             unless ($success) {
    576                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    577                 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    578             }
    579 
    580             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    581             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    582             $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
    583             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    584                 run(command => $command, verbose => $verbose);
    585             unless ($success) {
    586                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    587                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    588             }
    589 
    590         } else {
    591             my $command = "$stacktool -updaterun  -stack_id $stage_id -set_state $error_state";
    592             $command .= " -dbname $dbname" if defined $dbname;
    593            
    594             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    595                 run(command => $command, verbose => $verbose);
    596             unless ($success) {
    597                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    598                 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    599             }
    600 #           exit $PS_EXIT_UNKNOWN_ERROR;
    601         }
     512        my $path_base = $skyfile->{path_base};
     513        my $skycell_id = $skyfile->{skycell_id};
     514
     515        my $status = 1;
     516        if ((!exists($skyfile->{path_base}))||
     517            (!defined($path_base))) {
     518            $status = 0;
     519        }
     520        if ($status) {
     521            if ($mode eq "goto_cleaned") {
     522                my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
     523
     524                unless ($ipprc->file_exists($config_file)) {
     525                    print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" .
     526                        " because config file is missing\n";
     527                    $status = 0;
     528                }
     529            }
     530            elsif ($mode eq "goto_scrubbed") {
     531                my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
     532
     533                if ($ipprc->file_exists($config_file)) {
     534                    print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" .
     535                        " because config file is present\n";
     536                    $status = 0;
     537                }
     538            }
     539        }
     540
     541        if ($status) {
     542            my @files = ();
     543            # delete the temporary image datafiles
     544            addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id);
     545            addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
     546            addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id);
     547
     548            if ($mode eq "goto_purged") {
     549                # additional files to remove for 'purge' mode
     550                addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id);
     551                addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id);
     552                addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id);
     553                # Commented out to match warp files.
     554                #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id);
     555                #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id);
     556            }
     557
     558            $status = &delete_files(\@files);
     559        }
     560
     561        if ($status) {
     562            my $command = "$stacktool -stack_id $stage_id";
     563            if ($mode eq "goto_purged") {
     564                $command .= " -updaterun -set_state purged";
     565            }
     566            elsif ($mode eq "goto_cleaned") {
     567                $command .= " -updaterun -set_state cleaned";
     568            }
     569            elsif ($mode eq "goto_scrubbed") {
     570                $command .= " -updaterun -set_state scrubbed";
     571            }
     572            $command .= " -dbname $dbname" if defined $dbname;
     573
     574            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     575                run(command => $command, verbose => $verbose);
     576            unless ($success) {
     577                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     578                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     579            }
     580
     581            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     582            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     583            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     584            $command .= " -dbname $dbname" if defined $dbname;
     585            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     586                run(command => $command, verbose => $verbose);
     587            unless ($success) {
     588                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     589                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     590            }
     591
     592        } else {
     593            my $command = "$stacktool -updaterun  -stack_id $stage_id -set_state $error_state";
     594            $command .= " -dbname $dbname" if defined $dbname;
     595
     596            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     597                run(command => $command, verbose => $verbose);
     598            unless ($success) {
     599                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     600                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     601            }
     602#           exit $PS_EXIT_UNKNOWN_ERROR;
     603        }
    602604    }
    603605    exit 0;
    … …  
    605607
    606608if ($stage eq 'diff') {
    607    
     609
    608610    die "--stage_id required for stage diff\n" if !$stage_id;
    609611
    … …  
    617619    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    618620    unless ($success) {
    619         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    620         &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     621        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     622        &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    621623    }
    622624
    623625    if (@$stdout_buf == 0) {
    624         # No skycells were found for some reason.
    625         # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
    626         my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
    627         $command .= " -dbname $dbname" if defined $dbname;
    628        
    629         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    630             run(command => $command, verbose => $verbose);
    631         unless ($success) {
    632             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    633             &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    634         }
    635        
    636         exit(0);
    637     }
    638 
    639     my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
    640         &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    641 
    642     $skyfiles = parse_md_list($metadata) or
    643         &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    644    
     626        # No skycells were found for some reason.
     627        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
     628        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     629        $command .= " -dbname $dbname" if defined $dbname;
     630
     631        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     632            run(command => $command, verbose => $verbose);
     633        unless ($success) {
     634            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     635            &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     636        }
     637
     638        exit(0);
     639    }
     640
     641    $skyfiles = $mdcParser->parse_list(join "", @{ $stdout_buf }) or
     642        &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
     643
    645644    my @files = ();
    646645    foreach my $skyfile (@{ $skyfiles }) {
    647         my $path_base = $skyfile->{path_base};
    648         my $skycell_id = $skyfile->{skycell_id};
    649 
    650         my $status = 1;
    651         if ((!exists($skyfile->{path_base}))||
    652             (!defined($path_base))) {
    653             $status = 0;
    654         }
    655         if ($status) {
    656            
    657             if ($mode eq "goto_cleaned") {
    658                 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    659                
    660                 unless ($ipprc->file_exists($config_file)) {
    661                     print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
    662                         " because config file ($config_file) is missing\n";
    663                     $status = 0;
    664                 }
    665             }
    666             elsif ($mode eq "goto_scrubbed") {
    667                 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    668                
    669                 if ($ipprc->file_exists($config_file)) {
    670                     print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" .
    671                         " because config file ($config_file) is present\n";
    672                     $status = 0;
    673                 }
    674             }
    675         }
    676         if ($status) {
    677             my @files = ();
    678             # delete the temporary image datafiles
    679             addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
    680             addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
    681             addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
    682 
    683             addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
    684             addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
    685             addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
    686 
    687             addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
    688             addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id);
    689             addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id);
    690            
    691             addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id);
    692             addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id);
    693             addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id);
    694            
    695             if ($mode eq "goto_purged") {
    696                 # additional files to remove for 'purge' mode
    697                 addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id);
    698                 addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
    699                 addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id);
    700                 # Commented out to match warp files.
    701                 #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id);
    702                 addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id);
    703                 addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id);
    704                
    705             }
    706 #           print STDERR "MY FILES: @files\n";
    707             $status = &delete_files(\@files);
    708         }
    709 #       print STDERR "MY STATUS: $status\n";
    710         if ($status) {
    711             my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
    712 
    713             if ($mode eq "goto_purged") {
    714                 $command .= " -topurgedskyfile";
    715             }
    716             elsif ($mode eq "goto_cleaned") {
    717                 $command .= " -tocleanedskyfile";
    718             }
    719             elsif ($mode eq "goto_scrubbed") {
    720                 $command .= " -toscrubbedskyfile";
    721             }
    722 
    723             $command .= " -dbname $dbname" if defined $dbname;
    724            
    725             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    726                 run(command => $command, verbose => $verbose);
    727             unless ($success) {
    728                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    729                 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    730             }
    731 
    732             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    733             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    734             $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
    735             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    736                 run(command => $command, verbose => $verbose);
    737             unless ($success) {
    738                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    739                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    740             }
    741 
    742         } else {
    743             my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
    744 
    745             $command .= " -dbname $dbname" if defined $dbname;
    746            
    747             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    748                 run(command => $command, verbose => $verbose);
    749             unless ($success) {
    750                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    751                 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    752             }
    753 #           exit $PS_EXIT_UNKNOWN_ERROR;
    754         }
     646        my $path_base = $skyfile->{path_base};
     647        my $skycell_id = $skyfile->{skycell_id};
     648
     649        my $status = 1;
     650        if ((!exists($skyfile->{path_base}))||
     651            (!defined($path_base))) {
     652            $status = 0;
     653        }
     654        if ($status) {
     655
     656            if ($mode eq "goto_cleaned") {
     657                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
     658
     659                unless ($ipprc->file_exists($config_file)) {
     660                    print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
     661                        " because config file ($config_file) is missing\n";
     662                    $status = 0;
     663                }
     664            }
     665            elsif ($mode eq "goto_scrubbed") {
     666                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
     667
     668                if ($ipprc->file_exists($config_file)) {
     669                    print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" .
     670                        " because config file ($config_file) is present\n";
     671                    $status = 0;
     672                }
     673            }
     674        }
     675        if ($status) {
     676            my @files = ();
     677            # delete the temporary image datafiles
     678            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
     679            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
     680            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
     681
     682            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
     683            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
     684            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
     685
     686            addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
     687            addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id);
     688            addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id);
     689
     690            addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id);
     691            addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id);
     692            addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id);
     693
     694            if ($mode eq "goto_purged") {
     695                # additional files to remove for 'purge' mode
     696                addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id);
     697                addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
     698                addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id);
     699                # Commented out to match warp files.
     700                #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id);
     701                addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id);
     702                addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id);
     703
     704            }
     705#           print STDERR "MY FILES: @files\n";
     706            $status = &delete_files(\@files);
     707        }
     708#       print STDERR "MY STATUS: $status\n";
     709        if ($status) {
     710            my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     711
     712            if ($mode eq "goto_purged") {
     713                $command .= " -topurgedskyfile";
     714            }
     715            elsif ($mode eq "goto_cleaned") {
     716                $command .= " -tocleanedskyfile";
     717            }
     718            elsif ($mode eq "goto_scrubbed") {
     719                $command .= " -toscrubbedskyfile";
     720            }
     721
     722            $command .= " -dbname $dbname" if defined $dbname;
     723
     724            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     725                run(command => $command, verbose => $verbose);
     726            unless ($success) {
     727                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     728                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     729            }
     730
     731            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     732            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     733            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     734            $command .= " -dbname $dbname" if defined $dbname;
     735            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     736                run(command => $command, verbose => $verbose);
     737            unless ($success) {
     738                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     739                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     740            }
     741
     742        } else {
     743            my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     744
     745            $command .= " -dbname $dbname" if defined $dbname;
     746
     747            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     748                run(command => $command, verbose => $verbose);
     749            unless ($success) {
     750                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     751                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     752            }
     753#           exit $PS_EXIT_UNKNOWN_ERROR;
     754        }
    755755    }
    756756    exit 0;
    … …  
    778778    # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    779779    if (@$stdout_buf == 0)  {
    780         my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state";
    781         $command .= " -dbname $dbname" if defined $dbname;
    782 
    783         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    784             run(command => $command, verbose => $verbose);
    785         unless ($success) {
    786             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    787             &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
    788         }
    789         exit 0;
    790     }
    791 
    792     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     780        my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state";
     781        $command .= " -dbname $dbname" if defined $dbname;
     782
     783        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     784            run(command => $command, verbose => $verbose);
     785        unless ($success) {
     786            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     787            &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
     788        }
     789        exit 0;
     790    }
     791
     792    # extract the metadata for the files into a hash list
     793    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
    793794        &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
    794 
    795     # extract the metadata for the files into a hash list
    796     $imfiles = parse_md_list($metadata) or
    797         &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
    798795
    799796    # loop over all of the imfiles, determine the path_base and class_id for each
    … …  
    805802        # don't clean up unless the data needed to update is available
    806803        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    807         # goto_scrubbed now requires the config file to not exist.
     804        # goto_scrubbed now requires the config file to not exist.
    808805        if ($mode eq "goto_cleaned") {
    809806            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
    810807
    811             unless ($ipprc->file_exists($config_file)) {
     808            unless ($ipprc->file_exists($config_file)) {
    812809                print STDERR "skipping cleanup for fakeRun $stage_id $class_id "
    813810                    . " because config file is missing\n";
    … …  
    815812            }
    816813        }
    817         elsif ($mode eq "goto_scrubbed") {
    818             my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
    819 
    820             if ($ipprc->file_exists($config_file)) {
    821                 print STDERR "skipping scrubbed for fakeRun $stage_id $class_id "
    822                     . " because config file is present\n";
    823                 $status = 0;
    824             }
    825         }
     814        elsif ($mode eq "goto_scrubbed") {
     815            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
     816
     817            if ($ipprc->file_exists($config_file)) {
     818                print STDERR "skipping scrubbed for fakeRun $stage_id $class_id "
     819                    . " because config file is present\n";
     820                $status = 0;
     821            }
     822        }
    826823
    827824        if ($status) {
    … …  
    836833            if ($mode eq "goto_purged") {
    837834                # additional files to remove for 'purge' mode
    838                 addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id);
    839                 addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id);
    840                 addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id);
     835                addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id);
     836                addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id);
     837                addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id);
    841838            }
    842839
    … …  
    850847                $command .= " -topurgedimfile";
    851848            }
    852             elsif ($mode eq "goto_cleaned") {
     849            elsif ($mode eq "goto_cleaned") {
    853850                $command .= " -tocleanedimfile";
    854851            }
    855             elsif ($mode eq "goto_scrubbed") {
    856                 $command .= " -toscrubbedimfile";
    857             }
     852            elsif ($mode eq "goto_scrubbed") {
     853                $command .= " -toscrubbedimfile";
     854            }
    858855
    859856            $command .= " -dbname $dbname" if defined $dbname;
    … …  
    867864        } else {
    868865
    869             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    870             my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state";
    871             $command .= " -dbname $dbname" if defined $dbname;
     866            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
     867            my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state";
     868            $command .= " -dbname $dbname" if defined $dbname;
    872869
    873870            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    881878    exit 0;
    882879
    883 } 
     880}
    884881# Detrend stages
    885 if ($stage eq "detrend.process.imfile") {
     882if ($stage eq "detrend.processed") {
    886883
    887884    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
    888885    ### select the imfiles for this entry
     886
     887    # Neither det_id nor exp_id uniquely determine a det exposure, so we pack them.
     888    my ($det_id,$exp_id) = split /\./, $stage_id;       #/ trailing slash for emacs;
    889889
    890890    # this stage uses 'dettool'
    … …  
    894894    # XXX may need a different my_die for each stage
    895895    my $imfiles;                      # Array of component files
    896     my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
     896    my $metadata;
     897    my $command = "$dettool -pendingcleanup_processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run
    897898    $command .= " -dbname $dbname" if defined $dbname;
    898899    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    899900    unless ($success) {
    900901        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    901         &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     902        &my_die("Unable to perform dettool: $error_code", "detrend.processed", $stage_id, $error_code);
    902903    }
    903904
    904905    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    905     if (@$stdout_buf == 0)  {
    906         exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
    907     }
    908 
    909     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     906    if (@$stdout_buf != 0)  {
     907#       exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
     908
     909
     910        # extract the metadata for the files into a hash list
     911        $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
    910912        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    911 
    912     # extract the metadata for the files into a hash list
    913     $imfiles = parse_md_list($metadata) or
    914         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    915 
     913    }
    916914    # loop over all of the imfiles, determine the path_base and class_id for each
    917915    foreach my $imfile (@$imfiles) {
    918         my $exp_id   = $imfile->{exp_id};
     916        my $iexp_id   = $imfile->{exp_id};
    919917        my $class_id = $imfile->{class_id};
    920918        my $path_base = $imfile->{path_base};
    921919        my $status = 1;
    922920
    923         # don't clean up unless the data needed to update is available
    924         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    925         # goto_scrubbed now requires the config file to not exist.
    926        
    927         # Possibly not the correct config file, but simtest doesn't leave any around to check.
    928         if ($mode eq "goto_cleaned") {
    929             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    930 
    931             unless ($ipprc->file_exists($config_file)) {
    932                 print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
    933                     . " because config file is missing\n";
    934                 $status = 0;
    935             }
    936         }
    937         elsif ($mode eq "goto_scrubbed") {
    938             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    939 
    940             if ($ipprc->file_exists($config_file)) {
    941                 print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
    942                     . " because config file is present\n";
    943                 $status = 0;
    944             }
    945         }
    946 
     921        unless (defined($path_base)) {
     922            print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
     923            $status = 0;
     924        }
     925        unless (defined($class_id)) {
     926            print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
     927            $status = 0;
     928        }
     929        # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent,
     930        # and so there is no check for config files.
    947931        if ($status) {
    948932            # array of actual filenames to delete
    949933            my @files = ();
    950 
    951934            # delete the temporary image datafiles
    952935            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    … …  
    955938            if ($mode eq "goto_purged") {
    956939                # additional files to remove for 'purge' mode
    957                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
    958                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     940                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     941                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    959942                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    960943            }
    961 
    962944            # actual command to delete the files
    963945            $status = &delete_files (\@files);
    … …  
    965947
    966948        if ($status)  {
    967             my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
     949            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile";
    968950            if ($mode eq "goto_purged") {
    969951                $command .= " -data_state purged";
    970952            }
    971             elsif ($mode eq "goto_cleaned") {
     953            elsif ($mode eq "goto_cleaned") {
    972954                $command .= " -data_state cleaned";
    973955            }
    974             elsif ($mode eq "goto_scrubbed") {
    975                 $command .= " -data_state scrubbed";
    976             }
    977 
     956            elsif ($mode eq "goto_scrubbed") {
     957                $command .= " -data_state scrubbed";
     958            }
    978959            $command .= " -dbname $dbname" if defined $dbname;
    979960
    … …  
    986967
    987968        } else {
    988 
    989             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    990             my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
    991             $command .= " -dbname $dbname" if defined $dbname;
     969            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_*
     970            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile ";
     971            $command .= " -data_state $error_state";
     972            $command .= " -dbname $dbname" if defined $dbname;
    992973
    993974            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    1000981    }
    1001982
    1002     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1003 
    1004     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     983    # Flag the detProcessedExp as clean now as well, if it is marked to be cleaned (this is clunky, but works for now).
     984
     985    $command = "$dettool -pendingcleanup_processedexp -det_id $det_id -exp_id $exp_id";
    1005986    $command .= " -dbname $dbname" if defined $dbname;
    1006987    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1007988    unless ($success) {
    1008         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1009         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     989        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     990        &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
    1010991    }
    1011992    if (@$stdout_buf != 0) {
    1012         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1013             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1014         my $exps = parse_md_list($metadata) or
    1015             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1016        
    1017         foreach my $exp (@$exps) {
    1018             my $iteration = $exp->{iteration};
    1019             my $command;
    1020             if ($mode eq "goto_cleaned") {
    1021                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1022             }
    1023             if ($mode eq "goto_scrubbed") {
    1024                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1025             }
    1026             if ($mode eq "goto_purged") {
    1027                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1028             }
    1029             $command .= " -dbname $dbname" if defined $dbname;
    1030             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1031                 run(command => $command, verbose => $verbose);
    1032             unless ($success) {
    1033                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1034                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1035             }
    1036         }
    1037     }
    1038    
     993        my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     994            &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR);
     995
     996        foreach my $exp (@$exps) {
     997            my $exp_id = $exp->{exp_id};
     998            my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id ";
     999            if ($mode eq "goto_cleaned") {
     1000                $command .= " -data_state cleaned ";
     1001            }
     1002            if ($mode eq "goto_scrubbed") {
     1003                $command .= " -data_state scrubbed ";
     1004            }
     1005            if ($mode eq "goto_purged") {
     1006                $command .= " -data_state purged ";
     1007            }
     1008            $command .= " -dbname $dbname" if defined $dbname;
     1009#           print "$command\n";
     1010            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1011                run(command => $command, verbose => $verbose);
     1012            unless ($success) {
     1013                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1014                &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
     1015            }
     1016        }
     1017    }
     1018
    10391019    exit 0;
    10401020}
    1041 if ($stage eq "detrend.process.exp") {
     1021if ($stage eq "detrend.resid") {
     1022
    10421023    die "--stage_id required for stage $stage\n" if !$stage_id;
     1024    ### select the imfiles for this entry
     1025
     1026    # Neither det_id nor exp_id uniquely determine the det exposure, so we pack them
     1027    my ($det_id,$exp_id) = split /\./, $stage_id;          #/ trailing slash for emacs;
     1028
    10431029    # this stage uses 'dettool'
    10441030    my $dettool = can_run('dettool') or die "Can't find dettool";
    … …  
    10461032    # Get list of component imfiles
    10471033    # XXX may need a different my_die for each stage
    1048     my $exps;                      # Array of component files
    1049     my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
     1034    my $imfiles;                      # Array of component files
     1035    my $metadata;
     1036    my $command = "$dettool -pendingcleanup_residimfile -det_id $det_id -exp_id $exp_id"; # Command to run
    10501037    $command .= " -dbname $dbname" if defined $dbname;
    10511038    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    10521039    unless ($success) {
    10531040        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1054         &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1055     }
    1056    
    1057     if (@$stdout_buf == 0) {
    1058         exit 0; #silently abort. I need to fix this for propers
    1059     }
    1060 
    1061     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1062         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1063 
    1064     $exps = parse_md_list($metadata) or
    1065         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1066 
    1067 
    1068     foreach my $exp (@$exps) {
    1069         my $path_base = $exp->{path_base};
    1070         my $exp_id    = $exp->{exp_id};
    1071 
    1072         my $status = 1;
    1073         # don't clean up unless the data needed to update is available
    1074         # goto_scrubbed now requires the config file to not be present
    1075         if ($mode eq "goto_cleaned") {
    1076             my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
    1077            
    1078             unless ($ipprc->file_exists($config_file)) {
    1079                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1080                 $status = 0;
    1081             }
    1082         }
    1083         elsif ($mode eq "goto_scrubbed") {
    1084             my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
    1085            
    1086             if ($ipprc->file_exists($config_file)) {
    1087                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1088                 $status = 0;
    1089             }
    1090         }
    1091         if ($status) {
    1092             my @files = ();
    1093             # delete the temporary image datafiles
    1094             # I can't find anything to put here
    1095             if ($mode eq "goto_purged") {
    1096                 # additional files to remove for 'purge' mode
    1097                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
    1098                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
    1099             }
    1100             # actual command to delete the files
    1101             $status = &delete_files (\@files);
    1102         }
    1103        
    1104         if ($status)  {
    1105             my $command;
    1106             if ($mode eq "goto_cleaned") {
    1107                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
    1108             }
    1109             if ($mode eq "goto_scrubbed") {
    1110                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
    1111             }
    1112             if ($mode eq "goto_purged") {
    1113                 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
    1114             }
    1115             $command .= " -dbname $dbname" if defined $dbname;
    1116             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1117                 run(command => $command, verbose => $verbose);
    1118             unless ($success) {
    1119                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1120                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1121             }
    1122         } else {
    1123             # since 'camera' has only a single imfile, we can just update the run
    1124             my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
    1125             $command .= " -dbname $dbname" if defined $dbname;
    1126            
    1127             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1128                 run(command => $command, verbose => $verbose);
    1129             unless ($success) {
    1130                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1131                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1132             }
    1133             exit $PS_EXIT_UNKNOWN_ERROR;
    1134         }
    1135     }
    1136     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1137 
    1138     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
     1041        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     1042    }
     1043
     1044    # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit.
     1045    if (@$stdout_buf != 0) {
     1046        # extract the metadata for the files into a hash list
     1047        $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
     1048            &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1049    }
     1050    # loop over all of the imfiles, determine the path_base and class_id for each
     1051    foreach my $imfile (@$imfiles) {
     1052        my $iexp_id = $imfile->{exp_id};
     1053        my $class_id = $imfile->{class_id};
     1054        my $path_base = $imfile->{path_base};
     1055        my $iteration = $imfile->{iteration};
     1056
     1057        my $status = 1;
     1058
     1059        # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent,
     1060        # and so there is no check for config files.
     1061        unless (defined($path_base)) {
     1062            print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
     1063            $status = 0;
     1064        }
     1065        unless (defined($class_id)) {
     1066            print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
     1067            $status = 0;
     1068        }
     1069        if ($status) {
     1070            # array of actual filenames to delete
     1071            my @files = ();
     1072
     1073            # delete the temporary image datafiles
     1074            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     1075            if ($mode eq "goto_purged") {
     1076                # additional files to remove for 'purge' mode
     1077                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     1078                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     1079                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     1080            }
     1081#           foreach my $f (@files) {
     1082#               print "RESID: $f\n";
     1083#           }
     1084            # actual command to delete the files
     1085            $status = &delete_files (\@files);
     1086        }
     1087
     1088        if ($status)  {
     1089            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
     1090            if ($mode eq "goto_purged") {
     1091                $command .= " -data_state purged";
     1092            }
     1093            elsif ($mode eq "goto_cleaned") {
     1094                $command .= " -data_state cleaned";
     1095            }
     1096            elsif ($mode eq "goto_scrubbed") {
     1097                $command .= " -data_state scrubbed";
     1098            }
     1099
     1100            $command .= " -dbname $dbname" if defined $dbname;
     1101
     1102            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1103                    run(command => $command, verbose => $verbose);
     1104            unless ($success) {
     1105                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1106                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1107            }
     1108        } else {
     1109            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
     1110            $command .= " -data_state $error_state ";
     1111            $command .= " -dbname $dbname" if defined $dbname;
     1112
     1113            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1114                    run(command => $command, verbose => $verbose);
     1115            unless ($success) {
     1116                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1117                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1118            }
     1119        }
     1120    }
     1121    # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky).
     1122
     1123    $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id";
    11391124    $command .= " -dbname $dbname" if defined $dbname;
    11401125    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    11411126    unless ($success) {
    1142         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1143         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1127        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1128        &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
    11441129    }
    11451130    if (@$stdout_buf != 0) {
    1146         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1147             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1148         $exps = parse_md_list($metadata) or
    1149             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1150        
    1151         foreach my $exp (@$exps) {
    1152             my $iteration = $exp->{iteration};
    1153             my $command;
    1154             if ($mode eq "goto_cleaned") {
    1155                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1156             }
    1157             if ($mode eq "goto_scrubbed") {
    1158                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1159             }
    1160             if ($mode eq "goto_purged") {
    1161                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1162             }
    1163             $command .= " -dbname $dbname" if defined $dbname;
    1164             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1165                 run(command => $command, verbose => $verbose);
    1166             unless ($success) {
    1167                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1168                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1169             }
    1170         }
     1131        my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     1132            &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR);
     1133
     1134        foreach my $exp (@$exps) {
     1135            my $iteration = $exp->{iteration};
     1136            my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id ";
     1137            if ($mode eq "goto_cleaned") {
     1138                $command .= " -data_state cleaned";
     1139            }
     1140            if ($mode eq "goto_scrubbed") {
     1141                $command .= " -data_state scrubbed";
     1142            }
     1143            if ($mode eq "goto_purged") {
     1144                $command .= " -data_state purged";
     1145            }
     1146            $command .= " -dbname $dbname" if defined $dbname;
     1147            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1148                run(command => $command, verbose => $verbose);
     1149            unless ($success) {
     1150                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1151                &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
     1152            }
     1153        }
    11711154    }
    11721155
    11731156    exit 0;
    11741157}
     1158
    11751159if ($stage eq "detrend.stack.imfile") {
    1176    
     1160
    11771161    die "--stage_id required for stage $stage\n" if !$stage_id;
    11781162
    … …  
    11861170    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    11871171    unless ($success) {
    1188         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1189         &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1190     }
    1191     my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
    1192         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1193 
    1194     $stacks = parse_md_list($metadata) or
    1195         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1196    
     1172        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1173        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1174    }
     1175    $stacks = $mdcParser->parse_list(join "", @{ $stdout_buf }) or
     1176        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
     1177
    11971178    my @files = ();
    11981179    foreach my $stack (@{ $stacks }) {
    1199         # detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
    1200         my $path_base = $stack->{uri};
    1201         my $iteration = $stack->{iteration};
    1202         my $class_id  = $stack->{class_id};
    1203 
    1204         $path_base =~ s/\.fits$//; # That should do it?
    1205 
    1206         my $status = 1;
    1207 
    1208 #       if ($mode eq "goto_cleaned") {
    1209 #           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1210 
    1211 #           $config_file =~ s%^file://%%;
    1212 #           if (!$config_file or ! -e $config_file) {
    1213 #               print STDERR "skipping cleanup for $stage $stage_id $path_base" .
    1214 #                   " because config file is missing\n";
    1215 #               $status = 0;
    1216 #           }
    1217 #           $config_file = 'file://' . $config_file;
    1218 #       }
    1219 #       elsif ($mode eq "goto_scrubbed") {
    1220 #           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
    1221 #           $config_file =~ s%^file://%%;
    1222 #           if ($config_file and -e $config_file) {
    1223 #               print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
    1224 #                   " because config file is present\n";
    1225 #               $status = 0;
    1226 #           }
    1227 #       }
    1228 
    1229         if ($status) {
    1230             my @files = ();
    1231             # delete the temporary image datafiles
    1232             # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion.
    1233             # I understand that they all point to the same filename right now, but that may not be true in
    1234             # the future.
    1235             addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id);
    1236             addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id);
    1237             addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id);
    1238             addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id);
    1239             addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id);
    1240             addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id);
    1241            
    1242 
    1243             addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
    1244             addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);
    1245 
    1246             if ($mode eq "goto_purged") {
    1247                 # additional files to remove for 'purge' mode
    1248 #               addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
    1249             }
    1250 
    1251             $status = &delete_files(\@files);
    1252         }
    1253 
    1254         if ($status) {
    1255             my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
    1256             if ($mode eq "goto_purged") {
    1257                 $command .= " -updatestacked -data_state purged";
    1258             }
    1259             elsif ($mode eq "goto_cleaned") {
    1260                 $command .= " -updatestacked -data_state cleaned";
    1261             }
    1262             elsif ($mode eq "goto_scrubbed") {
    1263                 $command .= " -updatestacked -data_state scrubbed";
    1264             }
    1265             $command .= " -dbname $dbname" if defined $dbname;
    1266            
    1267             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1268                 run(command => $command, verbose => $verbose);
    1269             unless ($success) {
    1270                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1271                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1272             }
    1273         } else {
    1274             my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
    1275             $command .= " -dbname $dbname" if defined $dbname;
    1276            
    1277             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1278                 run(command => $command, verbose => $verbose);
    1279             unless ($success) {
    1280                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1281                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1282             }
    1283             exit $PS_EXIT_UNKNOWN_ERROR;
    1284         }
     1180        # detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
     1181        my $path_base = $stack->{uri};
     1182        my $iteration = $stack->{iteration};
     1183        my $class_id  = $stack->{class_id};
     1184
     1185        $path_base =~ s/\.fits$//; # That should do it?
     1186
     1187        my $status = 1;
     1188
     1189        if ($status) {
     1190            my @files = ();
     1191            # delete the temporary image datafiles
     1192            # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion.
     1193            # I understand that they all point to the same filename right now, but that may not be true in
     1194            # the future.
     1195            addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id);
     1196            addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id);
     1197            addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id);
     1198            addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id);
     1199            addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id);
     1200            addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id);
     1201
     1202
     1203            addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
     1204            addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);
     1205
     1206            if ($mode eq "goto_purged") {
     1207                # additional files to remove for 'purge' mode
     1208#               addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
     1209            }
     1210
     1211            $status = &delete_files(\@files);
     1212        }
     1213
     1214        if ($status) {
     1215            my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
     1216            if ($mode eq "goto_purged") {
     1217                $command .= " -updatestacked -data_state purged";
     1218            }
     1219            elsif ($mode eq "goto_cleaned") {
     1220                $command .= " -updatestacked -data_state cleaned";
     1221            }
     1222            elsif ($mode eq "goto_scrubbed") {
     1223                $command .= " -updatestacked -data_state scrubbed";
     1224            }
     1225            $command .= " -dbname $dbname" if defined $dbname;
     1226
     1227            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1228                run(command => $command, verbose => $verbose);
     1229            unless ($success) {
     1230                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1231                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1232            }
     1233        } else {
     1234            my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1235            $command .= " -dbname $dbname" if defined $dbname;
     1236
     1237            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1238                run(command => $command, verbose => $verbose);
     1239            unless ($success) {
     1240                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1241                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1242            }
     1243            exit $PS_EXIT_UNKNOWN_ERROR;
     1244        }
    12851245    }
    12861246    # Check to see if we can mark the whole detRunSummary object as cleaned.
    12871247
    1288     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
     1248    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    12891249    $command .= " -dbname $dbname" if defined $dbname;
    12901250    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    12911251    unless ($success) {
    1292         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1293         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1252        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1253        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    12941254    }
    12951255    if (@$stdout_buf != 0) {
    1296         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1297             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1298         my $exps = parse_md_list($metadata) or
    1299             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1300        
    1301         foreach my $exp (@$exps) {
    1302             my $iteration = $exp->{iteration};
    1303             my $command;
    1304             if ($mode eq "goto_cleaned") {
    1305                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1306             }
    1307             if ($mode eq "goto_scrubbed") {
    1308                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1309             }
    1310             if ($mode eq "goto_purged") {
    1311                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1312             }
    1313             $command .= " -dbname $dbname" if defined $dbname;
    1314             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1315                 run(command => $command, verbose => $verbose);
    1316             unless ($success) {
    1317                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1318                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1319             }
    1320         }
     1256        my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     1257            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1258
     1259        foreach my $exp (@$exps) {
     1260            my $iteration = $exp->{iteration};
     1261            my $command;
     1262            if ($mode eq "goto_cleaned") {
     1263                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1264            }
     1265            if ($mode eq "goto_scrubbed") {
     1266                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1267            }
     1268            if ($mode eq "goto_purged") {
     1269                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1270            }
     1271            $command .= " -dbname $dbname" if defined $dbname;
     1272            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1273                run(command => $command, verbose => $verbose);
     1274            unless ($success) {
     1275                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1276                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1277            }
     1278        }
    13211279    }
    13221280    exit 0;
    … …  
    13351293        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    13361294    }
    1337     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1295    my $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
    13381296        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    13391297
    1340     my $exps = parse_md_list($metadata) or
    1341         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1342 
    13431298    foreach my $exp (@$exps) {
    1344 #       my $path_base = $exp->{path_base};
    1345         my $iteration = $exp->{iteration};
    1346         my $class_id  = $exp->{class_id};
    1347 
    1348         my $status = 1;
    1349         if ($status)  {
    1350             my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
    1351             if ($mode eq "goto_cleaned") {
    1352                 $command .= " -data_state cleaned";
    1353             }
    1354             if ($mode eq "goto_scrubbed") {
    1355                 $command .= " -data_state scrubbed";
    1356             }
    1357             if ($mode eq "goto_purged") {
    1358                 $command .= " -data_state purged";
    1359             }
    1360             $command .= " -dbname $dbname" if defined $dbname;
    1361             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1362                 run(command => $command, verbose => $verbose);
    1363             unless ($success) {
    1364                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1365                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1366             }
    1367         } else {
    1368             my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
    1369             $command .= " -dbname $dbname" if defined $dbname;
    1370            
    1371             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1372                 run(command => $command, verbose => $verbose);
    1373             unless ($success) {
    1374                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1375                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1376             }
    1377             exit $PS_EXIT_UNKNOWN_ERROR;
    1378         }
     1299#       my $path_base = $exp->{path_base};
     1300        my $iteration = $exp->{iteration};
     1301        my $class_id  = $exp->{class_id};
     1302
     1303        my $status = 1;
     1304        if ($status)  {
     1305            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
     1306            if ($mode eq "goto_cleaned") {
     1307                $command .= " -data_state cleaned";
     1308            }
     1309            if ($mode eq "goto_scrubbed") {
     1310                $command .= " -data_state scrubbed";
     1311            }
     1312            if ($mode eq "goto_purged") {
     1313                $command .= " -data_state purged";
     1314            }
     1315            $command .= " -dbname $dbname" if defined $dbname;
     1316            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1317                run(command => $command, verbose => $verbose);
     1318            unless ($success) {
     1319                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1320                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1321            }
     1322        } else {
     1323            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1324            $command .= " -dbname $dbname" if defined $dbname;
     1325
     1326            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1327                run(command => $command, verbose => $verbose);
     1328            unless ($success) {
     1329                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1330                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1331            }
     1332            exit $PS_EXIT_UNKNOWN_ERROR;
     1333        }
    13791334    }
    13801335    # Check to see if we can mark the whole detRunSummary object as cleaned.
    13811336
    1382     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
     1337    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    13831338    $command .= " -dbname $dbname" if defined $dbname;
    13841339    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    13851340    unless ($success) {
    1386         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1387         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1341        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1342        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    13881343    }
    13891344    if (@$stdout_buf != 0) {
    1390         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1391             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1392         $exps = parse_md_list($metadata) or
    1393             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1394        
    1395         foreach my $exp (@$exps) {
    1396             my $iteration = $exp->{iteration};
    1397             my $command;
    1398             if ($mode eq "goto_cleaned") {
    1399                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1400             }
    1401             if ($mode eq "goto_scrubbed") {
    1402                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1403             }
    1404             if ($mode eq "goto_purged") {
    1405                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1406             }
    1407             $command .= " -dbname $dbname" if defined $dbname;
    1408             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1409                 run(command => $command, verbose => $verbose);
    1410             unless ($success) {
    1411                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1412                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1413             }
    1414         }
     1345        $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     1346            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1347
     1348        foreach my $exp (@$exps) {
     1349            my $iteration = $exp->{iteration};
     1350            my $command;
     1351            if ($mode eq "goto_cleaned") {
     1352                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1353            }
     1354            if ($mode eq "goto_scrubbed") {
     1355                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1356            }
     1357            if ($mode eq "goto_purged") {
     1358                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1359            }
     1360            $command .= " -dbname $dbname" if defined $dbname;
     1361            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1362                run(command => $command, verbose => $verbose);
     1363            unless ($success) {
     1364                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1365                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1366            }
     1367        }
    14151368    }
    14161369
    … …  
    14331386        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    14341387    }
    1435     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1388    $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
    14361389        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    14371390
    1438     $exps = parse_md_list($metadata) or
    1439         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1440 
    14411391    foreach my $exp (@$exps) {
    1442         my $path_base = $exp->{path_base};
    1443         my $iteration = $exp->{iteration};
    1444         my $class_id  = $exp->{class_id};
    1445 
    1446         my $status = 1;
    1447         # don't clean up unless the data needed to update is available
    1448         # goto_scrubbed now requires the config file to not be present
    1449         if ($mode eq "goto_cleaned") {
    1450             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1451            
    1452             unless ($ipprc->file_exists($config_file)) {
    1453                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1454                 $status = 0;
    1455             }
    1456         }
    1457         elsif ($mode eq "goto_scrubbed") {
    1458             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1459            
    1460             if ($ipprc->file_exists($config_file)) {
    1461                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1462                 $status = 0;
    1463             }
    1464         }
    1465         if ($status) {
    1466             my @files = ();
    1467 
    1468             if ($mode eq "goto_purged") {
    1469                 # additional files to remove for 'purge' mode
    1470                 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
    1471                 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
    1472 
    1473                 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
    1474                 addFilename (\@files, "PPIMAGE.STATS", $path_base);
    1475             }
    1476             # actual command to delete the files
    1477             $status = &delete_files (\@files);
    1478         }
    1479        
    1480         if ($status)  {
    1481             my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
    1482             if ($mode eq "goto_cleaned") {
    1483                 $command .= " -data_state cleaned";
    1484             }
    1485             if ($mode eq "goto_scrubbed") {
    1486                 $command .= " -data_state scrubbed";
    1487             }
    1488             if ($mode eq "goto_purged") {
    1489                 $command .= " -data_state purged";
    1490             }
    1491             $command .= " -dbname $dbname" if defined $dbname;
    1492             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1493                 run(command => $command, verbose => $verbose);
    1494             unless ($success) {
    1495                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1496                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1497             }
    1498         } else {
    1499             my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
    1500             $command .= " -dbname $dbname" if defined $dbname;
    1501            
    1502             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1503                 run(command => $command, verbose => $verbose);
    1504             unless ($success) {
    1505                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1506                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1507             }
    1508             exit $PS_EXIT_UNKNOWN_ERROR;
    1509         }
     1392        my $path_base = $exp->{path_base};
     1393        my $iteration = $exp->{iteration};
     1394        my $class_id  = $exp->{class_id};
     1395
     1396        my $status = 1;
     1397        # don't clean up unless the data needed to update is available
     1398        # goto_scrubbed now requires the config file to not be present
     1399        if ($mode eq "goto_cleaned") {
     1400            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1401
     1402            unless ($ipprc->file_exists($config_file)) {
     1403                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
     1404                $status = 0;
     1405            }
     1406        }
     1407        elsif ($mode eq "goto_scrubbed") {
     1408            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1409
     1410            if ($ipprc->file_exists($config_file)) {
     1411                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1412                $status = 0;
     1413            }
     1414        }
     1415        if ($status) {
     1416            my @files = ();
     1417
     1418            if ($mode eq "goto_purged") {
     1419                # additional files to remove for 'purge' mode
     1420                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
     1421                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
     1422
     1423                addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
     1424                addFilename (\@files, "PPIMAGE.STATS", $path_base);
     1425            }
     1426            # actual command to delete the files
     1427            $status = &delete_files (\@files);
     1428        }
     1429
     1430        if ($status)  {
     1431            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
     1432            if ($mode eq "goto_cleaned") {
     1433                $command .= " -data_state cleaned";
     1434            }
     1435            if ($mode eq "goto_scrubbed") {
     1436                $command .= " -data_state scrubbed";
     1437            }
     1438            if ($mode eq "goto_purged") {
     1439                $command .= " -data_state purged";
     1440            }
     1441            $command .= " -dbname $dbname" if defined $dbname;
     1442            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1443                run(command => $command, verbose => $verbose);
     1444            unless ($success) {
     1445                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1446                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1447            }
     1448        } else {
     1449            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
     1450            $command .= " -dbname $dbname" if defined $dbname;
     1451
     1452            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1453                run(command => $command, verbose => $verbose);
     1454            unless ($success) {
     1455                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1456                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
     1457            }
     1458            exit $PS_EXIT_UNKNOWN_ERROR;
     1459        }
    15101460    }
    15111461    # Check to see if we can mark the whole detRunSummary object as cleaned.
    15121462
    1513     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
     1463    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    15141464    $command .= " -dbname $dbname" if defined $dbname;
    15151465    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    15161466    unless ($success) {
    1517         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1518         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1467        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1468        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    15191469    }
    15201470    if (@$stdout_buf != 0) {
    1521         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1522             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1523         $exps = parse_md_list($metadata) or
    1524             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1525        
    1526         foreach my $exp (@$exps) {
    1527             my $iteration = $exp->{iteration};
    1528             my $command;
    1529             if ($mode eq "goto_cleaned") {
    1530                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1531             }
    1532             if ($mode eq "goto_scrubbed") {
    1533                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1534             }
    1535             if ($mode eq "goto_purged") {
    1536                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1537             }
    1538             $command .= " -dbname $dbname" if defined $dbname;
    1539             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1540                 run(command => $command, verbose => $verbose);
    1541             unless ($success) {
    1542                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1543                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1544             }
    1545         }
     1471        $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     1472            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1473
     1474        foreach my $exp (@$exps) {
     1475            my $iteration = $exp->{iteration};
     1476            my $command;
     1477            if ($mode eq "goto_cleaned") {
     1478                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1479            }
     1480            if ($mode eq "goto_scrubbed") {
     1481                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1482            }
     1483            if ($mode eq "goto_purged") {
     1484                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1485            }
     1486            $command .= " -dbname $dbname" if defined $dbname;
     1487            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1488                run(command => $command, verbose => $verbose);
     1489            unless ($success) {
     1490                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1491                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1492            }
     1493        }
    15461494    }
    15471495
    … …  
    15651513
    15661514    if (@$stdout_buf == 0) {
    1567         exit 0;
    1568     }
    1569     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     1515        exit 0;
     1516    }
     1517    $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
    15701518        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    15711519
    1572     $exps = parse_md_list($metadata) or
    1573         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1574 
    15751520    foreach my $exp (@$exps) {
    1576         my $exp_id = $exp->{exp_id};
    1577         my $iteration = $exp->{iteration};
    1578         my $path_base = $exp->{path_base};
    1579 
    1580         my $status = 1;
    1581         # don't clean up unless the data needed to update is available
    1582         # goto_scrubbed now requires the config file to not be present
    1583         if ($mode eq "goto_cleaned") {
    1584             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1585            
    1586             unless ($ipprc->file_exists($config_file)) {
    1587                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1588                 $status = 0;
    1589             }
    1590         }
    1591         elsif ($mode eq "goto_scrubbed") {
    1592             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1593            
    1594             if ($ipprc->file_exists($config_file)) {
    1595                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1596                 $status = 0;
    1597             }
    1598         }
    1599         if ($status) {
    1600             my @files = ();
    1601             # delete the temporary image datafiles
    1602             if ($mode eq "goto_purged") {
    1603                 # additional files to remove for 'purge' mode
    1604                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
    1605                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
    1606             }
    1607             # actual command to delete the files
    1608             $status = &delete_files (\@files);
    1609         }
    1610        
    1611         if ($status)  {
    1612             my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
    1613             if ($mode eq "goto_cleaned") {
    1614                 $command .= " -data_state cleaned";
    1615             }
    1616             if ($mode eq "goto_scrubbed") {
    1617                 $command .= " -data_state scrubbed";
    1618             }
    1619             if ($mode eq "goto_purged") {
    1620                 $command .= " -data_state purged";
    1621             }
    1622             $command .= " -dbname $dbname" if defined $dbname;
    1623             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1624                 run(command => $command, verbose => $verbose);
    1625             unless ($success) {
    1626                 print STDERR " residexp had an issue setting the state:? $success $error_code\n";
    1627                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1628                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1629             }
    1630         } else {
    1631             my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
    1632             $command .= " -dbname $dbname" if defined $dbname;
    1633            
    1634             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1635                 run(command => $command, verbose => $verbose);
    1636             unless ($success) {
    1637                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1638                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1639             }
    1640             exit $PS_EXIT_UNKNOWN_ERROR;
    1641         }
    1642     }
    1643     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1644 
    1645     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    1646     $command .= " -dbname $dbname" if defined $dbname;
    1647     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1648     unless ($success) {
    1649         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1650         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1651     }
    1652     if (@$stdout_buf != 0) {
    1653         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1654             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1655         $exps = parse_md_list($metadata) or
    1656             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1657        
    1658         foreach my $exp (@$exps) {
    1659             my $iteration = $exp->{iteration};
    1660             my $command;
    1661             if ($mode eq "goto_cleaned") {
    1662                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1663             }
    1664             if ($mode eq "goto_scrubbed") {
    1665                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1666             }
    1667             if ($mode eq "goto_purged") {
    1668                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1669             }
    1670             $command .= " -dbname $dbname" if defined $dbname;
    1671             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1672                 run(command => $command, verbose => $verbose);
    1673             unless ($success) {
    1674                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1675                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1676             }
    1677         }
    1678     }
    1679     exit 0;
    1680 }
    1681 if ($stage eq "detrend.resid.imfile") {
    1682 
    1683     die "--stage_id required for stage $stage\n" if !$stage_id;
    1684     ### select the imfiles for this entry
    1685 
    1686     # this stage uses 'dettool'
    1687     my $dettool = can_run('dettool') or die "Can't find dettool";
    1688 
    1689     # Get list of component imfiles
    1690     # XXX may need a different my_die for each stage
    1691     my $imfiles;                      # Array of component files
    1692     my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
    1693     $command .= " -dbname $dbname" if defined $dbname;
    1694     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1695     unless ($success) {
    1696         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1697         &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
    1698     }
    1699 
    1700     if (@$stdout_buf == 0) {
    1701         exit 0;
    1702     }
    1703 
    1704     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1705         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1706 
    1707     # extract the metadata for the files into a hash list
    1708     $imfiles = parse_md_list($metadata) or
    1709         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1710 
    1711     # loop over all of the imfiles, determine the path_base and class_id for each
    1712     foreach my $imfile (@$imfiles) {
    1713         my $class_id = $imfile->{class_id};
    1714         my $iteration = $imfile->{iteration};
    1715         my $exp_id = $imfile->{exp_id};
    1716         my $path_base = $imfile->{path_base};
     1521        my $exp_id = $exp->{exp_id};
     1522        my $iteration = $exp->{iteration};
     1523        my $path_base = $exp->{path_base};
     1524
    17171525        my $status = 1;
    1718 
    17191526        # don't clean up unless the data needed to update is available
    1720         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    1721         # goto_scrubbed now requires the config file to not exist.
    1722        
    1723         # Possibly not the correct config file, but simtest doesn't leave any around to check.
     1527        # goto_scrubbed now requires the config file to not be present
    17241528        if ($mode eq "goto_cleaned") {
    1725             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    1726 
    1727             unless ($ipprc->file_exists($config_file)) {
    1728                 print STDERR "skipping cleanup for $stage $stage_id $class_id "
    1729                     . " because config file is missing\n";
     1529            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1530
     1531            unless ($ipprc->file_exists($config_file)) {
     1532                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    17301533                $status = 0;
    17311534            }
    17321535        }
    1733         elsif ($mode eq "goto_scrubbed") {
    1734             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    1735 
    1736             if ($ipprc->file_exists($config_file)) {
    1737                 print STDERR "skipping scrubbed for $stage $stage_id $class_id "
    1738                     . " because config file is present\n";
    1739                 $status = 0;
    1740             }
    1741         }
    1742 
     1536        elsif ($mode eq "goto_scrubbed") {
     1537            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
     1538
     1539            if ($ipprc->file_exists($config_file)) {
     1540                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
     1541                $status = 0;
     1542            }
     1543        }
    17431544        if ($status) {
    1744             # array of actual filenames to delete
    17451545            my @files = ();
    1746 
    17471546            # delete the temporary image datafiles
    1748             addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    17491547            if ($mode eq "goto_purged") {
    17501548                # additional files to remove for 'purge' mode
    1751                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
    1752                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    1753                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    1754             }
    1755 
     1549                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     1550                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     1551            }
    17561552            # actual command to delete the files
    17571553            $status = &delete_files (\@files);
    … …  
    17591555
    17601556        if ($status)  {
    1761             my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
     1557            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
     1558            if ($mode eq "goto_cleaned") {
     1559                $command .= " -data_state cleaned";
     1560            }
     1561            if ($mode eq "goto_scrubbed") {
     1562                $command .= " -data_state scrubbed";
     1563            }
    17621564            if ($mode eq "goto_purged") {
    17631565                $command .= " -data_state purged";
    17641566            }
    1765             elsif ($mode eq "goto_cleaned") {
    1766                 $command .= " -data_state cleaned";
    1767             }
    1768             elsif ($mode eq "goto_scrubbed") {
    1769                 $command .= " -data_state scrubbed";
    1770             }
    1771 
    1772             $command .= " -dbname $dbname" if defined $dbname;
    1773 
    1774             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1775                     run(command => $command, verbose => $verbose);
    1776             unless ($success) {
     1567            $command .= " -dbname $dbname" if defined $dbname;
     1568            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1569                run(command => $command, verbose => $verbose);
     1570            unless ($success) {
     1571                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
    17771572                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    17781573                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    17791574            }
    17801575        } else {
    1781             my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
    1782             $command .= " -dbname $dbname" if defined $dbname;
    1783 
    1784             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1785                     run(command => $command, verbose => $verbose);
     1576            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
     1577            $command .= " -dbname $dbname" if defined $dbname;
     1578
     1579            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1580                run(command => $command, verbose => $verbose);
    17861581            unless ($success) {
    17871582                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    17881583                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    17891584            }
     1585            exit $PS_EXIT_UNKNOWN_ERROR;
    17901586        }
    17911587    }
    17921588    # Check to see if we can mark the whole detRunSummary object as cleaned.
    17931589
    1794     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
     1590    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    17951591    $command .= " -dbname $dbname" if defined $dbname;
    17961592    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    17971593    unless ($success) {
    1798         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1799         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1594        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1595        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    18001596    }
    18011597    if (@$stdout_buf != 0) {
    1802         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1803             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1804         my $exps = parse_md_list($metadata) or
    1805             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1806        
    1807         foreach my $exp (@$exps) {
    1808             my $iteration = $exp->{iteration};
    1809             my $command;
    1810             if ($mode eq "goto_cleaned") {
    1811                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1812             }
    1813             if ($mode eq "goto_scrubbed") {
    1814                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1815             }
    1816             if ($mode eq "goto_purged") {
    1817                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1818             }
    1819             $command .= " -dbname $dbname" if defined $dbname;
    1820             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1821                 run(command => $command, verbose => $verbose);
    1822             unless ($success) {
    1823                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1824                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1825             }
    1826         }
    1827     }
    1828 
    1829     exit 0;
    1830 }
    1831 if ($stage eq "detrend.resid.exp") {
    1832     die "--stage_id required for stage $stage\n" if !$stage_id;
    1833     # this stage uses 'camtool'
    1834     my $dettool = can_run('dettool') or die "Can't find dettool";
    1835 
    1836     # Get list of component imfiles
    1837     # XXX may need a different my_die for each stage
    1838     my $exps;                      # Array of component files
    1839     my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
    1840     $command .= " -dbname $dbname" if defined $dbname;
    1841     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1842     unless ($success) {
    1843         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1844         &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1845     }
    1846     # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
    1847     if (@$stdout_buf == 0) {
    1848         exit 0;
    1849     }
    1850     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1851         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1852 
    1853     $exps = parse_md_list($metadata) or
    1854         &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
    1855 
    1856     foreach my $exp (@$exps) {
    1857         my $exp_id = $exp->{exp_id};
    1858         my $iteration = $exp->{iteration};
    1859         my $path_base = $exp->{path_base};
    1860 #       my $class_id  = $exp->{class_id}
    1861         my $status = 1;
    1862         # don't clean up unless the data needed to update is available
    1863         # goto_scrubbed now requires the config file to not be present
    1864         if ($mode eq "goto_cleaned") {
    1865             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1866            
    1867             unless ($ipprc->file_exists($config_file)) {
    1868                 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
    1869                 $status = 0;
    1870             }
    1871         }
    1872         elsif ($mode eq "goto_scrubbed") {
    1873             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
    1874            
    1875             if ($ipprc->file_exists($config_file)) {
    1876                 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
    1877                 $status = 0;
    1878             }
    1879         }
    1880         if ($status) {
    1881             my @files = ();
    1882             # delete the temporary image datafiles
    1883             if ($mode eq "goto_purged") {
    1884                 # additional files to remove for 'purge' mode
    1885                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
    1886                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
    1887             }
    1888             # actual command to delete the files
    1889             $status = &delete_files (\@files);
    1890         }
    1891        
    1892         if ($status)  {
    1893             my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
    1894             if ($mode eq "goto_cleaned") {
    1895                 $command .= " -data_state cleaned";
    1896             }
    1897             if ($mode eq "goto_scrubbed") {
    1898                 $command .= " -data_state scrubbed";
    1899             }
    1900             if ($mode eq "goto_purged") {
    1901                 $command .= " -data_state purged";
    1902             }
    1903             $command .= " -dbname $dbname" if defined $dbname;
    1904             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1905                 run(command => $command, verbose => $verbose);
    1906             unless ($success) {
    1907                 print STDERR " residexp had an issue setting the state:? $success $error_code\n";
    1908                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1909                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1910             }
    1911         } else {
    1912             my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
    1913             $command .= " -dbname $dbname" if defined $dbname;
    1914            
    1915             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1916                 run(command => $command, verbose => $verbose);
    1917             unless ($success) {
    1918                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1919                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
    1920             }
    1921             exit $PS_EXIT_UNKNOWN_ERROR;
    1922         }
    1923     }
    1924     # Check to see if we can mark the whole detRunSummary object as cleaned.
    1925 
    1926     $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id";
    1927     $command .= " -dbname $dbname" if defined $dbname;
    1928     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    1929     unless ($success) {
    1930         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1931         &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1932     }
    1933     if (@$stdout_buf != 0) {
    1934         $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    1935             &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1936         $exps = parse_md_list($metadata) or
    1937             &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
    1938        
    1939         foreach my $exp (@$exps) {
    1940             my $iteration = $exp->{iteration};
    1941             my $command;
    1942             if ($mode eq "goto_cleaned") {
    1943                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
    1944             }
    1945             if ($mode eq "goto_scrubbed") {
    1946                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
    1947             }
    1948             if ($mode eq "goto_purged") {
    1949                 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
    1950             }
    1951             $command .= " -dbname $dbname" if defined $dbname;
    1952             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1953                 run(command => $command, verbose => $verbose);
    1954             unless ($success) {
    1955                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1956                 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
    1957             }
    1958         }
     1598        $exps = $mdcParser->parse_list(join "", @$stdout_buf) or
     1599            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
     1600
     1601        foreach my $exp (@$exps) {
     1602            my $iteration = $exp->{iteration};
     1603            my $command;
     1604            if ($mode eq "goto_cleaned") {
     1605                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
     1606            }
     1607            if ($mode eq "goto_scrubbed") {
     1608                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
     1609            }
     1610            if ($mode eq "goto_purged") {
     1611                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
     1612            }
     1613            $command .= " -dbname $dbname" if defined $dbname;
     1614            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1615                run(command => $command, verbose => $verbose);
     1616            unless ($success) {
     1617                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1618                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
     1619            }
     1620        }
    19591621    }
    19601622    exit 0;
    … …  
    19681630    my $files = shift; # reference to a list of files to unlink
    19691631#     my $test_verbose = 1;
    1970    
     1632
    19711633#     if ($test_verbose == 1) {
    1972 #       open(TMPLOG,">>/tmp/czw.cleanup.log");
    1973 #       flock(TMPLOG,2);
     1634#       open(TMPLOG,">>/tmp/czw.cleanup.log");
     1635#       flock(TMPLOG,2);
    19741636#     }
    19751637
    … …  
    19771639    foreach my $file (@$files) {
    19781640        print STDERR "unlinking $stage $stage_id $file";
    1979         unless ($ipprc->file_exists($file)) {
    1980             print STDERR "\t File not found\n";
    1981         }
    1982         else {
    1983             print STDERR "\n";
    1984         }
    1985 
    1986 #       if ($test_verbose == 1) {
    1987 #           print TMPLOG "$stage $stage_id $file";
    1988            
    1989 #           else {
    1990 #               print TMPLOG "\n";
    1991 #           }
    1992 #       }
     1641        unless ($ipprc->file_exists($file)) {
     1642            print STDERR "\t File not found\n";
     1643        }
     1644        else {
     1645            print STDERR "\n";
     1646        }
     1647
     1648#       if ($test_verbose == 1) {
     1649#           print TMPLOG "$stage $stage_id $file";
     1650
     1651#           else {
     1652#               print TMPLOG "\n";
     1653#           }
     1654#       }
    19931655
    19941656        $ipprc->file_delete($file);
    … …  
    19961658
    19971659#     if ($test_verbose == 1) {
    1998 #       flock(TMPLOG,8);
    1999 #       close(TMPLOG);
     1660#       flock(TMPLOG,8);
     1661#       close(TMPLOG);
    20001662#     }
    20011663
Note: See TracChangeset for help on using the changeset viewer.