Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 25299) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 25299) @@ -0,0 +1,250 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +#my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +#my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +#my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +#my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op, $redirect, $save_temps, $run_state); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'add_id=s' => \$add_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'camroot|w=s' => \$camroot, # camera stage root name. + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'run-state=s' => \$run_state, # 'new' or 'update' + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --add_id --camera --outroot", + -exitval => 3, + ) unless + defined $exp_tag and + defined $add_id and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if (not defined $run_state) { $run_state = 'new'; } +if ($run_state eq 'update') { + $logDest .= '.update'; +} + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +&my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; + +#my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use +#&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $cmdflags; + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addtool: $error_code", $add_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); +} + +my $chipObjectsExist = 0; + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($run_state eq 'update') { + $traceDest .= '.update'; +} + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + +my $dtime_addstar = 0; + +unless ($no_op) { + if (defined $dvodbReal and ($run_state eq 'new')) { + ## XXX the camera analysis can either save the full set of + ## detections, or just the image metadata, in the dvodb + + ## get the addstar recipe for this camera and CAMERA reduction + my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); + + ## allow the dvodb to save only images, or the full detection set + my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -image" if $imagesOnly; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + } +} + + +# This needs to be updated when addtool is written. BROKEN +my $fpaCommand = "$addtool -add_id $add_id"; +if ($run_state eq 'new') { + $fpaCommand .= " -addprocessedexp"; + $fpaCommand .= " -path_base $outroot"; + $fpaCommand .= " $cmdflags"; + $fpaCommand .= " -dtime_addstar $dtime_addstar"; +} else { + $fpaCommand .= " -updaterun -state full"; +} +$fpaCommand .= " -dbname $dbname" if defined $dbname; + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $add_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + if (defined $add_id and not $no_update) { +# This needs to be updated when addtool is written. BROKEN + my $command = "$addtool -add_id $add_id"; + if ($run_state eq 'new') { + $command .= " -addprocessedexp"; + $command .= " -uri UNKNOWN"; + $command .= " -fault $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updateprocessedexp"; + $command .= " -fault $exit_code"; + } + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/camera_exp.pl =================================================================== --- trunk/ippScripts/scripts/camera_exp.pl (revision 25256) +++ trunk/ippScripts/scripts/camera_exp.pl (revision 25299) @@ -93,6 +93,6 @@ &my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; +#my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +#&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use @@ -205,5 +205,5 @@ } -my $dtime_addstar = 0; +#my $dtime_addstar = 0; unless ($no_op) { @@ -297,47 +297,47 @@ # run addstar on the output fpaObjects (if a DVO database is defined) - if (defined $dvodbReal and ($run_state eq 'new')) { - - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb - - ## get the addstar recipe for this camera and CAMERA reduction - $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $command; - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; - $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); - $command .= " $realFile"; - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } +# if (defined $dvodbReal and ($run_state eq 'new')) { + +# ## XXX the camera analysis can either save the full set of +# ## detections, or just the image metadata, in the dvodb + +# ## get the addstar recipe for this camera and CAMERA reduction +# $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; +# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); +# } +# my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); + +# ## allow the dvodb to save only images, or the full detection set +# my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + +# # XXX this construct requires the user to have a valid .ptolemyrc +# # XXX which in turn points at ippconfig/dvo.site +# # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) +# # XXX this needs to be converted to addstar_client... + +# my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar +# my $command; +# $command = "$addstar -D CAMERA $camdir -update"; +# $command .= " -image" if $imagesOnly; +# $command .= " -D CATDIR $dvodbReal"; + +# my $realFile = $ipprc->file_resolve($fpaObjects); +# $command .= " $realFile"; + +# my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); +# } +# $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script +# } } } @@ -353,5 +353,5 @@ $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= " -dtime_script $dtime_script"; - $fpaCommand .= " -dtime_addstar $dtime_addstar"; +# $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { $fpaCommand .= " -updaterun -state full"; Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25256) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25299) @@ -60,8 +60,8 @@ -my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1, - detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1, - detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1, - detrend.resid.imfile => 1, detrend.resid.exp => 1 ); +my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, + "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, + "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); unless ($stages{$stage}) { die "unknown stage $stage for ipp_cleanup.pl\n"; @@ -119,7 +119,8 @@ # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - print STDERR "CHIP: CONFIG_FILE : $config_file\n"; + if (!$config_file or ! -e $config_file) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " @@ -128,4 +129,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } if ($status) { @@ -160,7 +170,12 @@ if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -218,4 +233,5 @@ my $status = 1; # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); @@ -225,4 +241,12 @@ $status = 0; } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -246,5 +270,5 @@ } if ($mode eq "goto_scrubbed") { - $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; + $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; } if ($mode eq "goto_purged") { @@ -311,4 +335,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } if ($status) { # delete the temporary image datafiles @@ -337,7 +370,11 @@ if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -395,6 +432,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -404,4 +440,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } } if ($status) { @@ -428,6 +473,10 @@ if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; } $command .= " -dbname $dbname" if defined $dbname; @@ -485,6 +534,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -494,4 +542,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -529,10 +586,15 @@ if ($status) { my $command = "$difftool -diff_id $stage_id"; -# my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -545,5 +607,5 @@ } else { my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; -# my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; @@ -560,7 +622,1208 @@ } if ($stage eq 'fake') { - die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; + print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; + exit(0); + die "--stage_id required for stage fake\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'chiptool' + my $faketool = can_run('faketool') or die "Can't find faketool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + + # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for fakeRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); + addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$faketool -fake_id $stage_id -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedimfile"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -tocleanedimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } + } + exit 0; + } -# fake : faketool : -pendingcleanupimfile (loop over imfiles) +# Detrend stages +if ($stage eq "detrend.process.imfile") { + + die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find chiptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $exp_id = $imfile->{exp_id}; + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.process.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; #silently abort. I need to fix this for propers + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $exp_id = $exp->{exp_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + # I can't find anything to put here + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + # since 'camera' has only a single imfile, we can just update the run + my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.stack.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + my $stacks; # Array reference of component files + my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $stacks = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $stack (@{ $stacks }) { + # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. + my $path_base = $stack->{uri}; + my $iteration = $stack->{iteration}; + my $class_id = $stack->{class_id}; + + $path_base =~ s/\.fits$//; # That should do it? + + my $status = 1; + +# if ($mode eq "goto_cleaned") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); + +# $config_file =~ s%^file://%%; +# if (!$config_file or ! -e $config_file) { +# print STDERR "skipping cleanup for $stage $stage_id $path_base" . +# " because config file is missing\n"; +# $status = 0; +# } +# $config_file = 'file://' . $config_file; +# } +# elsif ($mode eq "goto_scrubbed") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); +# $config_file =~ s%^file://%%; +# if ($config_file and -e $config_file) { +# print STDERR "skipping scrubbed for $stage $stage_id $path_base" . +# " because config file is present\n"; +# $status = 0; +# } +# } + + if ($status) { + # delete the temporary image datafiles + # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. + # I understand that they all point to the same filename right now, but that may not be true in + # the future. + addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); + + + addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode +# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -updatestacked -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updatestacked -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updatestacked -data_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.normstat.imfile") { + print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { +# my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + if ($status) { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR "WTF: $success TTT $error_code QQQ\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; + +} +if ($stage eq "detrend.norm.imfile") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); + + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); + addFilename (\@files, "PPIMAGE.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.norm.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.resid.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $iteration = $imfile->{iteration}; + my $exp_id = $imfile->{exp_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for $stage $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.resid.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; +# my $class_id = $exp->{class_id} + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} + die "ipp_cleanup.pl -stage $stage not yet implemented\n"; @@ -570,9 +1833,26 @@ my $files = shift; # reference to a list of files to unlink +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); + # this script is, of course, very dangerous. foreach my $file (@$files) { print STDERR "unlinking $file\n"; +# print TMPLOG "$stage $stage_id $file"; +# my $ff = $file; +# $ff =~ s%^file://%%; +# unless (-e $ff) { +# print TMPLOG "\t File not found\n"; +# } +# else { +# print TMPLOG "\n"; +# } + $ipprc->file_delete($file); } + +# flock(TMPLOG,8); +# close(TMPLOG); + return 1; } Index: trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- trunk/ippScripts/scripts/stack_skycell.pl (revision 25256) +++ trunk/ippScripts/scripts/stack_skycell.pl (revision 25299) @@ -65,4 +65,6 @@ my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$| = 1; +print "I've set up: $stack_id\n"; # XXX camera is not known here; cannot use filerules... @@ -107,4 +109,6 @@ &my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless scalar @$files >= 2; + +print "I've loaded my inputs: $stack_id\n"; # Parse the list of input files to get the tesselation, skycell identifiers and camera @@ -146,4 +150,6 @@ } +print "I've configured everything: $stack_id\n"; + # Generate MDC file with the inputs my $tess_base = basename($tess_id); @@ -180,4 +186,6 @@ print $listFile "END\n\n"; } + +print "I've checked everything: $stack_id\n"; # Get the output filenames @@ -200,4 +208,5 @@ my $cmdflags; + # Perform stacking @@ -265,5 +274,6 @@ # &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); } - + + print "I've stacked $listName: $stack_id\n"; } @@ -298,4 +308,6 @@ } +print "I've updated teh database: $stack_id\n"; + my $my_die_called = 0; @@ -372,4 +384,5 @@ } +print "I've reached the end of processing with a code of $?: $stack_id\n"; END {