Changeset 42560
- Timestamp:
- Dec 3, 2023, 2:57:22 PM (3 years ago)
- Location:
- branches/eam_branches/ipp-20230313/ippScripts
- Files:
-
- 3 edited
-
Build.PL (modified) (1 diff)
-
scripts/stack_skycell.pl (modified) (1 diff)
-
scripts/xcstack_skycell.pl (modified) (15 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20230313/ippScripts/Build.PL
r42388 r42560 156 156 scripts/remote_md5sum.pl 157 157 scripts/fpcamera_exp.pl 158 scripts/xcstack_skycell.pl 158 159 )], 159 160 dist_abstract => 'Scripts for running the Pan-STARRS IPP', -
branches/eam_branches/ipp-20230313/ippScripts/scripts/stack_skycell.pl
r42099 r42560 185 185 my $recipe_psphot = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot 186 186 my $recipe_ppstats = 'STACKSTATS'; 187 if ($ 188 run_state eq 'update') { 187 if ($run_state eq 'update') { 189 188 $recipe_ppstats = 'WARPSTATS'; 190 189 } -
branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl
r42548 r42560 40 40 } 41 41 42 my ($xcstack_id, $dbname, $ outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps, $delete_convolved_images);42 my ($xcstack_id, $dbname, $camera, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); 43 43 GetOptions( 44 'xcstack_id|d=s' => \$xcstack <_id,# xcstack identifier45 'dbname|d=s' => \$dbname, # Database name46 ' outroot=s' => \$outroot, # Output rootname47 ' run-state=s' => \$run_state,48 ' debug' => \$debug, # Print to stdout49 ' threads=s' => \$threads, # Number of threads to use for ppStack50 ' reduction=s' => \$reduction, # Reduction class51 ' verbose' => \$verbose, # Print to stdout52 ' no-update' => \$no_update, # Don't update the database?53 'no- op' => \$no_op, # Don't do any operations?54 ' redirect-output' => \$redirect,55 ' save-temps' => \$save_temps, # Save temporary files?56 ' delete-convolved' => \$delete_convolved_images,44 'xcstack_id|d=s' => \$xcstack_id, # xcstack identifier 45 'dbname|d=s' => \$dbname, # Database name 46 'camera|c=s' => \$camera, # Camera name 47 'outroot=s' => \$outroot, # Output root name 48 'run-state=s' => \$run_state, # new or update? 49 'debug' => \$debug, # Print to stdout 50 'threads=s' => \$threads, # Number of threads to use for ppStack 51 'reduction=s' => \$reduction, # Reduction class 52 'verbose' => \$verbose, # Print to stdout 53 'no-update' => \$no_update, # Don't update the database? 54 'no-op' => \$no_op, # Don't do any operations? 55 'redirect-output' => \$redirect, # send output to the logfile 56 'save-temps' => \$save_temps, # Save temporary files? 57 57 ) or pod2usage( 2 ); 58 58 59 59 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; 60 60 pod2usage( 61 -msg => "Required options: --xcstack_id --outroot --run-state ",61 -msg => "Required options: --xcstack_id --outroot --run-state --dbname --camera", 62 62 -exitval => 3, 63 ) unless defined $xcstack_id 64 and defined $outroot 65 and defined $run_state; 63 ) unless defined $xcstack_id and defined $outroot and defined $run_state and defined $dbname and defined $camera; 66 64 67 65 # lists of file rules which we expect to produce output when convolving 68 66 my @outputList = qw( 69 PP XCSTACK.OUTPUT70 PP XCSTACK.OUTPUT.MASK71 PP XCSTACK.OUTPUT.VARIANCE72 PP XCSTACK.OUTPUT.EXP73 PP XCSTACK.OUTPUT.EXPNUM74 PP XCSTACK.OUTPUT.EXPWT75 PP XCSTACK.TARGET.PSF76 67 PPSTACK.UNCONV 68 PPSTACK.UNCONV.MASK 69 PPSTACK.UNCONV.VARIANCE 70 PPSTACK.UNCONV.EXP 71 PPSTACK.UNCONV.EXPNUM 72 PPSTACK.UNCONV.EXPWT 73 PPSTACK.CONFIG 74 ); 77 75 78 76 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $xcstack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration 79 77 $| = 1; 80 78 print "I've set up: $xcstack_id\n"; 79 80 $ipprc->define_camera($camera); 81 81 82 82 # XXX camera is not known here; cannot use filerules... … … 96 96 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_XCSTACK); 97 97 98 # temporary file to list the inputs (in metadata config format) 99 my ($listFile, $listName) = tempfile("/tmp/xcstk.$xcstack_id.list.XXXX", UNLINK => !$save_temps ); 100 my $num = 0; 101 98 102 # Get list of cameras for this xcstack 99 103 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 100 my $cameras; 101 { 102 my $command = "$xcstacktool -inputcameras -xcstack_id $xcstack_id"; 103 $command .= " -dbname $dbname" if defined $dbname; 104 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 105 unless ($success) { 106 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 107 &my_die("Unable to perform xcstacktool -inputcameras: $error_code", $xcstack_id, $error_code); 108 } 109 110 if (@$stdout_buf == 0) { 111 &my_die("No input cameras selected", $xcstack_id, $PS_EXIT_PROG_ERROR); 112 } 113 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 114 $cameras = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); 115 } 116 117 &my_die("no cameras listed for xcstack", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar @$files > 0; 118 104 my $cameras = &run_command_ipptool ("$xcstacktool -dbname $dbname -inputcameras -xcstack_id $xcstack_id"); 105 &my_die("no cameras listed for xcstack", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar @$cameras > 0; 119 106 print "I've loaded my cameras: $xcstack_id\n"; 120 107 121 108 # Gather the list of input stacks from each of the cameras. There may be multiple stacks per camera 122 foreach my $camera (@$cameras) {109 foreach my $cameraData (@$cameras) { 123 110 # Get list of components for xcstacking 124 111 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 125 112 126 $camera_name = $camera->{camera};113 my $camera_name = $cameraData->{camera}; 127 114 128 my $inputs; 129 { 130 my $command = "$xcstacktool -inputskyfile -xcstack_id $xcstack_id -xcamera $camera_name"; 131 $command .= " -dbname $dbname" if defined $dbname; 132 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 133 unless ($success) { 134 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 135 &my_die("Unable to perform xcstacktool -inputskyfile: $error_code", $xcstack_id, $error_code); 136 } 137 138 if (@$stdout_buf == 0) { 139 &my_die("No input skyfiles selected", $xcstack_id, $PS_EXIT_PROG_ERROR); 140 } 141 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 142 $inputs = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); 143 } 115 my $inputs = &run_command_ipptool ("$xcstacktool -dbname $dbname -inputskyfile -xcstack_id $xcstack_id -xcamera $camera_name"); 116 &my_die("no inputs for camera $camera_name", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar @$inputs > 0; 144 117 145 118 foreach my $input (@$inputs) { 146 $stack_id = $input->{stack_id}; 147 148 my $files; 149 { 150 my $command = "$stacktool -sumskyfile -dbname $camera_name -stack_id $stack_id"; 151 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 152 run(command => $command, verbose => $verbose); 153 unless ($success) { 154 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 155 &my_die("Unable to perform xcstacktool -inputskyfile: $error_code", $xcstack_id, $error_code); 156 } 157 158 if (@$stdout_buf == 0) { 159 &my_die("No input skyfiles selected", $xcstack_id, $PS_EXIT_PROG_ERROR); 160 } 161 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 162 &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 163 $files = parse_md_list($metadata) or 164 &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); 165 } 166 167 foreach my $file (@$files) { 168 # check for the input files 169 my $image = $ipprc->filename("PPSTACK.OUTPUT", $file->{path_base} ); # Image name 170 my $mask = $ipprc->filename( "PPSTACK.OUTPUT.MASK", $file->{path_base} ); # Mask name 171 my $weight = $ipprc->filename( "PPSTACK.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name 172 my $sources = $ipprc->filename("PPSTACK.OUTPUT.SOURCES", $file->{path_base}); # Sources name 173 174 my $have_image = $ipprc->file_exists( $image ); 175 my $have_mask = $ipprc->file_exists( $mask ); 176 my $have_weight = $ipprc->file_exists( $weight ); 177 my $have_sources = $ipprc->file_exists( $sources ); 178 179 if ($skip_missing_inputs) { 180 my $missing_inputs = 0; 181 unless ($have_image) { print "WARNING: MISSING INPUT $image \n"; $missing_inputs = 1; } 182 unless ($have_mask) { print "WARNING: MISSING INPUT $mask \n"; $missing_inputs = 1; } 183 unless ($have_weight) { print "WARNING: MISSING INPUT $weight \n"; $missing_inputs = 1; } 184 unless ($have_sources){ print "WARNING: MISSING INPUT $sources\n"; $missing_inputs = 1; } 185 if ($missing_inputs) { next; } 186 } else { 187 &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_image; 188 &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; 189 &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; 190 &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; 191 } 192 193 print $listFile "INPUT$num\tMETADATA\n"; 194 195 print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; 196 print $listFile "\tMASK\tSTR\t" . $mask . "\n"; 197 print $listFile "\tVARIANCE\tSTR\t" . $weight . "\n"; 198 print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve; 199 print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; 200 print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel ); 201 202 print $listFile "END\n\n"; 203 $num++; 204 } 119 my $stack_id = $input->{stack_id}; 120 121 # check for the input files 122 my $image = $ipprc->filename("PPSTACK.UNCONV", $input->{path_base}); # Image name 123 my $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $input->{path_base}); # Mask name 124 my $weight = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $input->{path_base}); # Weight name 125 my $sources = $ipprc->filename("PPSTACK.OUTPUT.SOURCES", $input->{path_base}); # Sources name 126 127 my $have_image = $ipprc->file_exists( $image ); 128 my $have_mask = $ipprc->file_exists( $mask ); 129 my $have_weight = $ipprc->file_exists( $weight ); 130 my $have_sources = $ipprc->file_exists( $sources ); 131 132 &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_image; 133 &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; 134 &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; 135 &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; 136 137 print $listFile "INPUT$num\tMETADATA\n"; 138 139 print $listFile " IMAGE STR " . $image . "\n"; 140 print $listFile " MASK STR " . $mask . "\n"; 141 print $listFile " VARIANCE STR " . $weight . "\n"; 142 print $listFile " SOURCES STR " . $sources . "\n"; 143 print $listFile "END\n\n"; 144 $num++; 205 145 } 206 146 } 207 147 close($listFile); 208 148 209 &my_die("Xcstack list contains less than two elements", $xcstack_id, $PS_EXIT_DATA_ERROR) unless 210 scalar @$files >= 2; 149 &my_die("input list contains less than two elements", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar $num >= 2; 211 150 212 151 print "I've loaded my inputs: $xcstack_id\n"; 213 214 &my_die("Can't find camera", $xcstack_id, $PS_EXIT_SYS_ERROR) unless defined $camera;215 $ipprc->define_camera($camera);216 152 217 153 print "I've configured everything: $xcstack_id\n"; … … 219 155 # Recipes to use based on reduction class 220 156 $reduction = 'DEFAULT' unless defined $reduction; 221 my $recipe_ppStack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use for ppStack 222 unless ($recipe_ppStack) { 223 &my_die("Couldn't find selected reduction for STACK_PPSTACK: $reduction\n", $xcstack_id, $PS_EXIT_CONFIG_ERROR); 224 } 225 226 # XXX maybe have different reduction class name (XCSTACK?) 227 my $recipe; 228 { 229 my $command = "$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"; 230 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 231 run(command => $command, verbose => $verbose); 232 unless ($success) { 233 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 234 &my_die("Unable to perform ppConfigDump: $error_code", $xcstack_id, $error_code); 235 } 236 $recipe = $mdcParser->parse(join "", @$stdout_buf) or 237 &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 238 } 239 my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images 240 my $output_logflux = metadataLookupBool($recipe, 'OUTPUT.LOGFLUX'); # change filerules to produce logflux compressed output images. 241 my $output_deepexp = metadataLookupBool($recipe, 'OUTPUT.DEEPEXP'); # change filerules to produce exptime map with more depth 242 my $replicate_outputs = (defined($neb) and metadataLookupBool($recipe, 'OUTPUT.REPLICATE')); # replicate output images 243 my $skip_missing_inputs = metadataLookupBool($recipe, 'SKIP.MISSING.INPUTS'); # ignore missing inputs 157 my $recipe_ppStack = $ipprc->reduction($reduction, 'XCSTACK_PPSTACK'); # Recipe to use for ppStack 158 my $recipe_ppstats = 'STACKSTATS'; 159 &my_die("Couldn't find selected reduction for XCSTACK_PPSTACK: $reduction\n", $xcstack_id, $PS_EXIT_CONFIG_ERROR) unless ($recipe_ppStack); 160 161 my $recipe = &run_command ("$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"); 162 163 my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images 164 my $output_logflux = metadataLookupBool($recipe, 'OUTPUT.LOGFLUX'); # change filerules to produce logflux compressed output images. 165 my $output_deepexp = metadataLookupBool($recipe, 'OUTPUT.DEEPEXP'); # change filerules to produce exptime map with more depth 166 my $replicate_outputs = (defined($neb) and metadataLookupBool($recipe, 'OUTPUT.REPLICATE')); # replicate output images 244 167 if ($output_nocomp and $output_logflux) { 245 168 &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$xcstack_id, $PS_EXIT_CONFIG_ERROR); … … 250 173 my $stack_type = metadataLookupStr($recipe, 'STACK.TYPE'); 251 174 &my_die("STACK.TYPE not found in recipe. Check config.",$xcstack_id, $PS_EXIT_CONFIG_ERROR) unless $stack_type; 252 ## XXX not sure how stack type is used in ppStack 253 254 ## XXX first ask for the list of cameras relevant to this stack 255 256 # Generate MDC file with the inputs 257 my $tess_base = basename($tess_id); 258 my ($listFile, $listName) = tempfile("/tmp/$tess_base.$skycell_id.stk.$xcstack_id.list.XXXX", UNLINK => !$save_temps ); 259 my $num = 0; 260 my $inputSources; # Sources to use as stamps 261 foreach my $file (@$files) { 262 if ($file->{ignored}) { next; } 263 264 # check for the input files 265 my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name 266 my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name 267 my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name 268 my $psf = $convolve ? $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ) : undef; # PSF name 269 my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name 270 my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base}); 271 272 my $have_image = $ipprc->file_exists( $image ); 273 my $have_mask = $ipprc->file_exists( $mask ); 274 my $have_weight = $ipprc->file_exists( $weight ); 275 my $have_sources = $ipprc->file_exists( $sources ); 276 my $have_psf; 277 if ($convolve) { $have_psf = $ipprc->file_exists( $psf ); } 278 279 if ($skip_missing_inputs) { 280 my $missing_inputs = 0; 281 unless ($have_image) { print "WARNING: MISSING INPUT $image \n"; $missing_inputs = 1; } 282 unless ($have_mask) { print "WARNING: MISSING INPUT $mask \n"; $missing_inputs = 1; } 283 unless ($have_weight) { print "WARNING: MISSING INPUT $weight \n"; $missing_inputs = 1; } 284 unless ($have_sources){ print "WARNING: MISSING INPUT $sources\n"; $missing_inputs = 1; } 285 if ($convolve) { unless ($have_psf) { print "WARNING: MISSING INPUT $psf\n"; $missing_inputs = 1;}} 286 if ($missing_inputs) { next; } 287 } else { 288 &my_die("Image $image does not exist", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_image; 289 &my_die("Mask $mask does not exist", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; 290 &my_die("Weight $weight does not exist", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; 291 &my_die("Sources $sources does not exist", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; 292 &my_die("PSF $psf does not exist", $xcstack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $have_psf); 293 } 294 295 print $listFile "INPUT$num\tMETADATA\n"; 296 297 print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; 298 print $listFile "\tMASK\tSTR\t" . $mask . "\n"; 299 print $listFile "\tVARIANCE\tSTR\t" . $weight . "\n"; 300 print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve; 301 print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; 302 print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel ); 303 304 print $listFile "END\n\n"; 305 $num++; 306 } 307 close($listFile); 308 309 print "I've checked everything: $xcstack_id\n"; 175 # stack_type must be one of: DEEP_STACK, NIGHTLY_STACK, IQ_STACK. this does not affect the analysis, but is added to the header 310 176 311 177 # Prepare the output files 312 178 my @outputFiles; 313 prepare_outputs(\@outputFiles, \@outputList, $outroot) if $convolve; 314 prepare_outputs(\@outputFiles, \@outputListPhotom, $outroot) if $photometry; 315 prepare_outputs(\@outputFiles, \@outputListUnconv, $outroot); # unconvolved output always go here 316 317 # we need the output image name for the database 318 my $outputName = $outputFiles[0]; 179 prepare_outputs(\@outputFiles, \@outputList, $outroot); # this list 319 180 320 181 ## use an explicit xcstack name for psphot output objects … … 344 205 $command .= " -stats $outputStats" if $do_stats; 345 206 $command .= " -recipe PPSTACK $recipe_ppStack"; 346 $command .= " -recipe PPSUB $recipe_ppSub";347 $command .= " -recipe PSPHOT $recipe_psphot";348 207 $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;; 349 $command .= " -stack-type $xcstack_type"; 350 $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; 351 $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; 352 $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; 353 $command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS"; 354 $command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL"; 355 $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID"; 208 $command .= " -stack-type $stack_type"; 356 209 if ($output_nocomp) { 357 210 $command .= " -F PPSTACK.OUTPUT PPSTACK.OUTPUT.NOCOMP"; … … 383 236 $command .= " -source_id $source_id" if defined $source_id; 384 237 385 $command .= " -xcstack_id $xcstack_id" if defined $xcstack_id;386 $command .= " -skycell_id $skycell_id" if defined $skycell_id;387 $command .= " -tess_id $tess_base" if defined $tess_base;388 389 238 if ($run_state eq 'new') { 390 239 $command .= " -dumpconfig $configuration"; … … 394 243 395 244 245 # cannot use run_command here (no metadata output to parse) 396 246 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 397 247 run(command => $command, verbose => $verbose); … … 401 251 } 402 252 403 my $quality; # Quality flag 253 # extract the stats flags, and capture the -quality flag 254 my $quality; 404 255 if ($do_stats) { 405 256 my $outputStatsReal = $ipprc->file_resolve($outputStats); … … 408 259 # measure stats 409 260 $command = "$ppStatsFromMetadata $outputStatsReal - XCSTACK_SKYCELL"; 410 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 411 run(command => $command, verbose => $verbose); 261 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 412 262 unless ($success) { 413 263 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 414 264 &my_die("Unable to perform ppStatsFromMetadata: $error_code", $xcstack_id, $error_code); 415 265 } 416 foreach my $line (@$stdout_buf) { 417 $cmdflags .= " $line"; 418 } 266 foreach my $line (@$stdout_buf) { $cmdflags .= " $line"; } 419 267 chomp $cmdflags; 420 268 … … 423 271 424 272 if (!$quality) { 425 426 427 273 check_outputs(\@outputFiles, $replicate_outputs); 428 429 if (($convolve)&&($delete_convolved_images)) { # We made convolved products, but do not wish to keep them anymore430 my @products_to_clear = ('PPSTACK.OUTPUT',431 'PPSTACK.OUTPUT.MASK',432 'PPSTACK.OUTPUT.VARIANCE',433 'PPSTACK.OUTPUT.EXP',434 'PPSTACK.OUTPUT.EXPNUM',435 'PPSTACK.OUTPUT.EXPWT');436 foreach my $product (@products_to_clear) {437 my $file = $ipprc->filename($product,$outroot,$skycell_id);438 print "Deleting unwanted convolved product: $file\n";439 my $error= $ipprc->kill_file($file);440 if ($error_code) {441 print "Failed to delete unwanted convolved product: $error_code\n";442 }443 }444 }445 446 274 } 447 275 448 276 print "I've stacked $listName: $xcstack_id\n"; 449 277 } 450 451 delete_temps() unless ($save_temps);452 278 453 279 unless ($no_update) { … … 459 285 if ($run_state eq 'new') { 460 286 $mode = "-addsumskyfile"; 461 $command .= " - uri $outputName -path_base $outroot";287 $command .= " -path_base $outroot"; 462 288 $command .= " -hostname $host" if defined $host; 463 289 $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); … … 481 307 print "I've updated the database: $xcstack_id\n"; 482 308 483 484 my $my_die_called = 0; 309 print "I've reached the end of processing with a code of $?: $xcstack_id\n"; 310 311 END { 312 my $exit = $?; 313 system("sync") == 0 or die "failed to execute sync: $!"; 314 $? = $exit; 315 } 316 485 317 sub my_die 486 318 { … … 493 325 carp($msg); 494 326 495 unless ($my_die_called) {496 $my_die_called = 1;497 delete_temps() unless ($save_temps);498 }499 500 327 if (defined $xcstack_id and not $no_update) { 501 328 my $command = "$xcstacktool -xcstack_id $xcstack_id -fault $exit_code"; 502 329 if ($run_state eq 'new') { 503 330 $command .= " -addsumskyfile"; 331 $command .= " -path_base $outroot" if defined $outroot; 504 332 $command .= " -hostname $host" if defined $host; 505 $command .= " -path_base $outroot" if defined $outroot;506 333 $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); 507 334 } else { … … 514 341 } 515 342 516 # Ensure temporary images are deleted517 sub delete_temps518 {519 unless ($temp_images_exist) {520 print "No temporary images yet generated\n";521 return 1;522 }523 524 print "Ensuring temporary images are deleted.\n";525 526 my $temp_delete = defined($recipe) and metadataLookupBool($recipe, 'TEMP.DELETE'); # Delete temp files?527 if ($temp_delete) {528 my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, 'TEMP.DIR') or &my_die("Unable to find temporary directory in site configuration", $xcstack_id, $PS_EXIT_CONFIG_ERROR); # Directory with temporary files529 my $temp_image = metadataLookupStr($recipe, 'TEMP.IMAGE'); # Suffix for image530 my $temp_mask = metadataLookupStr($recipe, 'TEMP.MASK'); # Suffix for mask531 my $temp_weight = metadataLookupStr($recipe, 'TEMP.VARIANCE'); # Suffix for weight532 my $temp_root = basename($outroot); # Root name for temporary files533 534 if (not defined $temp_dir or not defined $temp_image or not defined $temp_mask or535 not defined $temp_weight) {536 &my_die("Unable to find details for temporary images.", $xcstack_id, $PS_EXIT_CONFIG_ERROR);537 }538 539 for (my $i = 0; $i < $num; $i++) {540 foreach my $ext ( $temp_image, $temp_mask, $temp_weight ) {541 my $file = $temp_dir . '/' . $temp_root . '.' . $i . '.' . $ext;542 unlink $file if -e $file;543 }544 }545 }546 }547 343 sub prepare_output 548 344 { … … 614 410 # Get list of cameras for this xcstack 615 411 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 616 { 617 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 618 unless ($success) { 619 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 620 &my_die("Unable to perform $command: $error_code"); 621 } 622 623 if (@$stdout_buf == 0) { 624 &my_die("No input cameras selected", $xcstack_id, $PS_EXIT_PROG_ERROR); 625 } 626 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 627 $cameras = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); 628 } 629 630 631 632 } 633 print "I've reached the end of processing with a code of $?: $xcstack_id\n"; 634 635 END { 636 my $exit = $?; 637 system("sync") == 0 or die "failed to execute sync: $!"; 638 $? = $exit; 639 } 640 412 413 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 414 unless ($success) { 415 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 416 &my_die("Unable to perform $command", $xcstack_id, $error_code); 417 } 418 419 # is the result allowed to be empty? 420 if (@$stdout_buf == 0) { 421 &my_die("No output from command $command", $xcstack_id, $PS_EXIT_PROG_ERROR); 422 } 423 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); 424 425 return $metadata; 426 } 427 428 sub run_command_ipptool { 429 430 my $command = shift; 431 432 my $metadata = &run_command ($command); 433 my $result = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); 434 return $result; 435 } 641 436 642 437 __END__ 438
Note:
See TracChangeset
for help on using the changeset viewer.
