- Timestamp:
- May 3, 2010, 8:45:22 AM (16 years ago)
- Location:
- branches/simmosaic_branches
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
ippScripts/scripts/magic_destreak_revert.pl (modified) (17 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/simmosaic_branches
- Property svn:mergeinfo changed
-
branches/simmosaic_branches/ippScripts/scripts/magic_destreak_revert.pl
r24856 r27839 8 8 use Sys::Hostname; 9 9 my $host = hostname(); 10 my $date = `date`; 10 11 print "\n\n"; 11 print "Starting script $0 on $host \n\n";12 print "Starting script $0 on $host at $date\n\n"; 12 13 13 14 use vars qw( $VERSION ); … … 30 31 my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); 31 32 my $isdestreaked = can_run('isdestreaked') or (warn "Can't find isdestreaked" and $missing_tools = 1); 33 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); 32 34 if ($missing_tools) { 33 35 warn("Can't find required tools."); … … 36 38 37 39 # Parse the command-line arguments 38 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $ inverse, $cam_path_base);40 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction); 39 41 my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum); 40 42 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); … … 48 50 'path_base=s' => \$path_base, # path_base of the input 49 51 'cam_path_base=s'=> \$cam_path_base, # path_base of the associated camera run 52 'cam_reduction=s'=> \$cam_reduction, # reduction class of the associated camera run 50 53 'outroot=s' => \$outroot, # "directory" for temporary images (may be nebulous) 51 54 'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels 52 55 'replace=s' => \$replace, # replace the input images with the results. 56 'bothways=s' => \$bothways, # run has inverse files (bothways diff) 53 57 'save-temps' => \$save_temps, # Save temporary files? 54 58 'dbname=s' => \$dbname, # Database name … … 78 82 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; 79 83 84 $cam_reduction = 'DEFAULT' if !defined $cam_reduction or ($cam_reduction eq "NULL"); 85 my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use 86 &my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; 87 80 88 my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' ); 81 89 &my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; … … 95 103 } elsif ($stage eq "diff") { 96 104 $skycell_id = $component; 97 } els e{105 } elsif ($stage ne "camera") { 98 106 &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 99 107 } … … 111 119 } 112 120 113 &my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 121 &my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 114 122 if ($replace eq "T") and !$nebulousInput; 115 123 … … 142 150 143 151 144 my ($image, $mask, $ch_mask, $weight, $sources );145 my ($bimage, $bmask, $bch_mask, $bweight, $bsources );152 my ($image, $mask, $ch_mask, $weight, $sources, $astrom); 153 my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); 146 154 147 155 if ($stage eq "raw") { … … 155 163 $bimage = $backup_path_base . ".fits"; 156 164 } elsif ($stage eq "chip") { 165 # Check to see if we're using dynamic masks 166 my $dynamicMasks; # Use dynamic masks? 167 { 168 # Get the PSASTRO recipe 169 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; 170 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 171 run(command => $command, verbose => $verbose); 172 unless ($success) { 173 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 174 &my_die("Unable to perform ppConfigDump: $error_code", $magic_ds_id, $component, 175 $PS_EXIT_CONFIG_ERROR); 176 } 177 my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or 178 &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, 179 $PS_EXIT_CONFIG_ERROR); 180 181 $dynamicMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK'); 182 } 183 157 184 # we use the mask output from the camera stage for input and replace 158 185 # the output of the chip stage with that mask as well. 159 186 $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); 160 $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);161 $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);162 187 $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); 188 189 if ($dynamicMasks) { 190 $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); 191 $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); 192 } else { 193 $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); 194 } 163 195 164 196 $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id); … … 167 199 $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); 168 200 $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id); 201 } elsif ($stage eq "camera") { 202 $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base); 203 $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); 169 204 } elsif ($stage eq "warp") { 170 205 $image = $ipprc->filename("PSWARP.OUTPUT", $path_base); … … 177 212 $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); 178 213 } elsif ($stage eq "diff") { 179 my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images214 my $name = "PPSUB.OUTPUT"; 180 215 $image = $ipprc->filename($name, $path_base); 181 216 $mask = $ipprc->filename("$name.MASK", $path_base); 182 217 $weight = $ipprc->filename("$name.VARIANCE", $path_base); 183 $sources = $ipprc->filename(" PPSUB.OUTPUT.SOURCES", $path_base);218 $sources = $ipprc->filename("$name.SOURCES", $path_base); 184 219 $bimage = $ipprc->filename($name, $backup_path_base); 185 220 $bmask = $ipprc->filename("$name.MASK", $backup_path_base); 186 221 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); 187 $bsources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $backup_path_base); 188 } 189 190 revert_file($image, $bimage) or 191 &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 192 193 # don't commit next the next couple of lines and see xxx below 194 # change to if 0 to revert chip runs without masks 195 if ($mask) { 196 if (!revert_file($mask, $bmask)) { 197 if ($stage eq 'chip') { 198 # XXX: don't fail if the mask file fails to revert. It probably doesn't exist 199 # Handle this properly 200 print STDERR "failure to revert mask file, ignored\n"; 201 } else { 202 &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 203 } 204 } 205 } 206 207 if ($ch_mask) { 208 # Handle this properly 209 if (!revert_file($ch_mask, $bch_mask)) { 210 # print STDERR "failure to revert chip mask file, ignored\n"; 211 &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 212 } 213 } 214 215 216 if ($weight) { 217 revert_file($weight, $bweight) or 218 &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 219 } 220 221 if ($sources) { 222 revert_file($sources, $bsources) or 223 &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 222 $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); 223 } 224 225 revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom); 226 227 if ($stage eq "diff" and $bothways) { 228 my $name = "PPSUB.INVERSE"; 229 $image = $ipprc->filename($name, $path_base); 230 $mask = $ipprc->filename("$name.MASK", $path_base); 231 $weight = $ipprc->filename("$name.VARIANCE", $path_base); 232 $sources = $ipprc->filename("$name.SOURCES", $path_base); 233 $bimage = $ipprc->filename($name, $backup_path_base); 234 $bmask = $ipprc->filename("$name.MASK", $backup_path_base); 235 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); 236 $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); 237 revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef); 224 238 } 225 239 226 240 # now revert the row in the database 227 241 { 228 my $command = "$magicdstool -revertdestreakedfile ";242 my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; 229 243 $command .= " -magic_ds_id $magic_ds_id"; 230 244 $command .= " -component $component"; … … 248 262 ### Pau. 249 263 264 sub revert_files { 265 my $replace = shift; 266 return if !$replace; 267 268 my $image = shift; 269 my $mask = shift; 270 my $weight = shift; 271 my $sources = shift; 272 my $astrom = shift; 273 my $bimage = shift; 274 my $bmask = shift; 275 my $bweight = shift; 276 my $bsources = shift; 277 my $bastrom = shift; 278 279 if ($image) { 280 revert_file($image, $bimage) or 281 &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 282 } 283 284 if ($mask) { 285 if (!revert_file($mask, $bmask)) { 286 &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 287 } 288 } 289 290 if ($ch_mask) { 291 if (!revert_file($ch_mask, $bch_mask)) { 292 &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 293 } 294 } 295 296 297 if ($weight) { 298 revert_file($weight, $bweight) or 299 &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 300 } 301 302 if ($sources) { 303 revert_file($sources, $bsources) or 304 &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 305 } 306 307 if ($astrom) { 308 revert_file($astrom, $bastrom) or 309 &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); 310 } 311 } 312 250 313 sub check_keyword 251 314 { … … 256 319 run(command => $command, verbose => $verbose); 257 320 321 if (!defined $error_code) { 322 print STDERR "run( $command ) returned undef error_code!!!!!\n"; 323 return $PS_EXIT_UNKNOWN_ERROR; 324 } 258 325 return $error_code >> 8; 259 326 } … … 263 330 my $original = shift; 264 331 my $backup = shift; 332 my $force = 1; # force deletion of backup files in nebulous 265 333 266 334 my $o_path = $ipprc->file_resolve($original); … … 288 356 289 357 if ($backup_is_not_destreaked) { 358 if ($original_is_not_destreaked) { 359 # this is unexpected result. throw an error so this can be checked manually 360 print STDERR "both files appear to not be destreaked\n"; 361 print STDERR "original: $original\n"; 362 print STDERR "backup: $backup\n"; 363 my $o_basename = basename($o_path); 364 my $b_basename = basename($b_path); 365 if (($o_basename =~ /SR_/) and !($b_basename =~ /SR_/)) { 366 print " basenames are as expected it is safe to revert\n"; 367 } else { 368 return 0; 369 } 370 } 290 371 # XXX TODO if stage is raw, check that backup has the correct size and md5sum 291 372 373 if ($verbose) { 374 print "ready to swap $backup\n"; 375 print " to $original\n"; 376 } 292 377 # Do we need to make this test? After the swap we're going to delete the file anyways 293 print "ready to swap $backup to $original\n" if $verbose;294 378 295 379 if (! $nebulous->swap($backup, $original)) { 296 print "failed to swap $backup to $original\n"; 380 print "failed to swap $backup\n"; 381 print " to $original\n"; 297 382 return 0; 298 383 } 299 384 300 if ($b_path and -e $b_path) {385 if ($b_path) { 301 386 print "ready to delete backup\n" if $verbose; 302 if (! $ipprc->file_delete($backup )) {387 if (! $ipprc->file_delete($backup, $force)) { 303 388 print "failed to delete $backup\n"; 304 389 return 0; … … 309 394 print "original uri: $original is not a destreaked file no need to swap backup_result: $backup_result\n"; 310 395 # delete the 'backup' (destreaked target) file if it exists 311 if ($b_path and -e $b_path) {312 if (! $ipprc->file_delete($backup )) {396 if ($b_path) { 397 if (! $ipprc->file_delete($backup, $force)) { 313 398 print "failed to delete $backup\n"; 314 399 return 0; … … 351 436 $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; 352 437 353 # we wouldn't be here unless an entry already exists in the database 354 # my $command = "$magicdstool -adddestreakedfile"; 355 # $command .= " -magic_ds_id $magic_ds_id"; 356 # $command .= " -component $component"; 357 # $command .= " -fault $exit_code"; 358 # $command .= " -dbname $dbname" if defined $dbname; 359 360 # # Add the processed file to the database 361 # unless ($no_update) { 362 # my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 363 # run(command => $command, verbose => $verbose); 364 # unless ($success) { 365 # carp("failed to update database for $magic_ds_id $component"); 366 # } 367 # } else { 368 # print "Skipping command: $command\n"; 369 # } 370 371 carp($msg); 438 # fault the whole run if one of the components fails to revert 439 my $command = "$magicdstool -updaterun"; 440 $command .= " -magic_ds_id $magic_ds_id"; 441 $command .= " -set_state failed_revert"; 442 $command .= " -dbname $dbname" if defined $dbname; 443 444 # Add the processed file to the database 445 unless ($no_update) { 446 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 447 run(command => $command, verbose => $verbose); 448 unless ($success) { 449 carp("failed to update database for $magic_ds_id $component"); 450 } 451 } else { 452 print "Skipping command: $command\n"; 453 } 454 372 455 exit $exit_code; 373 456 }
Note:
See TracChangeset
for help on using the changeset viewer.
