- Timestamp:
- Dec 10, 2012, 4:35:46 PM (14 years ago)
- Location:
- branches/czw_branch/20120906
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
ippScripts/scripts/ipp_cleanup.pl (modified) (32 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/czw_branch/20120906
- Property svn:mergeinfo changed
-
branches/czw_branch/20120906/ippScripts/scripts/ipp_cleanup.pl
r33664 r34786 18 18 19 19 # Parse the command-line arguments 20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); 20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all); 21 my $very_verbose = 0; 22 23 # this gets set to 1 the first time we set the corresponding destreak run to be cleaned 24 #my $ds_done = 0; 25 # magic is dead 26 my $ds_done = 1; 27 21 28 GetOptions('stage=s' => \$stage, # which analysis stage to clean? 22 29 'camera|i=s' => \$camera, # user-supplied camera name … … 24 31 'mode|m=s' => \$mode, # cleanup mode (clean / purge) 25 32 'path_base=s' => \$path_base, # basename for files 33 'check-all' => \$check_all, # if set clean all chips regardless of data_state 26 34 'dbname|d=s' => \$dbname, # Database name 27 35 'verbose' => \$verbose, # Print to stdout … … 56 64 $ipprc->redirect_output($logfile) or 57 65 &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; 66 67 68 my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n'; 58 69 59 70 # set this to 1 to enable checking for files on dead nodes … … 91 102 # this stage uses 'chiptool' 92 103 my $chiptool = can_run('chiptool') or die "Can't find chiptool"; 104 my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects"; 93 105 94 106 # Get list of component imfiles … … 96 108 my $imfiles; # Array of component files 97 109 my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run 110 $command .= ' -all' if ($check_all); 98 111 $command .= " -dbname $dbname" if defined $dbname; 99 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 112 113 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) 114 = run(command => $command, verbose => $very_verbose); 100 115 unless ($success) { 101 116 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); … … 114 129 unless ($success) { 115 130 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 116 &my_die("Unable to perform chiptool : $error_code", "chip", $stage_id, $error_code);131 &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code); 117 132 } 118 133 exit 0; … … 123 138 &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); 124 139 140 my $numchips = scalar @$imfiles; 141 print "Found $numchips to clean\n"; 142 143 my $clean_sources = 0; 144 if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) { 145 # go and find the smf file(s) for the associated camRun and check the status of the file 146 # if a good one is found we have the sources for this chipRun and thus can clean the cmfs 147 my $command = "$chiptool -listrun -chip_id $stage_id"; 148 $command .= " -dbname $dbname" if defined $dbname; 149 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); 150 unless ($success) { 151 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 152 &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code); 153 } 154 my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or 155 &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); 156 my $good_smf = 0; 157 foreach my $entry (@$entries) { 158 my $camRun_state = $entry->{camRun_state}; 159 next if $camRun_state ne 'full'; 160 my $cam_id = $entry->{cam_id}; 161 if (!$cam_id) { 162 carp('no cam_id for listrun entry'); 163 next; 164 } 165 my $cam_path_base = $entry->{cam_path_base}; 166 if ( !defined $cam_path_base ) { 167 carp("no path_base for $cam_id\n"); 168 next; 169 } 170 171 # XXX: This assumes that the filerules are filerules-split 172 my $smf = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); 173 if (!$ipprc->file_exists($smf)) { 174 carp("smf for $cam_path_base not found"); 175 next; 176 } 177 # we run the program censorObjects in the check mode 178 # If this program succeeds the smf is a valid fits file and each of the 179 # extensions was succesfully read. 180 # XXX: create a new program outside of magic that performs this check 181 182 my $command = "$censorObjects -checkinputonly -file $smf"; 183 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 184 run(command => $command, verbose => $very_verbose); 185 unless ($success) { 186 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 187 print STDERR "censorObjects failed:\n"; 188 print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf; 189 print STDERR "\nSTDERR:\n" . join "", @$stderr_buf; 190 &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code); 191 } 192 $good_smf++; 193 } 194 if ($good_smf) { 195 # we have a good one so we can clean the sources 196 $clean_sources = 1; 197 print "Found $good_smf good smf files will clean sources\n"; 198 } else { 199 print "Unable to find good smf file will NOT clean sources\n"; 200 } 201 } 202 125 203 # loop over all of the imfiles, determine the path_base and class_id for each 204 my $num_errors = 0; 126 205 foreach my $imfile (@$imfiles) { 127 206 my $class_id = $imfile->{class_id}; 128 207 my $path_base = $imfile->{path_base}; 208 my $data_state = $imfile->{data_state}; 129 209 my $status = 1; 130 210 $status = 0 unless defined $path_base and $path_base ne "NULL"; 131 211 132 my $poor_quality = $imfile->{quality} > 0; 212 my $quality = $imfile->{quality}; 213 my $good_quality = ($quality == 0); 214 215 print "Starting cleanup for $class_id\n"; 133 216 134 217 # don't clean up unless the data needed to update is available 135 218 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 136 219 # goto_scrubbed now requires the config file to not exist. 137 if ($status and !$poor_quality) {220 if ($status and $good_quality) { 138 221 if ($mode eq "goto_cleaned") { 139 222 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 140 223 141 224 unless ($ipprc->file_exists($config_file)) { 225 my $fault = $imfile->{fault}; 226 142 227 if (file_gone($config_file)) { 143 print STDERR "forcing cleanup for chipRun $stage_id $class_id " 228 # config file was lost. Clean up. If the chip is ever updated a new config 229 # file will be created 230 print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality" 144 231 . " because config file ($config_file) is gone\n"; 232 } elsif ($fault == 0 and $quality == 0) { 233 print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality" 234 . " because config file ($config_file) is missing\n"; 235 $status = 0; 145 236 } else { 146 print STDERR "skipping cleanup for chipRun $stage_id $class_id "147 . " because config file ($config_file) is missing\n";148 $status = 0;237 # config file is missing but this is a bad chip anyways so clean it 238 print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality" 239 . " even though config file ($config_file) is missing\n"; 149 240 } 150 241 } … … 154 245 155 246 if ($ipprc->file_exists($config_file)) { 156 print STDERR "skipping scrubbed for chip Run$stage_id $class_id "247 print STDERR "skipping scrubbed for chip $stage_id $class_id " 157 248 . " because config file ($config_file) is present\n"; 158 249 $status = 0; … … 165 256 my @files = (); 166 257 167 # delete the temporary image datafiles168 # addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);169 # addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);170 # addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);171 258 addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1); 172 259 addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1); 173 260 addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1); 174 261 addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0); 262 if ($clean_sources) { 263 addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); 264 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 265 } 175 266 if ($mode eq "goto_purged") { 176 267 # additional files to remove for 'purge' mode 268 if (!$clean_sources) { 269 # these weren't added above but we do want to clean it 270 addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); 271 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 272 } 273 274 # background model is needed to build stack background images so we do not remove it 275 # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id); 276 277 addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id); 177 278 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); 178 279 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); 179 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? 180 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? 280 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 181 281 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); 182 282 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); 183 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean?283 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 184 284 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); 185 285 } … … 188 288 $status = &delete_files (\@files); 189 289 } 290 bzip2_file("LOG.IMFILE", $path_base, $class_id); 291 bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id); 190 292 191 293 if ($status) { 294 my $update_chip = 1; 192 295 my $command = "$chiptool -chip_id $stage_id -class_id $class_id"; 193 296 if ($mode eq "goto_purged") { 194 297 $command .= " -topurgedimfile"; 298 if ($data_state eq 'purged') { 299 $update_chip = 0; 300 } 195 301 } 196 302 elsif ($mode eq "goto_cleaned") { 197 303 $command .= " -tocleanedimfile"; 304 if ($data_state eq 'cleaned') { 305 $update_chip = 0; 306 } 198 307 } 199 308 elsif ($mode eq "goto_scrubbed") { 200 309 $command .= " -toscrubbedimfile"; 201 } 202 203 $command .= " -dbname $dbname" if defined $dbname; 204 205 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 206 run(command => $command, verbose => $verbose); 207 unless ($success) { 208 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 209 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 210 } 211 212 set_destreak_goto_cleaned(); 213 310 if ($data_state eq 'scrubbed') { 311 $update_chip = 0; 312 } 313 } 314 315 if ($update_chip) { 316 $command .= " -dbname $dbname" if defined $dbname; 317 318 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 319 run(command => $command, verbose => $verbose); 320 unless ($success) { 321 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 322 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 323 } 324 set_destreak_goto_cleaned(); 325 } 214 326 } else { 327 $num_errors++; 215 328 216 329 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) … … 218 331 $command .= " -dbname $dbname" if defined $dbname; 219 332 220 if (0) {221 # XXX Don't set components to error cleaned anymore222 333 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 223 334 run(command => $command, verbose => $verbose); … … 226 337 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); 227 338 } 228 }229 339 230 340 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. … … 241 351 } 242 352 } 353 print "Cleanup completed for chip_id $stage_id."; 354 print " num_errors: $num_errors" if $num_errors; 355 print "\n"; 243 356 exit 0; 244 357 } … … 350 463 my $skyfiles; # Array of component files 351 464 my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run 465 $command .= ' -all' if $check_all; 352 466 $command .= " -dbname $dbname" if defined $dbname; 353 467 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 354 run(command => $command, verbose => $ver bose);468 run(command => $command, verbose => $very_verbose); 355 469 unless ($success) { 356 470 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); … … 378 492 &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); 379 493 494 my $numskycells = scalar @$skyfiles; 495 print "Found $numskycells to clean\n"; 496 380 497 my @files = (); 498 my $num_errors = 0; 381 499 foreach my $skyfile (@$skyfiles) { 382 500 my $path_base = $skyfile->{path_base}; 383 501 my $skycell_id = $skyfile->{skycell_id}; 502 my $data_state = $skyfile->{data_state}; 384 503 385 504 my $status = 1; … … 391 510 392 511 unless ($ipprc->file_exists($config_file)) { 512 my $fault = $skyfile->{fault}; 513 my $quality = $skyfile->{quality}; 393 514 if (file_gone($config_file)) { 394 print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" . 395 " because config file is gone\n"; 515 print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality" 516 . " because config file ($config_file) is gone\n"; 517 } elsif ($fault == 0 and $quality == 0) { 518 print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" 519 . " because config file ($config_file) is missing\n"; 520 $status = 0; 396 521 } else { 397 print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .398 " because config file is missing\n";399 $status = 0;522 # config file is missing but this is a bad warp anyways so clean it 523 print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" 524 . " even though config file ($config_file) is missing\n"; 400 525 } 401 526 } … … 413 538 414 539 if ($status) { 415 if ($skyfile->{quality} != 8007) { 540 # XXX: what is special about quality == 8007? 541 if ($skyfile->{quality} != 8007 || $check_all) { 416 542 my @files = (); 417 543 … … 420 546 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1); 421 547 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1); 422 # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 548 # these are rebuilt during update so we can delete them here 549 addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 550 addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id ); 423 551 if ($mode eq "goto_purged") { 424 552 # additional files to remove for 'purge' mode … … 426 554 addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); 427 555 addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); 428 # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); 429 430 # XXX: do we want to delete these? 556 addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id ); 557 addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 558 559 # XXX: do we want to delete these? trace file is empty 431 560 # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); 432 # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); 433 } 434 # actual command to delete the files 561 } 562 # actual command to delete the files 435 563 $status = &delete_files (\@files); 436 564 } 437 565 } 566 bzip2_file("LOG.EXP", $path_base, $skycell_id); 567 bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); 438 568 439 569 if ($status) { 570 my $update_skyfile = 1; 440 571 my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id"; 441 572 if ($mode eq "goto_purged") { 442 573 $command .= " -topurgedskyfile"; 574 if ($data_state eq 'purged') { 575 $update_skyfile = 0; 576 } 443 577 } 444 578 elsif ($mode eq "goto_cleaned") { 445 579 $command .= " -tocleanedskyfile"; 580 if ($data_state eq 'cleaned') { 581 $update_skyfile = 0; 582 } 446 583 } 447 584 elsif ($mode eq "goto_scrubbed") { 448 585 $command .= " -toscrubbedskyfile"; 449 } 450 $command .= " -dbname $dbname" if defined $dbname; 451 452 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 453 run(command => $command, verbose => $verbose); 586 if ($data_state eq 'scrubbed') { 587 $update_skyfile = 0; 588 } 589 } 590 $command .= " -dbname $dbname" if defined $dbname; 591 592 if ($update_skyfile) { 593 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 594 run(command => $command, verbose => $verbose); 595 unless ($success) { 596 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 597 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 598 } 599 600 set_destreak_goto_cleaned(); 601 } 602 603 } else { 604 $num_errors++; 605 my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; 606 $command .= " -dbname $dbname" if defined $dbname; 607 608 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 609 run(command => $command, verbose => $verbose); 454 610 unless ($success) { 455 611 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); … … 457 613 } 458 614 459 set_destreak_goto_cleaned(); 460 461 } else { 462 my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; 463 $command .= " -dbname $dbname" if defined $dbname; 464 465 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 615 # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. 616 $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state"; 617 $command .= " -dbname $dbname" if defined $dbname; 618 619 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 466 620 run(command => $command, verbose => $verbose); 467 621 unless ($success) { … … 469 623 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); 470 624 } 471 472 # exit $PS_EXIT_UNKNOWN_ERROR; 473 } 474 } 625 } 626 } 627 print "Cleanup completed for warp_id $stage_id."; 628 print " num_errors: $num_errors" if $num_errors; 629 print "\n"; 475 630 exit 0; 476 631 } … … 603 758 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); 604 759 } 605 # exit $PS_EXIT_UNKNOWN_ERROR;606 760 } 607 761 } … … 619 773 my $skyfiles; # Array reference of component files 620 774 my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run 775 $command .= ' -all' if $check_all; 621 776 $command .= " -dbname $dbname" if defined $dbname; 622 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $ver bose);777 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); 623 778 unless ($success) { 624 779 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); … … 646 801 &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); 647 802 803 my $num_errors = 0; 648 804 my @files = (); 649 805 foreach my $skyfile (@{ $skyfiles }) { 650 806 my $path_base = $skyfile->{path_base}; 651 807 my $skycell_id = $skyfile->{skycell_id}; 808 my $data_state = $skyfile->{data_state}; 652 809 653 810 my $status = 1; … … 661 818 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 662 819 820 if (0) { 663 821 unless ($ipprc->file_exists($config_file)) { 664 822 if (file_gone($config_file)) { … … 672 830 } 673 831 } 832 unless ($ipprc->file_exists($config_file)) { 833 my $fault = $skyfile->{fault}; 834 my $quality = $skyfile->{quality}; 835 if (file_gone($config_file)) { 836 print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality" 837 . " because config file ($config_file) is gone\n"; 838 } elsif ($fault == 0 and $quality == 0) { 839 print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" 840 . " because config file ($config_file) is missing\n"; 841 $status = 0; 842 } else { 843 # config file is missing but this is a bad diff anyways so clean it 844 print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" 845 . " even though config file ($config_file) is missing\n"; 846 } 847 } 848 } 674 849 elsif ($mode eq "goto_scrubbed") { 675 850 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); … … 712 887 713 888 } 714 # print STDERR "MY FILES: @files\n";715 889 $status = &delete_files(\@files); 716 890 } 717 # print STDERR "MY STATUS: $status\n"; 891 892 bzip2_file("LOG.EXP", $path_base, $skycell_id); 893 bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); 894 718 895 if ($status) { 719 896 my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 897 my $update_skyfile = 1; 720 898 721 899 if ($mode eq "goto_purged") { 722 900 $command .= " -topurgedskyfile"; 901 if ($data_state eq 'purged') { 902 $update_skyfile = 0; 903 } 723 904 } 724 905 elsif ($mode eq "goto_cleaned") { 725 906 $command .= " -tocleanedskyfile"; 907 if ($data_state eq 'cleaned') { 908 $update_skyfile = 0; 909 } 726 910 } 727 911 elsif ($mode eq "goto_scrubbed") { 728 912 $command .= " -toscrubbedskyfile"; 729 } 730 913 if ($data_state eq 'scrubbed') { 914 $update_skyfile = 0; 915 } 916 } 917 918 $command .= " -dbname $dbname" if defined $dbname; 919 920 if ($update_skyfile) { 921 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 922 run(command => $command, verbose => $verbose); 923 unless ($success) { 924 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 925 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 926 } 927 set_destreak_goto_cleaned(); 928 } 929 930 931 } else { 932 $num_errors++; 933 my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state"; 731 934 $command .= " -dbname $dbname" if defined $dbname; 732 935 … … 738 941 } 739 942 740 set_destreak_goto_cleaned(); 741 742 } else { 743 my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 744 745 $command .= " -dbname $dbname" if defined $dbname; 746 747 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 943 $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; 944 945 $command .= " -dbname $dbname" if defined $dbname; 946 947 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 748 948 run(command => $command, verbose => $verbose); 749 949 unless ($success) { … … 751 951 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); 752 952 } 753 # exit $PS_EXIT_UNKNOWN_ERROR; 754 } 755 } 953 } 954 } 955 print "Cleanup completed for diff_id $stage_id."; 956 print " num_errors: $num_errors" if $num_errors; 957 print "\n"; 756 958 exit 0; 757 959 } … … 1917 2119 1918 2120 foreach my $file (@$files) { 1919 print STDERR "unlinking $stage $stage_id $file\n" ;2121 print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose; 1920 2122 1921 2123 my $error_code = $ipprc->kill_file($file); … … 2008 2210 } 2009 2211 2010 # this gets set to 1 the first time we set the corresponding destreak run to be cleaned2011 my $ds_done = 0;2012 2212 sub set_destreak_goto_cleaned { 2013 2213 … … 2027 2227 } 2028 2228 2229 sub bzip2_file { 2230 my $filerule = shift; 2231 my $path_base = shift; 2232 my $component = shift; 2233 2234 my $filename = $ipprc->filename($filerule, $path_base, $component); 2235 if (!$ipprc->file_exists($filename)) { 2236 return 1; 2237 } 2238 if (my $resolved = $ipprc->file_resolve($filename)) { 2239 my $bzip2_filename = $filename . '.bz2'; 2240 if ($ipprc->file_exists($bzip2_filename)) { 2241 $ipprc->kill_file($bzip2_filename); 2242 } 2243 my $bzip2_file = $ipprc->file_create($bzip2_filename); 2244 my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file; 2245 2246 my $command = "$bzip2 < $resolved > $bzip2_file"; 2247 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2248 run(command => $command, verbose => $very_verbose); 2249 if ($success) { 2250 # success delete the original file 2251 my $error_code = $ipprc->kill_file($filename); 2252 } else { 2253 # if bzip2 failed. Carry on but don't delete the existing file 2254 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2255 print STDERR "Failed to bzip2 $filename: $error_code\n"; 2256 return 0; 2257 } 2258 } 2259 return 1; 2260 } 2261 2029 2262 # XXX we currently do not set the error state in the db on my_die 2030 2263 sub my_die
Note:
See TracChangeset
for help on using the changeset viewer.
