- Timestamp:
- Sep 15, 2009, 4:02:42 PM (17 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/20090715/ippScripts/scripts/ipp_cleanup.pl
r24642 r25406 60 60 61 61 62 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff=> 1,63 detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile=> 1,64 detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp=> 1,65 detrend.resid.imfile => 1, detrend.resid.exp=> 1 );62 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, 63 "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, 64 "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, 65 "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); 66 66 unless ($stages{$stage}) { 67 67 die "unknown stage $stage for ipp_cleanup.pl\n"; … … 119 119 # don't clean up unless the data needed to update is available 120 120 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 121 # goto_scrubbed now requires the config file to not exist. 121 122 if ($mode eq "goto_cleaned") { 122 123 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 123 print STDERR "CHIP: CONFIG_FILE : $config_file\n"; 124 124 125 if (!$config_file or ! -e $config_file) { 125 126 print STDERR "skipping cleanup for chipRun $stage_id $class_id " … … 128 129 } 129 130 } 131 elsif ($mode eq "goto_scrubbed") { 132 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 133 134 if ($config_file and -e $config_file) { 135 print STDERR "skipping scrubbed for chipRun $stage_id $class_id " 136 . " because config file is present\n"; 137 $status = 0; 138 } 139 } 130 140 131 141 if ($status) { … … 144 154 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); 145 155 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); 146 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 147 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 156 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? 157 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? 148 158 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); 149 159 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); 150 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 160 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? 151 161 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); 152 162 } … … 160 170 if ($mode eq "goto_purged") { 161 171 $command .= " -topurgedimfile"; 162 } else { 172 } 173 elsif ($mode eq "goto_cleaned") { 163 174 $command .= " -tocleanedimfile"; 164 175 } 176 elsif ($mode eq "goto_scrubbed") { 177 $command .= " -toscrubbedimfile"; 178 } 179 165 180 $command .= " -dbname $dbname" if defined $dbname; 166 181 … … 218 233 my $status = 1; 219 234 # don't clean up unless the data needed to update is available 235 # goto_scrubbed now requires the config file to not be present 220 236 if ($mode eq "goto_cleaned") { 221 237 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); … … 226 242 } 227 243 } 244 elsif ($mode eq "goto_scrubbed") { 245 my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); 246 247 if ($config_file and -e $config_file) { 248 print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; 249 $status = 0; 250 } 251 } 228 252 if ($status) { 229 253 my @files = (); 230 254 # delete the temporary image datafiles 231 addFilename (\@files, "PSASTRO.OUTPUT", $path_base);255 # addFilename (\@files, "PSASTRO.OUTPUT", $path_base); 232 256 if ($mode eq "goto_purged") { 233 257 # additional files to remove for 'purge' mode … … 246 270 } 247 271 if ($mode eq "goto_scrubbed") { 248 $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";272 $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; 249 273 } 250 274 if ($mode eq "goto_purged") { … … 311 335 } 312 336 } 337 elsif ($mode eq "goto_scrubbed") { 338 my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); 339 340 if ($config_file and -e $config_file) { 341 print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . 342 " because config file is present\n"; 343 $status = 0; 344 } 345 } 313 346 if ($status) { 314 347 # delete the temporary image datafiles … … 316 349 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); 317 350 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); 318 addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);351 # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); 319 352 320 353 if ($mode eq "goto_purged") { … … 337 370 if ($mode eq "goto_purged") { 338 371 $command .= " -topurgedskyfile"; 339 } else { 372 } 373 elsif ($mode eq "goto_cleaned") { 340 374 $command .= " -tocleanedskyfile"; 341 375 } 376 elsif ($mode eq "goto_scrubbed") { 377 $command .= " -toscrubbedskyfile"; 378 } 342 379 $command .= " -dbname $dbname" if defined $dbname; 343 380 … … 395 432 if ($mode eq "goto_cleaned") { 396 433 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 397 print STDERR "MY CONFIG FILE = $config_file\n"; 398 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 434 399 435 $config_file =~ s%^file://%%; 400 436 if (!$config_file or ! -e $config_file) { … … 404 440 } 405 441 $config_file = 'file://' . $config_file; 442 } 443 elsif ($mode eq "goto_scrubbed") { 444 my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); 445 $config_file =~ s%^file://%%; 446 if ($config_file and -e $config_file) { 447 print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . 448 " because config file is present\n"; 449 $status = 0; 450 } 406 451 } 407 452 if ($status) { … … 428 473 if ($mode eq "goto_purged") { 429 474 $command .= " -updaterun -state purged"; 430 } else { 475 } 476 elsif ($mode eq "goto_cleaned") { 431 477 $command .= " -updaterun -state cleaned"; 478 } 479 elsif ($mode eq "goto_scrubbed") { 480 $command .= " -updaterun -state scrubbed"; 432 481 } 433 482 $command .= " -dbname $dbname" if defined $dbname; … … 485 534 if ($mode eq "goto_cleaned") { 486 535 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 487 print STDERR "MY CONFIG FILE = $config_file\n"; 488 printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); 536 489 537 $config_file =~ s%^file://%%; 490 538 if (!$config_file or ! -e $config_file) { … … 495 543 $config_file = 'file://' . $config_file; 496 544 } 545 elsif ($mode eq "goto_scrubbed") { 546 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); 547 $config_file =~ s%^file://%%; 548 if ($config_file and -e $config_file) { 549 print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . 550 " because config file ($config_file) is present\n"; 551 $status = 0; 552 } 553 } 497 554 if ($status) { 498 555 # delete the temporary image datafiles … … 504 561 addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); 505 562 addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); 563 506 564 addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); 507 565 addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); … … 529 587 if ($status) { 530 588 my $command = "$difftool -diff_id $stage_id"; 531 # my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; 589 532 590 if ($mode eq "goto_purged") { 533 591 $command .= " -updaterun -state purged"; 534 } else { 592 } 593 elsif ($mode eq "goto_cleaned") { 535 594 $command .= " -updaterun -state cleaned"; 536 595 } 596 elsif ($mode eq "goto_scrubbed") { 597 $command .= " -updaterun -state scrubbed"; 598 } 599 537 600 $command .= " -dbname $dbname" if defined $dbname; 538 601 … … 545 608 } else { 546 609 my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; 547 # my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; 610 548 611 $command .= " -dbname $dbname" if defined $dbname; 549 612 … … 560 623 } 561 624 if ($stage eq 'fake') { 562 die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; 625 print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; 626 exit(0); 627 die "--stage_id required for stage fake\n" if !$stage_id; 628 ### select the imfiles for this entry 629 630 # this stage uses 'chiptool' 631 my $faketool = can_run('faketool') or die "Can't find faketool"; 632 633 # Get list of component imfiles 634 # XXX may need a different my_die for each stage 635 my $imfiles; # Array of component files 636 my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run 637 $command .= " -dbname $dbname" if defined $dbname; 638 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 639 unless ($success) { 640 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 641 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 642 } 643 644 # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 645 if (@$stdout_buf == 0) { 646 my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; 647 $command .= " -dbname $dbname" if defined $dbname; 648 649 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 650 run(command => $command, verbose => $verbose); 651 unless ($success) { 652 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 653 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 654 } 655 exit 0; 656 } 657 658 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 659 &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); 660 661 # extract the metadata for the files into a hash list 662 $imfiles = parse_md_list($metadata) or 663 &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); 664 665 # loop over all of the imfiles, determine the path_base and class_id for each 666 foreach my $imfile (@$imfiles) { 667 my $class_id = $imfile->{class_id}; 668 my $path_base = $imfile->{path_base}; 669 my $status = 1; 670 671 # don't clean up unless the data needed to update is available 672 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 673 # goto_scrubbed now requires the config file to not exist. 674 if ($mode eq "goto_cleaned") { 675 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 676 677 if (!$config_file or ! -e $config_file) { 678 print STDERR "skipping cleanup for fakeRun $stage_id $class_id " 679 . " because config file is missing\n"; 680 $status = 0; 681 } 682 } 683 elsif ($mode eq "goto_scrubbed") { 684 my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); 685 686 if ($config_file and -e $config_file) { 687 print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " 688 . " because config file is present\n"; 689 $status = 0; 690 } 691 } 692 693 if ($status) { 694 # array of actual filenames to delete 695 my @files = (); 696 697 # delete the temporary image datafiles 698 addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); 699 addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); 700 addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); 701 addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); 702 if ($mode eq "goto_purged") { 703 # additional files to remove for 'purge' mode 704 addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); 705 addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); 706 addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); 707 } 708 709 # actual command to delete the files 710 $status = &delete_files (\@files); 711 } 712 713 if ($status) { 714 my $command = "$faketool -fake_id $stage_id -class_id $class_id"; 715 if ($mode eq "goto_purged") { 716 $command .= " -topurgedimfile"; 717 } 718 elsif ($mode eq "goto_cleaned") { 719 $command .= " -tocleanedimfile"; 720 } 721 elsif ($mode eq "goto_scrubbed") { 722 $command .= " -toscrubbedimfile"; 723 } 724 725 $command .= " -dbname $dbname" if defined $dbname; 726 727 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 728 run(command => $command, verbose => $verbose); 729 unless ($success) { 730 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 731 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 732 } 733 } else { 734 735 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 736 my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; 737 $command .= " -dbname $dbname" if defined $dbname; 738 739 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 740 run(command => $command, verbose => $verbose); 741 unless ($success) { 742 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 743 &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); 744 } 745 } 746 } 747 exit 0; 748 563 749 } 564 # fake : faketool : -pendingcleanupimfile (loop over imfiles) 750 # Detrend stages 751 if ($stage eq "detrend.process.imfile") { 752 753 die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; 754 ### select the imfiles for this entry 755 756 # this stage uses 'dettool' 757 my $dettool = can_run('dettool') or die "Can't find chiptool"; 758 759 # Get list of component imfiles 760 # XXX may need a different my_die for each stage 761 my $imfiles; # Array of component files 762 my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run 763 $command .= " -dbname $dbname" if defined $dbname; 764 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 765 unless ($success) { 766 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 767 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 768 } 769 770 # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' 771 if (@$stdout_buf == 0) { 772 exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. 773 } 774 775 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 776 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 777 778 # extract the metadata for the files into a hash list 779 $imfiles = parse_md_list($metadata) or 780 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 781 782 # loop over all of the imfiles, determine the path_base and class_id for each 783 foreach my $imfile (@$imfiles) { 784 my $exp_id = $imfile->{exp_id}; 785 my $class_id = $imfile->{class_id}; 786 my $path_base = $imfile->{path_base}; 787 my $status = 1; 788 789 # don't clean up unless the data needed to update is available 790 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 791 # goto_scrubbed now requires the config file to not exist. 792 793 # Possibly not the correct config file, but simtest doesn't leave any around to check. 794 if ($mode eq "goto_cleaned") { 795 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 796 797 if (!$config_file or ! -e $config_file) { 798 print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " 799 . " because config file is missing\n"; 800 $status = 0; 801 } 802 } 803 elsif ($mode eq "goto_scrubbed") { 804 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 805 806 if ($config_file and -e $config_file) { 807 print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " 808 . " because config file is present\n"; 809 $status = 0; 810 } 811 } 812 813 if ($status) { 814 # array of actual filenames to delete 815 my @files = (); 816 817 # delete the temporary image datafiles 818 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 819 addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); 820 addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); 821 if ($mode eq "goto_purged") { 822 # additional files to remove for 'purge' mode 823 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 824 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 825 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 826 } 827 828 # actual command to delete the files 829 $status = &delete_files (\@files); 830 } 831 832 if ($status) { 833 my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; 834 if ($mode eq "goto_purged") { 835 $command .= " -data_state purged"; 836 } 837 elsif ($mode eq "goto_cleaned") { 838 $command .= " -data_state cleaned"; 839 } 840 elsif ($mode eq "goto_scrubbed") { 841 $command .= " -data_state scrubbed"; 842 } 843 844 $command .= " -dbname $dbname" if defined $dbname; 845 846 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 847 run(command => $command, verbose => $verbose); 848 unless ($success) { 849 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 850 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 851 } 852 853 } else { 854 855 # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) 856 my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; 857 $command .= " -dbname $dbname" if defined $dbname; 858 859 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 860 run(command => $command, verbose => $verbose); 861 unless ($success) { 862 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 863 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 864 } 865 } 866 } 867 868 # Check to see if we can mark the whole detRunSummary object as cleaned. 869 870 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 871 $command .= " -dbname $dbname" if defined $dbname; 872 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 873 unless ($success) { 874 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 875 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 876 } 877 if (@$stdout_buf != 0) { 878 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 879 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 880 my $exps = parse_md_list($metadata) or 881 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 882 883 foreach my $exp (@$exps) { 884 my $iteration = $exp->{iteration}; 885 my $command; 886 if ($mode eq "goto_cleaned") { 887 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 888 } 889 if ($mode eq "goto_scrubbed") { 890 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 891 } 892 if ($mode eq "goto_purged") { 893 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 894 } 895 $command .= " -dbname $dbname" if defined $dbname; 896 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 897 run(command => $command, verbose => $verbose); 898 unless ($success) { 899 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 900 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 901 } 902 } 903 } 904 905 exit 0; 906 } 907 if ($stage eq "detrend.process.exp") { 908 die "--stage_id required for stage $stage\n" if !$stage_id; 909 # this stage uses 'dettool' 910 my $dettool = can_run('dettool') or die "Can't find dettool"; 911 912 # Get list of component imfiles 913 # XXX may need a different my_die for each stage 914 my $exps; # Array of component files 915 my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run 916 $command .= " -dbname $dbname" if defined $dbname; 917 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 918 unless ($success) { 919 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 920 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 921 } 922 923 if (@$stdout_buf == 0) { 924 exit 0; #silently abort. I need to fix this for propers 925 } 926 927 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 928 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 929 930 $exps = parse_md_list($metadata) or 931 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 932 933 934 foreach my $exp (@$exps) { 935 my $path_base = $exp->{path_base}; 936 my $exp_id = $exp->{exp_id}; 937 938 my $status = 1; 939 # don't clean up unless the data needed to update is available 940 # goto_scrubbed now requires the config file to not be present 941 if ($mode eq "goto_cleaned") { 942 my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); 943 944 if (!$config_file or ! -e $config_file) { 945 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 946 $status = 0; 947 } 948 } 949 elsif ($mode eq "goto_scrubbed") { 950 my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); 951 952 if ($config_file and -e $config_file) { 953 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 954 $status = 0; 955 } 956 } 957 if ($status) { 958 my @files = (); 959 # delete the temporary image datafiles 960 # I can't find anything to put here 961 if ($mode eq "goto_purged") { 962 # additional files to remove for 'purge' mode 963 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 964 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 965 } 966 # actual command to delete the files 967 $status = &delete_files (\@files); 968 } 969 970 if ($status) { 971 my $command; 972 if ($mode eq "goto_cleaned") { 973 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; 974 } 975 if ($mode eq "goto_scrubbed") { 976 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; 977 } 978 if ($mode eq "goto_purged") { 979 $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; 980 } 981 $command .= " -dbname $dbname" if defined $dbname; 982 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 983 run(command => $command, verbose => $verbose); 984 unless ($success) { 985 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 986 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 987 } 988 } else { 989 # since 'camera' has only a single imfile, we can just update the run 990 my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; 991 $command .= " -dbname $dbname" if defined $dbname; 992 993 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 994 run(command => $command, verbose => $verbose); 995 unless ($success) { 996 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 997 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 998 } 999 exit $PS_EXIT_UNKNOWN_ERROR; 1000 } 1001 } 1002 # Check to see if we can mark the whole detRunSummary object as cleaned. 1003 1004 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1005 $command .= " -dbname $dbname" if defined $dbname; 1006 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1007 unless ($success) { 1008 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1009 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1010 } 1011 if (@$stdout_buf != 0) { 1012 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1013 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1014 $exps = parse_md_list($metadata) or 1015 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1016 1017 foreach my $exp (@$exps) { 1018 my $iteration = $exp->{iteration}; 1019 my $command; 1020 if ($mode eq "goto_cleaned") { 1021 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1022 } 1023 if ($mode eq "goto_scrubbed") { 1024 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1025 } 1026 if ($mode eq "goto_purged") { 1027 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1028 } 1029 $command .= " -dbname $dbname" if defined $dbname; 1030 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1031 run(command => $command, verbose => $verbose); 1032 unless ($success) { 1033 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1034 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1035 } 1036 } 1037 } 1038 1039 exit 0; 1040 } 1041 if ($stage eq "detrend.stack.imfile") { 1042 1043 die "--stage_id required for stage $stage\n" if !$stage_id; 1044 1045 # this stage uses 'dettool' 1046 my $dettool = can_run('dettool') or die "Can't find dettool"; 1047 1048 # Get list of component imfiles 1049 my $stacks; # Array reference of component files 1050 my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run 1051 $command .= " -dbname $dbname" if defined $dbname; 1052 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1053 unless ($success) { 1054 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1055 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1056 } 1057 my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 1058 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1059 1060 $stacks = parse_md_list($metadata) or 1061 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1062 1063 my @files = (); 1064 foreach my $stack (@{ $stacks }) { 1065 # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. 1066 my $path_base = $stack->{uri}; 1067 my $iteration = $stack->{iteration}; 1068 my $class_id = $stack->{class_id}; 1069 1070 $path_base =~ s/\.fits$//; # That should do it? 1071 1072 my $status = 1; 1073 1074 # if ($mode eq "goto_cleaned") { 1075 # my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); 1076 1077 # $config_file =~ s%^file://%%; 1078 # if (!$config_file or ! -e $config_file) { 1079 # print STDERR "skipping cleanup for $stage $stage_id $path_base" . 1080 # " because config file is missing\n"; 1081 # $status = 0; 1082 # } 1083 # $config_file = 'file://' . $config_file; 1084 # } 1085 # elsif ($mode eq "goto_scrubbed") { 1086 # my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); 1087 # $config_file =~ s%^file://%%; 1088 # if ($config_file and -e $config_file) { 1089 # print STDERR "skipping scrubbed for $stage $stage_id $path_base" . 1090 # " because config file is present\n"; 1091 # $status = 0; 1092 # } 1093 # } 1094 1095 if ($status) { 1096 # delete the temporary image datafiles 1097 # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. 1098 # I understand that they all point to the same filename right now, but that may not be true in 1099 # the future. 1100 addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); 1101 addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); 1102 addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); 1103 addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); 1104 addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); 1105 addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); 1106 1107 1108 addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); 1109 addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); 1110 1111 if ($mode eq "goto_purged") { 1112 # additional files to remove for 'purge' mode 1113 # addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); 1114 } 1115 1116 $status = &delete_files(\@files); 1117 } 1118 1119 if ($status) { 1120 my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; 1121 if ($mode eq "goto_purged") { 1122 $command .= " -updatestacked -data_state purged"; 1123 } 1124 elsif ($mode eq "goto_cleaned") { 1125 $command .= " -updatestacked -data_state cleaned"; 1126 } 1127 elsif ($mode eq "goto_scrubbed") { 1128 $command .= " -updatestacked -data_state scrubbed"; 1129 } 1130 $command .= " -dbname $dbname" if defined $dbname; 1131 1132 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1133 run(command => $command, verbose => $verbose); 1134 unless ($success) { 1135 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1136 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1137 } 1138 } else { 1139 my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1140 $command .= " -dbname $dbname" if defined $dbname; 1141 1142 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1143 run(command => $command, verbose => $verbose); 1144 unless ($success) { 1145 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1146 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1147 } 1148 exit $PS_EXIT_UNKNOWN_ERROR; 1149 } 1150 } 1151 # Check to see if we can mark the whole detRunSummary object as cleaned. 1152 1153 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1154 $command .= " -dbname $dbname" if defined $dbname; 1155 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1156 unless ($success) { 1157 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1158 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1159 } 1160 if (@$stdout_buf != 0) { 1161 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1162 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1163 my $exps = parse_md_list($metadata) or 1164 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1165 1166 foreach my $exp (@$exps) { 1167 my $iteration = $exp->{iteration}; 1168 my $command; 1169 if ($mode eq "goto_cleaned") { 1170 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1171 } 1172 if ($mode eq "goto_scrubbed") { 1173 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1174 } 1175 if ($mode eq "goto_purged") { 1176 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1177 } 1178 $command .= " -dbname $dbname" if defined $dbname; 1179 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1180 run(command => $command, verbose => $verbose); 1181 unless ($success) { 1182 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1183 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1184 } 1185 } 1186 } 1187 exit 0; 1188 } 1189 if ($stage eq "detrend.normstat.imfile") { 1190 print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; 1191 die "--stage_id required for stage $stage\n" if !$stage_id; 1192 # this stage uses 'camtool' 1193 my $dettool = can_run('dettool') or die "Can't find dettool"; 1194 1195 my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run 1196 $command .= " -dbname $dbname" if defined $dbname; 1197 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1198 unless ($success) { 1199 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1200 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1201 } 1202 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1203 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1204 1205 my $exps = parse_md_list($metadata) or 1206 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1207 1208 foreach my $exp (@$exps) { 1209 # my $path_base = $exp->{path_base}; 1210 my $iteration = $exp->{iteration}; 1211 my $class_id = $exp->{class_id}; 1212 1213 my $status = 1; 1214 if ($status) { 1215 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; 1216 if ($mode eq "goto_cleaned") { 1217 $command .= " -data_state cleaned"; 1218 } 1219 if ($mode eq "goto_scrubbed") { 1220 $command .= " -data_state scrubbed"; 1221 } 1222 if ($mode eq "goto_purged") { 1223 $command .= " -data_state purged"; 1224 } 1225 $command .= " -dbname $dbname" if defined $dbname; 1226 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1227 run(command => $command, verbose => $verbose); 1228 unless ($success) { 1229 print STDERR "WTF: $success TTT $error_code QQQ\n"; 1230 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1231 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1232 } 1233 } else { 1234 my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1235 $command .= " -dbname $dbname" if defined $dbname; 1236 1237 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1238 run(command => $command, verbose => $verbose); 1239 unless ($success) { 1240 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1241 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1242 } 1243 exit $PS_EXIT_UNKNOWN_ERROR; 1244 } 1245 } 1246 # Check to see if we can mark the whole detRunSummary object as cleaned. 1247 1248 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1249 $command .= " -dbname $dbname" if defined $dbname; 1250 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1251 unless ($success) { 1252 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1253 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1254 } 1255 if (@$stdout_buf != 0) { 1256 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1257 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1258 $exps = parse_md_list($metadata) or 1259 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1260 1261 foreach my $exp (@$exps) { 1262 my $iteration = $exp->{iteration}; 1263 my $command; 1264 if ($mode eq "goto_cleaned") { 1265 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1266 } 1267 if ($mode eq "goto_scrubbed") { 1268 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1269 } 1270 if ($mode eq "goto_purged") { 1271 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1272 } 1273 $command .= " -dbname $dbname" if defined $dbname; 1274 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1275 run(command => $command, verbose => $verbose); 1276 unless ($success) { 1277 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1278 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1279 } 1280 } 1281 } 1282 1283 exit 0; 1284 1285 } 1286 if ($stage eq "detrend.norm.imfile") { 1287 die "--stage_id required for stage $stage\n" if !$stage_id; 1288 # this stage uses 'dettool' 1289 my $dettool = can_run('dettool') or die "Can't find dettool"; 1290 1291 # Get list of component imfiles 1292 # XXX may need a different my_die for each stage 1293 my $exps; # Array of component files 1294 my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run 1295 $command .= " -dbname $dbname" if defined $dbname; 1296 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1297 unless ($success) { 1298 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1299 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1300 } 1301 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1302 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1303 1304 $exps = parse_md_list($metadata) or 1305 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1306 1307 foreach my $exp (@$exps) { 1308 my $path_base = $exp->{path_base}; 1309 my $iteration = $exp->{iteration}; 1310 my $class_id = $exp->{class_id}; 1311 1312 my $status = 1; 1313 # don't clean up unless the data needed to update is available 1314 # goto_scrubbed now requires the config file to not be present 1315 if ($mode eq "goto_cleaned") { 1316 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1317 1318 if (!$config_file or ! -e $config_file) { 1319 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1320 $status = 0; 1321 } 1322 } 1323 elsif ($mode eq "goto_scrubbed") { 1324 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1325 1326 if ($config_file and -e $config_file) { 1327 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1328 $status = 0; 1329 } 1330 } 1331 if ($status) { 1332 my @files = (); 1333 1334 if ($mode eq "goto_purged") { 1335 # additional files to remove for 'purge' mode 1336 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); 1337 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); 1338 1339 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); 1340 addFilename (\@files, "PPIMAGE.STATS", $path_base); 1341 } 1342 # actual command to delete the files 1343 $status = &delete_files (\@files); 1344 } 1345 1346 if ($status) { 1347 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; 1348 if ($mode eq "goto_cleaned") { 1349 $command .= " -data_state cleaned"; 1350 } 1351 if ($mode eq "goto_scrubbed") { 1352 $command .= " -data_state scrubbed"; 1353 } 1354 if ($mode eq "goto_purged") { 1355 $command .= " -data_state purged"; 1356 } 1357 $command .= " -dbname $dbname" if defined $dbname; 1358 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1359 run(command => $command, verbose => $verbose); 1360 unless ($success) { 1361 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1362 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1363 } 1364 } else { 1365 my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1366 $command .= " -dbname $dbname" if defined $dbname; 1367 1368 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1369 run(command => $command, verbose => $verbose); 1370 unless ($success) { 1371 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1372 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1373 } 1374 exit $PS_EXIT_UNKNOWN_ERROR; 1375 } 1376 } 1377 # Check to see if we can mark the whole detRunSummary object as cleaned. 1378 1379 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1380 $command .= " -dbname $dbname" if defined $dbname; 1381 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1382 unless ($success) { 1383 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1384 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1385 } 1386 if (@$stdout_buf != 0) { 1387 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1388 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1389 $exps = parse_md_list($metadata) or 1390 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1391 1392 foreach my $exp (@$exps) { 1393 my $iteration = $exp->{iteration}; 1394 my $command; 1395 if ($mode eq "goto_cleaned") { 1396 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1397 } 1398 if ($mode eq "goto_scrubbed") { 1399 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1400 } 1401 if ($mode eq "goto_purged") { 1402 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1403 } 1404 $command .= " -dbname $dbname" if defined $dbname; 1405 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1406 run(command => $command, verbose => $verbose); 1407 unless ($success) { 1408 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1409 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1410 } 1411 } 1412 } 1413 1414 exit 0; 1415 } 1416 if ($stage eq "detrend.norm.exp") { 1417 die "--stage_id required for stage $stage\n" if !$stage_id; 1418 # this stage uses 'dettool' 1419 my $dettool = can_run('dettool') or die "Can't find dettool"; 1420 1421 # Get list of component imfiles 1422 # XXX may need a different my_die for each stage 1423 my $exps; # Array of component files 1424 my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run 1425 $command .= " -dbname $dbname" if defined $dbname; 1426 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1427 unless ($success) { 1428 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1429 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1430 } 1431 1432 if (@$stdout_buf == 0) { 1433 exit 0; 1434 } 1435 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1436 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1437 1438 $exps = parse_md_list($metadata) or 1439 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1440 1441 foreach my $exp (@$exps) { 1442 my $exp_id = $exp->{exp_id}; 1443 my $iteration = $exp->{iteration}; 1444 my $path_base = $exp->{path_base}; 1445 1446 my $status = 1; 1447 # don't clean up unless the data needed to update is available 1448 # goto_scrubbed now requires the config file to not be present 1449 if ($mode eq "goto_cleaned") { 1450 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1451 1452 if (!$config_file or ! -e $config_file) { 1453 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1454 $status = 0; 1455 } 1456 } 1457 elsif ($mode eq "goto_scrubbed") { 1458 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1459 1460 if ($config_file and -e $config_file) { 1461 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1462 $status = 0; 1463 } 1464 } 1465 if ($status) { 1466 my @files = (); 1467 # delete the temporary image datafiles 1468 if ($mode eq "goto_purged") { 1469 # additional files to remove for 'purge' mode 1470 addFilename (\@files, "PPIMAGE.JPEG1", $path_base); 1471 addFilename (\@files, "PPIMAGE.JPEG2", $path_base); 1472 } 1473 # actual command to delete the files 1474 $status = &delete_files (\@files); 1475 } 1476 1477 if ($status) { 1478 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; 1479 if ($mode eq "goto_cleaned") { 1480 $command .= " -data_state cleaned"; 1481 } 1482 if ($mode eq "goto_scrubbed") { 1483 $command .= " -data_state scrubbed"; 1484 } 1485 if ($mode eq "goto_purged") { 1486 $command .= " -data_state purged"; 1487 } 1488 $command .= " -dbname $dbname" if defined $dbname; 1489 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1490 run(command => $command, verbose => $verbose); 1491 unless ($success) { 1492 print STDERR " residexp had an issue setting the state:? $success $error_code\n"; 1493 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1494 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1495 } 1496 } else { 1497 my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; 1498 $command .= " -dbname $dbname" if defined $dbname; 1499 1500 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1501 run(command => $command, verbose => $verbose); 1502 unless ($success) { 1503 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1504 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1505 } 1506 exit $PS_EXIT_UNKNOWN_ERROR; 1507 } 1508 } 1509 # Check to see if we can mark the whole detRunSummary object as cleaned. 1510 1511 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1512 $command .= " -dbname $dbname" if defined $dbname; 1513 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1514 unless ($success) { 1515 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1516 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1517 } 1518 if (@$stdout_buf != 0) { 1519 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1520 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1521 $exps = parse_md_list($metadata) or 1522 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1523 1524 foreach my $exp (@$exps) { 1525 my $iteration = $exp->{iteration}; 1526 my $command; 1527 if ($mode eq "goto_cleaned") { 1528 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1529 } 1530 if ($mode eq "goto_scrubbed") { 1531 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1532 } 1533 if ($mode eq "goto_purged") { 1534 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1535 } 1536 $command .= " -dbname $dbname" if defined $dbname; 1537 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1538 run(command => $command, verbose => $verbose); 1539 unless ($success) { 1540 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1541 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1542 } 1543 } 1544 } 1545 exit 0; 1546 } 1547 if ($stage eq "detrend.resid.imfile") { 1548 1549 die "--stage_id required for stage $stage\n" if !$stage_id; 1550 ### select the imfiles for this entry 1551 1552 # this stage uses 'dettool' 1553 my $dettool = can_run('dettool') or die "Can't find dettool"; 1554 1555 # Get list of component imfiles 1556 # XXX may need a different my_die for each stage 1557 my $imfiles; # Array of component files 1558 my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run 1559 $command .= " -dbname $dbname" if defined $dbname; 1560 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1561 unless ($success) { 1562 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1563 &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); 1564 } 1565 1566 if (@$stdout_buf == 0) { 1567 exit 0; 1568 } 1569 1570 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1571 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1572 1573 # extract the metadata for the files into a hash list 1574 $imfiles = parse_md_list($metadata) or 1575 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1576 1577 # loop over all of the imfiles, determine the path_base and class_id for each 1578 foreach my $imfile (@$imfiles) { 1579 my $class_id = $imfile->{class_id}; 1580 my $iteration = $imfile->{iteration}; 1581 my $exp_id = $imfile->{exp_id}; 1582 my $path_base = $imfile->{path_base}; 1583 my $status = 1; 1584 1585 # don't clean up unless the data needed to update is available 1586 # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent 1587 # goto_scrubbed now requires the config file to not exist. 1588 1589 # Possibly not the correct config file, but simtest doesn't leave any around to check. 1590 if ($mode eq "goto_cleaned") { 1591 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 1592 1593 if (!$config_file or ! -e $config_file) { 1594 print STDERR "skipping cleanup for $stage $stage_id $class_id " 1595 . " because config file is missing\n"; 1596 $status = 0; 1597 } 1598 } 1599 elsif ($mode eq "goto_scrubbed") { 1600 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); 1601 1602 if ($config_file and -e $config_file) { 1603 print STDERR "skipping scrubbed for $stage $stage_id $class_id " 1604 . " because config file is present\n"; 1605 $status = 0; 1606 } 1607 } 1608 1609 if ($status) { 1610 # array of actual filenames to delete 1611 my @files = (); 1612 1613 # delete the temporary image datafiles 1614 addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); 1615 if ($mode eq "goto_purged") { 1616 # additional files to remove for 'purge' mode 1617 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); 1618 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); 1619 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); 1620 } 1621 1622 # actual command to delete the files 1623 $status = &delete_files (\@files); 1624 } 1625 1626 if ($status) { 1627 my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; 1628 if ($mode eq "goto_purged") { 1629 $command .= " -data_state purged"; 1630 } 1631 elsif ($mode eq "goto_cleaned") { 1632 $command .= " -data_state cleaned"; 1633 } 1634 elsif ($mode eq "goto_scrubbed") { 1635 $command .= " -data_state scrubbed"; 1636 } 1637 1638 $command .= " -dbname $dbname" if defined $dbname; 1639 1640 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1641 run(command => $command, verbose => $verbose); 1642 unless ($success) { 1643 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1644 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1645 } 1646 } else { 1647 my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; 1648 $command .= " -dbname $dbname" if defined $dbname; 1649 1650 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1651 run(command => $command, verbose => $verbose); 1652 unless ($success) { 1653 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1654 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1655 } 1656 } 1657 } 1658 # Check to see if we can mark the whole detRunSummary object as cleaned. 1659 1660 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1661 $command .= " -dbname $dbname" if defined $dbname; 1662 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1663 unless ($success) { 1664 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1665 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1666 } 1667 if (@$stdout_buf != 0) { 1668 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1669 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1670 my $exps = parse_md_list($metadata) or 1671 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1672 1673 foreach my $exp (@$exps) { 1674 my $iteration = $exp->{iteration}; 1675 my $command; 1676 if ($mode eq "goto_cleaned") { 1677 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1678 } 1679 if ($mode eq "goto_scrubbed") { 1680 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1681 } 1682 if ($mode eq "goto_purged") { 1683 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1684 } 1685 $command .= " -dbname $dbname" if defined $dbname; 1686 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1687 run(command => $command, verbose => $verbose); 1688 unless ($success) { 1689 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1690 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1691 } 1692 } 1693 } 1694 1695 exit 0; 1696 } 1697 if ($stage eq "detrend.resid.exp") { 1698 die "--stage_id required for stage $stage\n" if !$stage_id; 1699 # this stage uses 'camtool' 1700 my $dettool = can_run('dettool') or die "Can't find dettool"; 1701 1702 # Get list of component imfiles 1703 # XXX may need a different my_die for each stage 1704 my $exps; # Array of component files 1705 my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run 1706 $command .= " -dbname $dbname" if defined $dbname; 1707 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1708 unless ($success) { 1709 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1710 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1711 } 1712 # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this 1713 if (@$stdout_buf == 0) { 1714 exit 0; 1715 } 1716 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1717 &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1718 1719 $exps = parse_md_list($metadata) or 1720 &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); 1721 1722 foreach my $exp (@$exps) { 1723 my $exp_id = $exp->{exp_id}; 1724 my $iteration = $exp->{iteration}; 1725 my $path_base = $exp->{path_base}; 1726 # my $class_id = $exp->{class_id} 1727 my $status = 1; 1728 # don't clean up unless the data needed to update is available 1729 # goto_scrubbed now requires the config file to not be present 1730 if ($mode eq "goto_cleaned") { 1731 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1732 1733 if (!$config_file or ! -e $config_file) { 1734 print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; 1735 $status = 0; 1736 } 1737 } 1738 elsif ($mode eq "goto_scrubbed") { 1739 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); 1740 1741 if ($config_file and -e $config_file) { 1742 print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; 1743 $status = 0; 1744 } 1745 } 1746 if ($status) { 1747 my @files = (); 1748 # delete the temporary image datafiles 1749 if ($mode eq "goto_purged") { 1750 # additional files to remove for 'purge' mode 1751 addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id); 1752 addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id); 1753 } 1754 # actual command to delete the files 1755 $status = &delete_files (\@files); 1756 } 1757 1758 if ($status) { 1759 my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; 1760 if ($mode eq "goto_cleaned") { 1761 $command .= " -data_state cleaned"; 1762 } 1763 if ($mode eq "goto_scrubbed") { 1764 $command .= " -data_state scrubbed"; 1765 } 1766 if ($mode eq "goto_purged") { 1767 $command .= " -data_state purged"; 1768 } 1769 $command .= " -dbname $dbname" if defined $dbname; 1770 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1771 run(command => $command, verbose => $verbose); 1772 unless ($success) { 1773 print STDERR " residexp had an issue setting the state:? $success $error_code\n"; 1774 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1775 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1776 } 1777 } else { 1778 my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; 1779 $command .= " -dbname $dbname" if defined $dbname; 1780 1781 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1782 run(command => $command, verbose => $verbose); 1783 unless ($success) { 1784 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1785 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); 1786 } 1787 exit $PS_EXIT_UNKNOWN_ERROR; 1788 } 1789 } 1790 # Check to see if we can mark the whole detRunSummary object as cleaned. 1791 1792 $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 1793 $command .= " -dbname $dbname" if defined $dbname; 1794 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); 1795 unless ($success) { 1796 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1797 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1798 } 1799 if (@$stdout_buf != 0) { 1800 $metadata = $mdcParser->parse(join "", @$stdout_buf) or 1801 &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1802 $exps = parse_md_list($metadata) or 1803 &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); 1804 1805 foreach my $exp (@$exps) { 1806 my $iteration = $exp->{iteration}; 1807 my $command; 1808 if ($mode eq "goto_cleaned") { 1809 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; 1810 } 1811 if ($mode eq "goto_scrubbed") { 1812 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; 1813 } 1814 if ($mode eq "goto_purged") { 1815 $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; 1816 } 1817 $command .= " -dbname $dbname" if defined $dbname; 1818 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1819 run(command => $command, verbose => $verbose); 1820 unless ($success) { 1821 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1822 &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); 1823 } 1824 } 1825 } 1826 exit 0; 1827 } 1828 565 1829 566 1830 die "ipp_cleanup.pl -stage $stage not yet implemented\n"; … … 570 1834 my $files = shift; # reference to a list of files to unlink 571 1835 1836 # open(TMPLOG,">>/tmp/czw.cleanup.log"); 1837 # flock(TMPLOG,2); 1838 572 1839 # this script is, of course, very dangerous. 573 1840 foreach my $file (@$files) { 574 1841 print STDERR "unlinking $file\n"; 1842 # print TMPLOG "$stage $stage_id $file"; 1843 # my $ff = $file; 1844 # $ff =~ s%^file://%%; 1845 # unless (-e $ff) { 1846 # print TMPLOG "\t File not found\n"; 1847 # } 1848 # else { 1849 # print TMPLOG "\n"; 1850 # } 1851 575 1852 $ipprc->file_delete($file); 576 1853 } 1854 1855 # flock(TMPLOG,8); 1856 # close(TMPLOG); 1857 577 1858 return 1; 578 1859 }
Note:
See TracChangeset
for help on using the changeset viewer.
