Index: trunk/ippScripts/scripts/chip_imfile.pl =================================================================== --- trunk/ippScripts/scripts/chip_imfile.pl (revision 25432) +++ trunk/ippScripts/scripts/chip_imfile.pl (revision 25442) @@ -19,4 +19,5 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); use File::Temp qw( tempfile ); @@ -143,7 +144,115 @@ ## XXX make the feature more general? my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE'); - if ($useDeburnedImage && $deburned) { - $uri =~ s/fits$/burn.fits/; - &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + if ($useDeburnedImage) { + ## Check that we have required programs: +# print STDERR "Inside burntool loop!\n"; + my $regtool = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## Check the current burntool processing version: + my $regtool_state_cmd = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -limit 1"; + if (defined($dbname)) { + $regtool_state_cmd .= " -dbname $dbname"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $regtool_state_cmd, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $regData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse regtool metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $regData2 = parse_md_list($regData); + + my $burntoolState = 999; + foreach my $regEntry (@$regData2) { +# print "$regEntry->{exp_id} $regEntry->{burntool_state}\n"; + + if ($regEntry->{exp_id} == $exp_id) { + $burntoolState = $regEntry->{burntool_state}; + } + } + if ($burntoolState == 999) { + &my_die("Unable to find burntool_state in metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Read camera config to get the current good burntool state : + ## XXX This is extremely slow. Any better way to do this? + my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $camera_config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $camData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + my $burntoolStateGood; + foreach my $camEntry (@$camData) { + if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") { + $burntoolStateGood = $camEntry->{value}; + } + } + + if (abs($burntoolState) != $burntoolStateGood) { + &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## We now know that we have an image that has been burntooled. + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX", + UNLINK => !$save_temps, SUFFIX => '.fits' ); + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + # funpack into the temp file. + my $funpack_cmd = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data. + my ($burntoolTable_uri, $burntoolTable_uriReal); + my $burntool_cmd = "$burntool "; + + if ($burntoolState == -1 * $burntoolStateGood) { # Burntool information stored in an external table. + $burntoolTable_uri = $uri; + $burntoolTable_uri =~ s/fits$/burn.tbl/; + $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + unless ($ipprc->file_exists($burntoolTable_uri)) { + &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR); + } + + $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} apply=t"; + } + elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table. + $burntool_cmd .= "$tempName apply=t"; + } + else { + &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Run burntool to change the pixels of tempfile, and repoint $uri to that file. + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $burntool_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + $uri = $tempName; + unless ($ipprc->file_exists($uri)) { + &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + +# print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n"; +# exit(100); } Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25432) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25442) @@ -123,7 +123,8 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { +# if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file is missing\n"; + . " because config file ($config_file) is missing\n"; $status = 0; } @@ -132,7 +133,7 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for chipRun $stage_id $class_id " - . " because config file is present\n"; + . " because config file ($config_file) is present\n"; $status = 0; } @@ -144,7 +145,7 @@ # delete the temporary image datafiles - addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -236,6 +237,6 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - - if (!$config_file or ! -e $config_file) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; $status = 0; @@ -245,5 +246,5 @@ my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -329,5 +330,5 @@ my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . " because config file is missing\n"; @@ -338,5 +339,5 @@ my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . " because config file is present\n"; @@ -345,4 +346,5 @@ } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); @@ -433,16 +435,14 @@ my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . " because config file is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if ($config_file and -e $config_file) { + + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . " because config file is present\n"; @@ -451,4 +451,5 @@ } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); @@ -535,16 +536,14 @@ my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . " because config file ($config_file) is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if ($config_file and -e $config_file) { + + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . " because config file ($config_file) is present\n"; @@ -553,4 +552,5 @@ } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); @@ -581,8 +581,8 @@ } - print STDERR "MY FILES: @files\n"; +# print STDERR "MY FILES: @files\n"; $status = &delete_files(\@files); } - print STDERR "MY STATUS: $status\n"; +# print STDERR "MY STATUS: $status\n"; if ($status) { my $command = "$difftool -diff_id $stage_id"; @@ -675,5 +675,5 @@ my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for fakeRun $stage_id $class_id " . " because config file is missing\n"; @@ -684,5 +684,5 @@ my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " . " because config file is present\n"; @@ -795,5 +795,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " . " because config file is missing\n"; @@ -804,5 +804,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " . " because config file is present\n"; @@ -942,5 +942,5 @@ my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -950,5 +950,5 @@ my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1094,4 +1094,5 @@ if ($status) { + my @files = (); # delete the temporary image datafiles # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. @@ -1227,5 +1228,4 @@ run(command => $command, verbose => $verbose); unless ($success) { - print STDERR "WTF: $success TTT $error_code QQQ\n"; $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); @@ -1316,5 +1316,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1324,5 +1324,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1450,5 +1450,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1458,5 +1458,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1591,5 +1591,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id $class_id " . " because config file is missing\n"; @@ -1600,5 +1600,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for $stage $stage_id $class_id " . " because config file is present\n"; @@ -1731,5 +1731,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1739,5 +1739,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1833,26 +1833,36 @@ { my $files = shift; # reference to a list of files to unlink - -# open(TMPLOG,">>/tmp/czw.cleanup.log"); -# flock(TMPLOG,2); +# my $test_verbose = 1; + +# if ($test_verbose == 1) { +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); +# } # this script is, of course, very dangerous. foreach my $file (@$files) { - print STDERR "unlinking $file\n"; -# print TMPLOG "$stage $stage_id $file"; -# my $ff = $file; -# $ff =~ s%^file://%%; -# unless (-e $ff) { -# print TMPLOG "\t File not found\n"; + print STDERR "unlinking $stage $stage_id $file"; + unless ($ipprc->file_exists($file)) { + print STDERR "\t File not found\n"; + } + else { + print STDERR "\n"; + } + +# if ($test_verbose == 1) { +# print TMPLOG "$stage $stage_id $file"; + +# else { +# print TMPLOG "\n"; +# } # } -# else { -# print TMPLOG "\n"; -# } $ipprc->file_delete($file); } -# flock(TMPLOG,8); -# close(TMPLOG); +# if ($test_verbose == 1) { +# flock(TMPLOG,8); +# close(TMPLOG); +# } return 1;