Changeset 36680 for branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl

Timestamp:

Apr 21, 2014, 5:42:34 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/pstamp_job_run.pl (modified) (11 diffs)

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl

@@ -57 +57 @@
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
 
-$options = 1 if !$options;
-
-# We don't need to muggle anymore. Ignore those options
-$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
+# ppstamp requires an input file
+$options = $PSTAMP_SELECT_IMAGE if !$options;
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -78 +76 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
 my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
-my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
-my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
 
@@ -96 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
     my $stage = $params->{stage};
-
-    # XXX: should we do any other sanity checking?
-    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $nan_masked = 1;
-    my $muggle = 0;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
-        # Attempt to find or create a muggle image
-        $nan_masked = 0;
-        $muggle = 1;
-    }
+    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
+
+    if ($stage eq 'stack_summary') {
+
+        # remove options not supported by stack summary
+        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE 
+            | $PSTAMP_RESTORE_BACKGROUND);
+
+        # stackSummary outputs do not follow the usual IPP conventions for file names.
+        # The parser (actually Job.pm) has deferred handling this until here
+        update_stack_summary_filenames($params);
+
+    } elsif ($stage ne 'stack') {
+        # ignore options only supported by stack and stack_summary
+        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+    }
+   
 
     if ($stage eq "raw") {
@@ -122 +126 @@
     my @file_list = ($params->{image});
     
-    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
+    if ($options & $PSTAMP_SELECT_MASK) {
         $mask = $params->{mask};
         $fileArgs .= " -mask $mask";
@@ -162 +166 @@
     # find our output directory
     my $outdir = dirname($outputBase);
-    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-    
-    if ($muggle) {
-        # first see if the original uncensored images are around
-        if (check_for_backups($params, \$fileArgs)) {
-            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
-            print "Making stamps from backup images\n";
-        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
-            # user required uncensored but since stage isn't chip we can't rebuild them
-            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
-            # we can only restore pixels for chip stage images if the data has been updated.
-            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
-            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
-            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
-            # make stamps from uncensored images
-            $muggle = 0;
+
+    my ($calib_fd, $calibfile);
+    if ($stage eq 'chip' or $stage eq 'warp') {
+        my $cam_id = $params->{cam_id};
+        if (!$cam_id) {
+            carp "no cam_id found in job params\n";
+            exit $PS_EXIT_PROG_ERROR;
+        }
+        ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps);
+        close $calib_fd;
+
+        my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        my $exitStatus;
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
         } else {
-            # Try and replace the streaks from the recovery images
-
-            my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
-            $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
-            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
-            # find the "directory" of the input path_base
-            my $inputdir = dirname($image);
-            my $base = basename($image);
-            $tmpImage = "$tmproot/$base";
-
-            @file_list = ();
-
-            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
-            # stage work. Import the rule here.
-            my $recImage = "$inputdir/REC_$base";
-            my $newFileArgs = " -file $tmpImage";
-            $muggle_command .= " -image $image -recimage $recImage";
-            push @file_list, $recImage;
-
-            if ($mask) {
-                $base = basename($mask);
-                $tmpMask = "$tmproot/$base";
-                $newFileArgs .= " -mask $tmpMask";
-                my $recMask = "$inputdir/REC_$base";
-                $muggle_command .= " -mask $mask -recmask $recMask";
-                push @file_list, $recMask;
-            }
-
-            if ($variance) {
-                $base = basename($variance);
-                $tmpVariance = "$tmproot/$base";
-                $newFileArgs .= " -variance $tmpVariance";
-                my $recVariance = "$inputdir/REC_$base";
-                $muggle_command .= " -weight $variance -recweight $recVariance";
-                push @file_list, $recVariance;
-            }
-            if (check_files(0, @file_list)) {
-                # recovery files exist and are accessible restore the excised pixels
-
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $muggle_command, verbose => $verbose);
-                unless ($success) {
-                    my $exitStatus = WEXITSTATUS($error_code);
-                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
-                }
-
-                # set the ppstamp file arguments
-                $fileArgs = $newFileArgs;
-            } else {
-                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
-                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
-                }
-                # just make stamps from the censored images
-                print "Unable to restore uncensored images, will extract stamps from censored images\n";
-                # these files won't be used so zap them
-                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
-            }
-        }
-    }
-
-    # MAGIC IS DEAD
-    $nan_masked = 0;
-
-    my $command = "$ppstamp $outputBase $argString $fileArgs";
-    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
-    $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
-    $command .= " -stage $stage";
-    $command .= " -censor_masked" if $nan_masked;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+            print STDERR "psgetcalibinfo failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        exit $exitStatus if $exitStatus;
+
+        if (-s $calibfile == 0) {
+            print "no calibration information found for $cam_id\n";
+            $calibfile = undef;
+        }
+    }
+
+    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
+    # otherwise we make copies of the input files to the outputs
+    my $use_ppstamp = ($stage ne 'stack_summary');
 
     my $exitStatus;
-    if (WIFEXITED($error_code)) {
-        $exitStatus = WEXITSTATUS($error_code);
-    } else {
-        print STDERR "ppstamp failed error_code: $error_code\n";
-        $exitStatus = $PS_EXIT_SYS_ERROR;
+    if ($use_ppstamp) {
+        my $command = "$ppstamp $outputBase $argString $fileArgs";
+        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
+        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
+        $command .= " -stage $stage";
+        $command .= " -forheader $calibfile" if $calibfile;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
+        } else {
+            print STDERR "ppstamp failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        # XXX: if stage is stack deal with EXP and NUM images if selected
+    } else {
+        $exitStatus = justCopyFiles($outputBase, \$options, $params);
     }
 
@@ -270 +230 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
-                           $PSTAMP_SELECT_MASK     => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
-                           $PSTAMP_SELECT_SOURCES  => "cmf",
-                           $PSTAMP_SELECT_JPEG     => "jpg");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits', 
+                           $PSTAMP_SELECT_MASK     => 'mk.fits',
+                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
+                           $PSTAMP_SELECT_SOURCES  => 'cmf',
+                           $PSTAMP_SELECT_JPEG     => 'jpg',
+                           $PSTAMP_SELECT_EXP      => 'exp.fits',
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -298 +266 @@
         close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP || $exitStatus == $PSTAMP_NO_VALID_PIXELS) {
+        $jobStatus = $exitStatus;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -419 +387 @@
         my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $filter = $params->{filter};
+        $filter = ' ' if !$filter;
+        $filter = substr $filter, 0, 1;
         my $cmf_file;
 #        if ($stage ne 'chip') {
@@ -431 +402 @@
             if (!$therest or !$rownum or !$jobnum);
 
-        my $prefix = "${rownum}_${jobnum}_";
+        # XXX: Here we are assuming the form of the output file names
+        # if we change this in pstampparse we'll need to remember ....
+        # (the last time I forgot)
+        my $prefix = "${rownum}_${jobnum}_${filter}_";
 
         if ($cmf_file) {
@@ -523 +497 @@
 }
 
-sub check_for_backups {
-    my $params = shift;
-    my $r_fileArgs = shift;
-
-    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
-    my $results = runToolAndParse($command, $verbose);
-    my $dsComponent  = $results->[0];
-
-    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
-
-        print "magicDSFile state is full. Backup images should exist\n";
-
-        # replace the file names with the backup paths
-        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
-        my @args = split " ", $$r_fileArgs;
-
-        my $newFileArgs;
-        while (@args) {
-            my $a = shift @args;
-            my $f = shift @args;
-            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
-
-            my $backup_image_name = $ipprc->destreaked_filename($f);
-            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
-            $newFileArgs .= " $a $backup_image_name";
-        }
-        $$r_fileArgs = $newFileArgs;
-
-        return 1;
-    }
-
-    return 0;
-}
-
 my $neb;
 sub storage_object_exists
@@ -630 +570 @@
 }
 
+# stack_summary stage does not currently use proper file rules.
+# The parser defers handlint this to us..
+
+sub update_stack_summary_filenames {
+    my $params  = shift;
+
+    my $path_base = $params->{path_base};
+
+    $params->{image}  = $path_base . ".image.b1.fits";
+    $params->{mask}   = $path_base . ".mask.b1.fits";
+    $params->{weight} = $path_base . ".variance.b1.fits";
+    $params->{jpeg}   = $path_base . ".image.0.b1.jpeg";
+    $params->{exp}    = $path_base . ".exp.b1.fits";
+    $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
+}
+
+sub myCopy {
+    my ($dest, $src, $type, $dieOnFail) = @_;
+
+    my $result;
+    my $resolved = $ipprc->file_resolve($src);
+    if ($resolved and $ipprc->file_exists($resolved)) {
+        print "Copying $src to $dest\n" if $verbose;;
+        $result = copy($resolved, $dest);
+    } else {
+        my $msg = "Specified source $type image $src not found.";
+        if ($dieOnFail) {
+            $msg .= "\n";
+        } else {
+            $msg .= " ignoring\n";
+        }
+        carp $msg;
+        $result = 0;
+    }
+    
+    if (!$result and $dieOnFail) {
+        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
+    }
+    return $result;
+}
+
+sub justCopyFiles {
+    my ($outputBase, $r_options, $params) = @_;
+
+    print "Just copying files for $job_id.\n";
+
+    my $options = $$r_options;
+    if ($options & $PSTAMP_SELECT_IMAGE) {
+        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
+    }
+    if ($options & $PSTAMP_SELECT_MASK) {
+        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
+            $options &= ~$PSTAMP_SELECT_MASK;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
+            $options =  ~$PSTAMP_SELECT_VARIANCE;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_JPEG) {
+        if (!myCopy("$outputBase.jpg", $params->{jpeg}, 'jpeg', 0)) {
+            $options &= ~$PSTAMP_SELECT_JPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXP) {
+        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXP;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUM) {
+        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
+        }
+    }
+
+    $$r_options = $options;
+
+    print "Done with copy.\n";
+
+    return 0;
+}
+
 sub my_die
 {