Changeset 36018

Timestamp:

Aug 23, 2013, 1:31:54 PM (13 years ago)

Author:

bills

Message:

Further progress on stack_summary postage stamp jobs.
Implementation ignores the ROI. The entire images are returned.
If a requested data product is not found (mask, variance, exp, or num images
from older processings) this is ignored

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (6 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (2 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (5 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -774 +774 @@
         }
 
+
         if ($skycells) {
+            # We have a list of skycells. Find matching projection cells.
+
+            #list of projection_cells that we've included
+            my %projection_cells_found;
             foreach my $skycell (@$skycells) {
                 my $tess_id = $skycell->{tess_id};
@@ -781 +786 @@
                 my $projection_cell = skycell_id_to_projection_cell($skycell_id);
 
-                print "$tess_id $skycell_id $projection_cell\n";
-                my $command = "$stacktool -dbname $imagedb -summary -tess_id $tess_id -projection_cell $projection_cell";
+                # only need to include a projection cell once
+                next if $projection_cells_found{$projection_cell};
+
+                $projection_cells_found{$projection_cell} = 1;
+
+                # print "$tess_id $skycell_id $projection_cell\n";
+
+                my $command = "$stacktool -dbname $imagedb -summary -tess_id $tess_id";
+                $command .= " -projection_cell $projection_cell";
 
                 $command .= " -filter $filter" if $filter;
@@ -791 +803 @@
             }
         }
-        # XXX: Lookup stackSummaries that match
-    }
+    }
+
     if ($results) {
         foreach my $summary (@$results) {
@@ -799 +811 @@
 
             # we need to flesh out the hashes returned with values that the parser expects
+            $summary->{stage} = 'stack_summary';
             $summary->{component} = $projection_cell;
             $summary->{stage_id} = $summary->{sass_id};
-            # $summary->{image} = "$path_base.$projection_cell.image.b1.fits";
-            $summary->{image} = "$path_base.$projection_cell";
+            $summary->{path_base} = "$path_base.$projection_cell";
+
+            # The stack_summary stage doesn't follow the usual ipp conventions for file rules
+            # and path_base. We leave it up to the program that runs the job to handle this.
+            # However, the parser uses the image parameter to set the job's outputBase (by removing the .fits)
+            # so we need to set it. Just use the new path_base.
+            $summary->{image} = $summary->{path_base};
             $summary->{state} = 'full';
             $summary->{data_state} = 'full';
@@ -809 +827 @@
             $summary->{row_index} = [];
             push @{$summary->{row_index}}, 0;
-            # XXX: is there anything else? 
+
+            # XXX: Are there any other required parameters? 
             # TODO: create a function to validate the components returned
-            # and make sure that any required elements are there
+            # and make sure that any required elements are there.
         }
     }
@@ -1970 +1989 @@
 }
 
-# not particularly elegant
+# Convert from skycell_id to projection_cell
+# This code is not particularly elegant but I think that it matches the SQL used
+# to create stackAssocations does. 
+
 sub skycell_id_to_projection_cell {
     my $skycell_id = shift;
+
+    # split skycell_id into '.' separated components
     my @strs = split '\.', $skycell_id;
 
     my $num = scalar @strs;
+
+    # projection_cell is 'skycell_id' plus all .num values except the last one
     my $projection_cell = $strs[0];
     for (my $i=1; $i < $num - 1; $i++) {

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -29 +29 @@
                     $PSTAMP_SELECT_BACKMDL
                     $PSTAMP_SELECT_JPEG
+                    $PSTAMP_SELECT_EXP
+                    $PSTAMP_SELECT_NUM
                     $PSTAMP_SELECT_UNCOMPRESSED
                     $PSTAMP_SELECT_INVERSE
@@ -71 +73 @@
 our $PSTAMP_SELECT_BACKMDL   = 32;
 our $PSTAMP_SELECT_JPEG      = 64;
-# unused 128
-# unused 256
+our $PSTAMP_SELECT_EXP       = 128;
+our $PSTAMP_SELECT_NUM       = 256;
 our $PSTAMP_SELECT_UNCOMPRESSED = 512;
 our $PSTAMP_SELECT_INVERSE      = 1024;

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -62 +62 @@
 $options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
 
+
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 if ($redirect_output) {
@@ -95 +96 @@
     my $argString;
     $argString = $params->{job_args};
+    
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
     my $stage = $params->{stage};
-
-    # XXX: should we do any other sanity checking?
-    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
+
+    if ($stage eq 'stack_summary') {
+
+        # remove options not supported by stack summary
+        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE 
+            | $PSTAMP_RESTORE_BACKGROUND);
+
+        # stackSummary outputs do not follow the usual IPP conventions for file names.
+        # The parser (actually Job.pm) has deferred handling this until here
+        update_stack_summary_filenames($params);
+
+    } elsif ($stage ne 'stack') {
+        # ignore options only supported by stack and stack_summary
+        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+    }
+   
 
     if ($stage eq "raw") {
@@ -181 +199 @@
     }
 
-    # my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-
-    my $command = "$ppstamp $outputBase $argString $fileArgs";
-    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
-    $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
-    $command .= " -stage $stage";
-    $command .= " -forheader $calibfile" if $calibfile;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
+    # otherwise we make copies of the input files to the outputs
+    my $use_ppstamp = ($stage ne 'stack_summary');
 
     my $exitStatus;
-    if (WIFEXITED($error_code)) {
-        $exitStatus = WEXITSTATUS($error_code);
-    } else {
-        print STDERR "ppstamp failed error_code: $error_code\n";
-        $exitStatus = $PS_EXIT_SYS_ERROR;
+    if ($use_ppstamp) {
+        my $command = "$ppstamp $outputBase $argString $fileArgs";
+        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
+        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
+        $command .= " -stage $stage";
+        $command .= " -forheader $calibfile" if $calibfile;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
+        } else {
+            print STDERR "ppstamp failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        # XXX: if stage is stack deal with EXP and NUM images if selected
+    } else {
+        $exitStatus = justCopyFiles($outputBase, \$options, $params);
     }
 
@@ -208 +233 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
-                           $PSTAMP_SELECT_MASK     => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
-                           $PSTAMP_SELECT_SOURCES  => "cmf",
-                           $PSTAMP_SELECT_JPEG     => "jpg");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits', 
+                           $PSTAMP_SELECT_MASK     => 'mk.fits',
+                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
+                           $PSTAMP_SELECT_SOURCES  => 'cmf',
+                           $PSTAMP_SELECT_JPEG     => 'jpg',
+                           $PSTAMP_SELECT_EXP      => 'exp.fits',
+                           $PSTAMP_SELECT_NUM      => 'num.fits');
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
 
         foreach my $key (keys (%extensions)) {
@@ -534 +562 @@
 }
 
+sub update_stack_summary_filenames {
+    my $params  = shift;
+
+    my $path_base = $params->{path_base};
+
+    $params->{image}  = $path_base . ".image.b1.fits";
+    $params->{mask}   = $path_base . ".mask.b1.fits";
+    $params->{weight} = $path_base . ".variance.b1.fits";
+    $params->{exp}    = $path_base . ".exp.b1.fits";
+    $params->{num}    = $path_base . ".num.b1.fits";
+
+}
+
+sub myCopy {
+    my ($dest, $src, $type, $dieOnFail) = @_;
+
+    my $result;
+    my $resolved = $ipprc->file_resolve($src);
+    if ($ipprc->file_exists($resolved)) {
+        $result = copy($resolved, $dest);
+    } else {
+        my $msg = "Specified source $type image $src not found.";
+        if ($dieOnFail) {
+            $msg .= "\n";
+        } else {
+            $msg .= " ignoring\n";
+        }
+        carp $msg;
+        $result = 0;
+    }
+    
+    if (!$result and $dieOnFail) {
+        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
+    }
+    return $result;
+}
+
+sub justCopyFiles {
+    my ($outputBase, $r_options, $params) = @_;
+
+    my $options = $$r_options;
+    if ($options & $PSTAMP_SELECT_IMAGE) {
+        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
+    }
+    if ($options & $PSTAMP_SELECT_MASK) {
+        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
+            $options &= ~$PSTAMP_SELECT_MASK;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
+            $options =  ~$PSTAMP_SELECT_VARIANCE;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXP) {
+        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXP;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUM) {
+        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+
+    $$r_options = $options;
+
+    return 0;
+}
+
 sub my_die
 {