Changeset 36122

Timestamp:

Sep 17, 2013, 11:22:17 AM (13 years ago)

Author:

bills

Message:

Support for creating postage stamp jobs for stack_summary stage.
The results are not actually postage stamps but the (relatively small)
full images.
Define bits in option mask for selecting exp and num images from stacks
and stack_summary stage, as well as jpegs for them.

Location:

tags/ipp-20130712

Files:

• Ohana/src/getstar/src/dvoImagesAtCoords.c (modified) (4 diffs, 1 prop)
• PS-IPP-PStamp (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (5 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (6 diffs)
• ippTools (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (5 diffs)
• ippTools/share/camtool_pendingexp.sql (modified) (1 prop)
• ippTools/share/disttool_definebyquery_sky.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_skycal.sql (copied) (copied from trunk/ippTools/share/disttool_definebyquery_skycal.sql )
• ippTools/share/disttool_pending_skycal.sql (copied) (copied from trunk/ippTools/share/disttool_pending_skycal.sql )
• ippTools/share/disttool_toadvance.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelexp.sql (modified) (1 prop)
• ippTools/share/releasetool_stacksummary.sql (copied) (copied from trunk/ippTools/share/releasetool_stacksummary.sql )
• ippTools/share/releasetool_summary.sql (copied) (copied from trunk/ippTools/share/releasetool_summary.sql )
• ippTools/share/stacktool_summary.sql (copied) (copied from trunk/ippTools/share/stacktool_summary.sql )
• ippTools/share/stacktool_tosummary.sql (modified) (1 prop)
• ippTools/share/staticskytool_export_skycalresult.sql (copied) (copied from trunk/ippTools/share/staticskytool_export_skycalresult.sql )
• ippTools/share/staticskytool_export_skycalrun.sql (copied) (copied from trunk/ippTools/share/staticskytool_export_skycalrun.sql )
• ippTools/share/staticskytool_skycalresult.sql (modified) (1 diff)
• ippTools/share/warptool_warped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/difftool.c (modified) (1 prop)
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (1 diff)
• ippTools/src/releasetool.c (modified) (3 diffs, 1 prop)
• ippTools/src/releasetool.h (modified) (1 diff)
• ippTools/src/releasetoolConfig.c (modified) (2 diffs, 1 prop)
• ippTools/src/stacktool.c (modified) (4 diffs, 1 prop)
• ippTools/src/stacktool.h (modified) (1 diff)
• ippTools/src/stacktoolConfig.c (modified) (3 diffs)
• ippTools/src/staticskytool.c (modified) (4 diffs)
• ippTools/src/staticskytool.h (modified) (1 diff)
• ippTools/src/staticskytoolConfig.c (modified) (2 diffs)
• pstamp (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/psgetcalibinfo (modified) (1 prop)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (5 diffs)
• pstamp/scripts/pstampparse.pl (modified) (15 diffs)
• pstamp/src/Makefile.am (modified) (1 diff)
• pstamp/src/ppstampMakeStamp.c (modified) (3 diffs)
• pstamp/src/pstamp.h (modified) (4 diffs)
• pstamp/test/gpc1/chip.100lines.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.100lines.txt )
• pstamp/test/gpc1/diff.difftype.txt (copied) (copied from trunk/pstamp/test/gpc1/diff.difftype.txt )
• pstamp/test/gpc1/fwhm_cuts.txt (copied) (copied from trunk/pstamp/test/gpc1/fwhm_cuts.txt )
• pstamp/test/gpc1/stack.calib.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.calib.txt )
• pstamp/test/gpc1/stack.invalid.mjd.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.invalid.mjd.txt )
• pstamp/test/gpc1/stack.md08.preview.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.md08.preview.txt )
• pstamp/test/gpc1/stack.sources.type.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.sources.type.txt )
• pstamp/test/gpc1/stacksummary.bycoord.all.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.bycoord.all.txt )
• pstamp/test/gpc1/stacksummary.bycoord.m31.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.bycoord.m31.txt )
• pstamp/test/gpc1/stacksummary.bycoord.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.bycoord.txt ) (1 diff)
• pstamp/test/gpc1/stacksummary.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.byid.txt )
• pstamp/test/gpc1/stacksummary.byskycell.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.byskycell.txt )
• pstamp/test/gpc1/stacksummary.getimage.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.getimage.txt )
• pstamp/test/gpc1/warp.byid.calib.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.byid.calib.txt )
• pstamp/test/gpc1/warp.invalidmjd.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.invalidmjd.txt )
• pstamp/test/gpc1/warp.md08.56414.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.md08.56414.txt )
• pstamp/test/maketestreq (modified) (1 diff)

tags/ipp-20130712/Ohana/src/getstar/src/dvoImagesAtCoords.c

@@ -9 +9 @@
 static int makePoint(double ra, double dec, Point **pointsOut);
 static int ListImagesAtCoords (Image *dbImages, Point *points, int Npoints);
+static int readPointFromFile(FILE *f, Point *pt);
 
 int main (int argc, char **argv) {
@@ -15 +16 @@
   int status;
   Image *dbImages;
-  int Npoints;
   FITS_DB db;
 
@@ -23 +23 @@
   
   Point *points;
+  int readStdin = 0;
+  int Npoints = 0;
   if (coordsFile) {
-      Npoints = readPoints(coordsFile, &points);
+      if (strcmp(coordsFile, "-") == 0) {
+        readStdin = 1;
+      } else {
+        Npoints = readPoints(coordsFile, &points);
+      }
   } else {
       Npoints = makePoint(cmd_line_ra, cmd_line_dec, &points);
   }
-  if (!Npoints) {
+  if (!Npoints && !readStdin) {
     exit(1);
   }
@@ -61 +67 @@
   }
   
-  if (MatchCoords (dbImages, NdbImages, points, Npoints)) {
-    ListImagesAtCoords(dbImages, points, Npoints);
-    exit(0);
+  if (readStdin) {
+    Point pt;
+    memset(&pt, 0, sizeof(pt));
+    pt.Nmatches = 0;
+    ALLOCATE(pt.matches, Match, 20);
+    pt.arrayLength = 20;
+
+    int status;
+    while ((status = readPointFromFile(stdin, &pt))) {
+      if (status < 0) {
+        fprintf(stdout, "ERROR malformed input line\n");
+        exit(1);
+      }
+      if (MatchCoords (dbImages, NdbImages, &pt, 1)) {
+        ListImagesAtCoords(dbImages, &pt, 1);
+      }
+      fprintf(stdout, "DONE\n");
+      fflush(stdout);
+      pt.Nmatches = 0;
+    }
   } else {
-    exit(PSTAMP_NO_OVERLAP);
-  }
+    if (MatchCoords (dbImages, NdbImages, points, Npoints)) {
+      ListImagesAtCoords(dbImages, points, Npoints);
+    } else {
+      exit(PSTAMP_NO_OVERLAP);
+    }
+  }
+  exit(0);
+}
+
+static int readPointFromFile(FILE *f, Point *pt) {
+    char buf[80];
+    char *good = fgets(buf, 80, f);
+    if (!good) {
+        return 0;
+    }
+    // fprintf(stderr, "READ: %s\n", buf);
+    int Nread = sscanf(buf, "%d %lf %lf\n", &pt->id, &pt->ra, &pt->dec);
+    if (Nread == 3) {
+        // got one
+        return 1;
+    } else if (Nread == -1) {
+        // all done time to go
+        return 0;
+    } else {
+        // fprintf(stderr, "malformed input line: %d\n", Nread);
+        return -1;
+    }
 }
 

tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
 use POSIX;
 use Time::HiRes qw(gettimeofday);
+use IO::Handle;
 
 my $dvo_verbose = 0;
@@ -244 +245 @@
     if (isnull($filter)) {
         $filter = undef;
+    }
+
+    if ($stage eq 'stack_summary') {
+        # stack_summary jobs are so different from others that we lookup in a specialized function
+        return lookup_stack_summary($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
     }
 
@@ -736 +742 @@
 }
 
+sub lookup_stack_summary {
+    my ($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose) = @_;
+
+    $row->{error_code} = $PSTAMP_NO_IMAGE_MATCH;
+
+    $component = '' if $component eq 'all';
+
+    my $req_type = $row->{REQ_TYPE};
+    my $results;
+    if ($req_type eq 'byid') {
+        my $sass_id = $row->{ID};
+        my $command = "$stacktool -dbname $imagedb -summary -sass_id $sass_id";
+        $results = runToolAndParse($command, $verbose);
+    } else {
+        my ($release_name, $survey, $default_tess_id) = get_release_info($row);
+        if (!$tess_id and $default_tess_id) {
+            $tess_id = $default_tess_id;
+        }
+
+        my $skycells;
+        if ($req_type eq 'bycoord') {
+            my $ra = $row->{CENTER_X};
+            my $dec = $row->{CENTER_Y};
+            $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
+        } elsif ($req_type eq 'byskycell') {
+            if (!defined $tess_id or !defined $component) {
+                print STDERR "Error: TESS_ID and  COMPONENT are required for REQ_TYPE byskycell\n";
+                $row->{error_code} = $PSTAMP_INVALID_REQUEST;
+            }
+            my $skycell = {
+                tess_id    => $tess_id,
+                component => $component
+            };
+            push @$skycells, $skycell
+        } else {
+            print STDERR "Error: $req_type is not a valid REQ_TYPE for IMG_TYPE stack_summary\n";
+            $row->{error_code} = $PSTAMP_INVALID_REQUEST;
+            return undef;
+        }
+
+        if ($skycells) {
+            # We have a list of skycells. Find matching projection cells.
+            # and then matching rows in stackSummary
+
+            my %projection_cells_found;
+            foreach my $skycell (@$skycells) {
+                my $tess_id = $skycell->{tess_id};
+                my $skycell_id = $skycell->{component};
+
+                my $projection_cell = skycell_id_to_projection_cell($skycell_id);
+
+                # only need to include a projection cell once
+                next if $projection_cells_found{$projection_cell};
+
+                $projection_cells_found{$projection_cell} = 1;
+
+                # print "$tess_id $skycell_id $projection_cell\n";
+
+                my $command;
+                if ($release_name or $survey) {
+                    $command = "$releasetool -dbname $imagedb";
+                    if ($release_name) {
+                        $command .= " -release_name $release_name";
+                    } else {
+                        $command .= " -priority_order";
+                    }
+                    $command .= " -surveyName $survey";
+                } else {
+                    $command = "$stacktool -dbname $imagedb";
+                }
+                $command .= " -summary -tess_id $tess_id";
+                $command .= " -projection_cell $projection_cell";
+
+                $command .= " -filter $filter" if $filter;
+                $command .= " -data_group $data_group" if $data_group;
+
+                my $these_results = runToolAndParse($command, $verbose);
+                push @$results, @$these_results if $these_results;
+            }
+        }
+    }
+
+    if ($results) {
+        foreach my $summary (@$results) {
+            my $path_base = $summary->{path_base};
+            my $projection_cell = $summary->{projection_cell};
+
+            # we need to flesh out the hashes returned with values that the parser expects
+            $summary->{stage} = 'stack_summary';
+            $summary->{component} = $projection_cell;
+            $summary->{stage_id} = $summary->{sass_id};
+            $summary->{path_base} = "$path_base.$projection_cell";
+
+            # The stack_summary stage doesn't follow the usual ipp conventions for file rules
+            # and path_base. We leave it up to the program that runs the job to handle this.
+            # However, the parser uses the image parameter to set the job's outputBase (by removing the .fits)
+            # so we need to set it. Just use the new path_base.
+            $summary->{image} = $summary->{path_base};
+            $summary->{state} = 'full';
+            $summary->{data_state} = 'full';
+            $summary->{fault} = 0;
+            $summary->{imagedb} = $imagedb;
+            $summary->{row_index} = [];
+            push @{$summary->{row_index}}, 0;
+
+            # XXX: Are there any other required parameters? 
+            # TODO: create a function to validate the components returned
+            # and make sure that any required elements are there.
+        }
+    }
+
+    return $results;
+}
+
 sub lookup_diff {
     my $ipprc    = shift;
@@ -1169 +1289 @@
 
     my @lines;
-    {
+    my $looked_up_fast;
+    if ($requested_tess_id) {
+        # try using the last method first
+        $looked_up_fast = lookup_skycells_fast($ipprc, \@lines, $requested_tess_id, $ra, $dec, $verbose);
+    } 
+    
+    if (!$looked_up_fast) {
         my $command = "$whichimage $ra $dec";
         $command .= " --tess_id $requested_tess_id" if $requested_tess_id;
@@ -1878 +2004 @@
     return $diff_mode;
 }
+
+sub setErrorCodesForRows {
+    my $rowList = shift;
+    my $error_code = shift;
+
+    foreach my $row (@$rowList) {
+        $row->{error_code} = $error_code;
+    }
+
+}
+
+# Convert from skycell_id to projection_cell
+# This code is not particularly elegant but I think that it matches the SQL used
+# to create stackAssocations does. 
+
+sub skycell_id_to_projection_cell {
+    my $skycell_id = shift;
+
+    # split skycell_id into '.' separated components
+    my @strs = split '\.', $skycell_id;
+
+    my $num = scalar @strs;
+
+    # projection_cell is 'skycell_id' plus all .num values except the last one
+    my $projection_cell = $strs[0];
+    for (my $i=1; $i < $num - 1; $i++) {
+        $projection_cell .= ".$strs[$i]";
+    }
+
+    return $projection_cell;
+}
+
+# lookup_skycells_fast() 
+# Lookup using a "skycell server" process which is implemented by running dvoImagesAtCoords in "server" mode.
+# Repeated calls to this function will reuse an existing process. This avoids having to loading the
+# tessellation over and over again for each skycell lookup.
+# When this parse process exists, the pipes are closed which causes the skycell server to exit.
+
+my $scs_tess_id;    # tess_id for existing scs (skycell server) (if any)
+my $scs_out;        # reference filehandle for sending data to the scs
+my $scs_in;         # filehandle for response from scs
+
+sub lookup_skycells_fast {
+    my ($ipprc, $results, $tess_id, $ra, $dec, $verbose) = @_;
+
+    # see if we need to start the scs (skycell server)
+
+    my $very_verbose = 0;
+
+    if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) {
+        if ($scs_tess_id) {
+            # tess_id has changed, close down existing scs
+            close $scs_out;
+            $scs_out = undef;
+            close $scs_in;
+            $scs_in = undef;
+            $scs_tess_id = undef;
+        }
+
+        # convert tess_id to a directory name
+        my $tess_dir = $ipprc->tessellation_catdir( $tess_id );
+        unless ($tess_dir) {
+            print STDERR "Unrecognized tess_id: $tess_id\n";
+            return 0;
+        }
+
+        # convert tess_dir to an absolute directory name
+        my $tess_dir_resolved = $ipprc->convert_filename_absolute( $tess_dir );
+        unless ($tess_dir_resolved and -d $tess_dir_resolved) {
+            print STDERR "failed to resolve tessellation directory for $tess_id\n";
+            return 0;
+        }
+
+        # start an scs
+        print "Starting skycell server for $tess_dir_resolved.\n";
+
+        # create pipes for communicating with the server
+        pipe $scs_in, SERVER_WRITER;
+        pipe SERVER_READER, $scs_out;
+
+        # set our output to be unbuffered
+        $scs_out->autoflush(1);
+
+        # FORK
+        my $scs_pid = fork();
+
+        if (!$scs_pid) {
+            unless (defined $scs_pid) {
+                # This is still in parent process. No joy.
+                print STDERR "fork of skycell server failed: $!";
+                return 0;
+            }
+
+            # This code is running in the child process - the skycell server
+            # Close file handles for the parent's ends of the pipes
+            close $scs_in;
+            close $scs_out;
+            # close stdio filehandles. we are about to redirect them
+            # (STDERR stays open pointing to the parser log)
+            close STDIN;
+            close STDOUT;
+
+            # redirect our stdio file handles to the proper end of the pipes
+            open (STDIN,  "<&SERVER_READER")   or die "\nSCS: failed to redirect stdin";
+            open (STDOUT, ">>&SERVER_WRITER")  or die "\nSCS: failed to redirect stdout";
+
+            # All set. Now exec dvoImagesAtCoords
+
+            my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -";
+
+            print STDERR "SCS: execing $command\n" if $very_verbose;
+
+            unless(exec $command) {
+                # note parent will notice that we are gone due to the pipes getting broken
+                # when this child dies.
+                die "SCS: failed to exec $command";
+            }
+        }
+
+        # This code is running in the parent process (the parser).
+        # We have succesfully launched the scs process.
+        # Close file handles for the scs' end of the pipes.
+        close SERVER_WRITER;
+        close SERVER_READER;
+
+        # remember the tess_id 
+        $scs_tess_id = $tess_id;
+    }
+
+    # send coordinates to the skycell server and wait for the results 
+    # (which come nearly instantly which is the point of doing this)
+
+    # If pipes to the skycell server die, don't abort. 
+    # Our reads and writes detect errors and act sensibly.
+    local $SIG{PIPE} = 'IGNORE';
+
+    # write the coordinates to the pipe
+    my $write_ok = print $scs_out "1 $ra $dec\n";
+    if (!$write_ok) {
+        # server process probably didn't start properly or died. Fail.
+        # Caller will attempt to use the conventional method.
+        print STDERR "write to skycell server failed. Error is $!\n";
+        return 0;
+    }
+
+    print STDERR "  sent coordinates to skycell server\n" if $very_verbose;
+
+    my $received_done = 0;
+
+    # Wait for response. If pipe breaks the read returns with no data.
+    while (my $line = <$scs_in>) {
+        chomp $line;
+        if ($line eq 'DONE') {
+            # No more results for these coordinates.
+            $received_done = 1;
+            print STDERR "    received DONE\n" if $very_verbose;
+            last;
+        }
+        print STDERR "    received $line\n" if $very_verbose;
+
+        # The caller expects the output to be in the format used by whichimage
+        # which omits the point number and includes the tess_id.
+        my ($ptnum, $raout, $decout, $skycell_id) = split " ", $line;
+
+        my $out = "$raout $decout $tess_id $skycell_id";
+
+        # print the result to the log
+        print "$out\n" if $verbose;
+
+        push @$results, $out;
+    }
+
+    print STDERR "Out of wait for responses loop\n" if $very_verbose;
+
+    if (!$received_done) {
+        # something has gone wrong.
+        print STDERR "Read loop exited without receiving DONE.\n";
+
+        die "BAILING out" if $very_verbose;
+
+        # forget about the existing server instance
+        $scs_tess_id = undef;
+        close $scs_out;
+        close $scs_in;
+
+        # drop any results received so far
+        @$results = ();
+        return 0;
+    }
+
+    return 1;
+}
+
         
 

tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -29 +29 @@
                     $PSTAMP_SELECT_BACKMDL
                     $PSTAMP_SELECT_JPEG
+                    $PSTAMP_SELECT_EXP
+                    $PSTAMP_SELECT_NUM
                     $PSTAMP_SELECT_UNCOMPRESSED
                     $PSTAMP_SELECT_INVERSE
@@ -35 +37 @@
                     $PSTAMP_USE_IMFILE_ID
                     $PSTAMP_NO_WAIT_FOR_UPDATE
-                    $PSTAMP_REQUEST_UNCENSORED
-                    $PSTAMP_REQUIRE_UNCENSORED
+                    $PSTAMP_SELECT_EXPJPEG
+                    $PSTAMP_SELECT_NUMJPEG
                     $PSTAMP_SUCCESS
                     $PSTAMP_FIRST_ERROR_CODE
@@ -71 +73 @@
 our $PSTAMP_SELECT_BACKMDL   = 32;
 our $PSTAMP_SELECT_JPEG      = 64;
-# unused 128
-# unused 256
+our $PSTAMP_SELECT_EXP       = 128;
+our $PSTAMP_SELECT_NUM       = 256;
 our $PSTAMP_SELECT_UNCOMPRESSED = 512;
 our $PSTAMP_SELECT_INVERSE      = 1024;
@@ -82 +84 @@
 our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
 
-# these bits will be repurposed
-our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
-our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
+our $PSTAMP_SELECT_EXPJPEG     = 0x10000;
+our $PSTAMP_SELECT_NUMJPEG     = 0x20000;
+
+# these bits have been repurposed. They were only exposed to MOPS and IFA and they have adapted.
+#our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
+#our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
 
 # job and result codes
@@ -97 +102 @@
 our $PSTAMP_INVALID_REQUEST  = 21;
 our $PSTAMP_UNKNOWN_PROJECT  = 22;
-our $PSTAMP_UNKNOWN_PRODUCT  = 22;  #this error code was mis-named it is left for compatabiliyt
+our $PSTAMP_UNKNOWN_PRODUCT  = 22;  #this error code was a typo it is left for compatabiliy
 our $PSTAMP_NO_IMAGE_MATCH   = 23;
 our $PSTAMP_NOT_DESTREAKED   = 24;
 our $PSTAMP_NOT_AVAILABLE    = 25;
-our $PSTAMP_GONE             = 26;  # this value is used in ippTools
+our $PSTAMP_GONE             = 26;  # this value is also used in ippTools
 our $PSTAMP_NO_JOBS_QUEUED   = 27;
 our $PSTAMP_NO_OVERLAP       = 28;
 our $PSTAMP_NOT_AUTHORIZED   = 29;
+our $PSTAMP_NO_VALID_PIXELS  = 30;
 
 
@@ -143 +149 @@
 PSTAMP_NO_OVERLAP
 PSTAMP_NOT_AUTHORIZED
-PSTAMP_NOT_AUTHORIZED
+PSTAMP_NO_VALID_PIXELS
 );
 

tags/ipp-20130712/ippTools/share/Makefile.am

@@ -193 +193 @@
         disttool_definebyquery_sky.sql \
         disttool_definebyquery_sky_singlefilter.sql \
+        disttool_definebyquery_skycal.sql \
         disttool_definebyquery_stack.sql \
         disttool_definebyquery_warp.sql \
@@ -207 +208 @@
         disttool_pending_raw.sql \
         disttool_pending_sky.sql \
+        disttool_pending_skycal.sql \
         disttool_pending_stack.sql \
         disttool_pending_warp.sql \
@@ -398 +400 @@
         stacktool_tosummary.sql \
         stacktool_addsummary.sql \
+        stacktool_summary.sql \
         staticskytool_definebyquery_select.sql \
         staticskytool_definebyquery_select_by_dg.sql \
@@ -408 +411 @@
         staticskytool_export_input.sql \
         staticskytool_export_result.sql \
+        staticskytool_export_skycalrun.sql \
+        staticskytool_export_skycalresult.sql \
         staticskytool_inputs.sql \
         staticskytool_todo.sql \
@@ -476 +481 @@
         releasetool_definerelstack_with_skycal.sql \
         releasetool_listrelstack.sql \
+        releasetool_summary.sql \
         releasetool_definerelgroup_select_lap.sql \
         releasetool_definerelgroup_select_data_group.sql \
         releasetool_definerelgroup_select_exp_data_group.sql \
         releasetool_definerelgroup_select_exp_lap.sql \
+        releasetool_stacksummary.sql \
         releasetool_pendingrelgroup.sql
 

tags/ipp-20130712/ippTools/share/disttool_definebyquery_sky.sql

@@ -3 +3 @@
     staticskyRun.sky_id AS stage_id,
     CAST(0 AS SIGNED) AS magicked,
-    -- run tag in the form 'sky.$skycell_id.$stack_id'
+    -- run tag in the form 'sky.$skycell_id.$sky_id'
     CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag,
     staticskyRun.label,

tags/ipp-20130712/ippTools/share/disttool_toadvance.sql

@@ -262 +262 @@
         AND distComponent.fault = 0
 UNION
+-- skycal stage
+-- NOTE this assumes that there is only one component per skycalRun
+-- (one skycell)
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
+    FROM distRun
+    JOIN skycalResult on stage_id = skycal_id
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'skycal'
+        AND distComponent.component IS NOT NULL
+        AND distComponent.fault = 0
+UNION
 -- SSdiff stage
 SELECT

tags/ipp-20130712/ippTools/share/staticskytool_skycalresult.sql

@@ -2 +2 @@
     skycalResult.*,
     skycalRun.stack_id,
+    skycalRun.workdir,
     stackRun.filter,
     skycalRun.state,
     skycalRun.label,
     skycalRun.data_group,
+    skycalRun.dist_group,
     skycalRun.sky_id,
     skycell.*

tags/ipp-20130712/ippTools/src/disttool.c

@@ -375 +375 @@
         }
         if (dist_group) {
-            psStringAppend(&query, " AND (sticskyRun.dist_group = '%s')", dist_group);
+            psStringAppend(&query, " AND (staticskyRun.dist_group = '%s')", dist_group);
         }
         // (static)sky stage doesn't require magic
+        magic = false;
+    } else if (!strcmp(stage, "skycal")) {
+        query = pxDataGet("disttool_definebyquery_skycal.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (skycalRun.label = '%s')", label);
+        }
+        if (dist_group) {
+            psStringAppend(&query, " AND (skycalRun.dist_group = '%s')", dist_group);
+        }
+        // skycal stage doesn't require magic
         magic = false;
     } else if (!strcmp(stage, "SSdiff")) {

tags/ipp-20130712/ippTools/src/pstamptool.c

@@ -1428 +1428 @@
     psFree(where);
 
-    psStringAppend(&query, " ORDER BY priority DESC, req_id, dep_id");
+    psStringAppend(&query, " GROUP BY dep_id ORDER BY priority DESC, MIN(req_id), dep_id");
 
     // treat limit == 0 as "no limit"

tags/ipp-20130712/ippTools/src/releasetool.c

@@ -47 +47 @@
 static bool updaterelstackMode(pxConfig *config);
 static bool listrelstackMode(pxConfig *config);
+static bool summaryMode(pxConfig *config);
 static bool definerelgroupMode(pxConfig *config);
 static bool pendingrelgroupMode(pxConfig *config);
@@ -84 +85 @@
         MODECASE(RELEASETOOL_MODE_UPDATERELSTACK,   updaterelstackMode);
         MODECASE(RELEASETOOL_MODE_LISTRELSTACK,     listrelstackMode);
+        MODECASE(RELEASETOOL_MODE_SUMMARY,          summaryMode);
 
         MODECASE(RELEASETOOL_MODE_DEFINERELGROUP,   definerelgroupMode);
@@ -1080 +1082 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC");
+        psStringAppend(&query, "\nAND priority > 0 order by priority DESC, stack_id");
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "relStack", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+static bool summaryMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-sass_id",     "stackAssociation.sass_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",    "relStack.stack_id", "==");
+//    PXOPT_COPY_S64(config->args, where, "-skycal_id",   "relStack.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
+//    PXOPT_COPY_STR(config->args, where, "-state",       "relStack.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "relStack.filter", "LIKE");
+//    PXOPT_COPY_F32(config->args, where, "-mjd_min",    "stackSumSkyfile.mjd_obs", ">=");
+//    PXOPT_COPY_F32(config->args, where, "-mjd_max",    "stackSumSkyfile.mjd_obs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",     "relStack.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "relStack.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-projection_cell",  "stackAssociation.projection_cell", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-stack_data_group",  "stackRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-data_group",  "stackAssociation.data_group", "LIKE");
+//    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
+
+    PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+    pxskycellAddWhere(config, where);
+
+//    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "IFNULL(skycalResult.fwhm_major, 999)", ">=");
+//    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "IFNULL(skycalResult.fwhm_major, 0)", "<=");
+
+    PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "==");
+
+    psString query = pxDataGet("releasetool_summary.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    if (priority_order) {
+        psStringAppend(&query, "\nAND priority > 0 order by priority DESC, stack_id");
     }
 

tags/ipp-20130712/ippTools/src/releasetool.h

@@ -38 +38 @@
     RELEASETOOL_MODE_TOCALIBSTACK,
     RELEASETOOL_MODE_LISTRELSTACK,
+    RELEASETOOL_MODE_SUMMARY,
     RELEASETOOL_MODE_DEFINERELGROUP,
     RELEASETOOL_MODE_PENDINGRELGROUP,

tags/ipp-20130712/ippTools/src/releasetoolConfig.c

@@ -248 +248 @@
     psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
+    // -summary
+    psMetadata *summaryArgs = psMetadataAlloc();
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL,  "-sass_id", 0,   "select by released SASS ID", 0);
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL,  "-relstack_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL,  "-stack_id", 0,   "select by stack ID", 0);
+//    psMetadataAddS64(summaryArgs, PS_LIST_TAIL,  "-skycal_id", 0,   "select by skycal ID", 0);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL);
+//    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-state", 0,        "select by released stack state", NULL);
+
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-filter", 0,       "select by filter name (LIKE comparison)", NULL);
+
+    pxskycellAddArguments(summaryArgs);
+
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-projection_cell", 0, "select by projection_cell (LIKE comparision)", NULL);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL);
+
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-data_group", 0, "select by stackAsociation.data_group (LIKE comparison)", NULL);
+
+    psMetadataAddStr(summaryArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
+    psMetadataAddS64(summaryArgs,  PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0);
+
+    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-priority_order",   0, "order by release priority", false);
+
+    psMetadataAddU64(summaryArgs,  PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
     // -definerelgroup
@@ -322 +351 @@
     PXOPT_ADD_MODE("-definerelstack",     "define a released stack",    RELEASETOOL_MODE_DEFINERELSTACK,  definerelstackArgs);
     PXOPT_ADD_MODE("-listrelstack",       "list released stacks",      RELEASETOOL_MODE_LISTRELSTACK,    listrelstackArgs);
+    PXOPT_ADD_MODE("-summary",            "list stackSummaryes for released stacks", RELEASETOOL_MODE_SUMMARY,    summaryArgs);
 
     PXOPT_ADD_MODE("-definerelgroup",     "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP,  definerelgroupArgs);

tags/ipp-20130712/ippTools/src/stacktool.c

@@ -44 +44 @@
 static bool tosummaryMode(pxConfig *config);
 static bool addsummaryMode(pxConfig *config);
+static bool summaryMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
@@ -84 +85 @@
         MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
         MODECASE(STACKTOOL_MODE_ADDSUMMARY,            addsummaryMode);
+        MODECASE(STACKTOOL_MODE_SUMMARY,               summaryMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
@@ -748 +750 @@
     PXOPT_COPY_S64(config->args, where, "-stack_id",  "stackRun.stack_id",   "==");
     PXOPT_COPY_STR(config->args, where, "-label",     "stackRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
     PXOPT_COPY_STR(config->args, where, "-state",     "stackRun.state",     "==");
     PXOPT_COPY_S64(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
@@ -1567 +1570 @@
 }
 
+static bool summaryMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-sass_id", "stackSummary.sass_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-projection_cell", "stackAssociation.projection_cell", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackAssociation.tess_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter", "stackAssociation.filter", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
+    pxAddLabelSearchArgs(config, where, "-data_group", "stackAssociation.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+//    PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "LIKE");
+//    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+//    PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackSumSkyfile.mjd_obs", ">=");
+//    PXOPT_COPY_F64(config->args, where, "-mjd_obs_end", "stackSumSkyfile.mjd_obs", "<=");
+//    PXOPT_COPY_S16(config->args, where, "-background_model","stackSumSkyfile.background_model", "==");
+
+//    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("stacktool_summary.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("stacktool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "stackSummary", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
 static bool pendingcleanuprunMode(pxConfig *config)
 {

tags/ipp-20130712/ippTools/src/stacktool.h

@@ -38 +38 @@
     STACKTOOL_MODE_TOSUMMARY,
     STACKTOOL_MODE_ADDSUMMARY,
+    STACKTOOL_MODE_SUMMARY,
     STACKTOOL_MODE_PENDINGCLEANUPRUN,
     STACKTOOL_MODE_PENDINGCLEANUPSKYFILE,

tags/ipp-20130712/ippTools/src/stacktoolConfig.c

@@ -121 +121 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "search by state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0,            "search by label", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0,       "search by data_group", 0);
     psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault",  0,           "search by fault code", 0);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sass_id", 0,          "search by stack association ID", 0);
@@ -264 +265 @@
     psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,     "set summary path base", NULL);
 
+    // -summary
+    psMetadata *summaryArgs = psMetadataAlloc();
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-sass_id", 0,  "search by stack association ID", 0);
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0,  "search by stack ID", 0);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0,   "search by tessellation ID", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-projection_cell", 0,   "search by projection cell ID", NULL);
+//    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-state", 0,     "search by state", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0,    "search by filter (LIKE comparison)", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by stackAssociation data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0,   "search by skycell ID", NULL);
+//     psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by stackRun dist_group (LIKE comparison)", NULL);
+
+//    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(summaryArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
+
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
@@ -324 +342 @@
     PXOPT_ADD_MODE("-updatesumskyfile",      "update fault code for sumskyfile",     STACKTOOL_MODE_UPDATESUMSKYFILE,          updatesumskyfileArgs);
     PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs);
+    PXOPT_ADD_MODE("-summary",              "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs);
     PXOPT_ADD_MODE("-addsummary",           "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs);
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs);

tags/ipp-20130712/ippTools/src/staticskytool.c

@@ -50 +50 @@
 static bool revertskycalresultMode(pxConfig *config);
 static bool updateskycalresultMode(pxConfig *config);
+static bool exportskycalrunMode(pxConfig *config);
+static bool importskycalrunMode(pxConfig *config);
 
 static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state);
@@ -88 +90 @@
         MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode);
         MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT,      skycalresultMode);
+        MODECASE(STATICSKYTOOL_MODE_EXPORTSKYCALRUN,   exportskycalrunMode);
+        MODECASE(STATICSKYTOOL_MODE_IMPORTSKYCALRUN,   importskycalrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -895 +899 @@
 bool importrunMode(pxConfig *config)
 {
-# if (0)
-  unsigned int nFail;
-
-  int numImportTables = 2;
-
-  char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"};
+  return false;
+}
+
+bool exportskycalrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+
+  int numExportTables = 2;
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
-
-  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
-
-#ifdef notdef
-  fprintf (stderr, "---- input ----\n");
-  psMetadataPrint (stderr, input, 1);
-#endif
-
-  if (!pxCheckImportVersion(config, input)) {
-      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+  // XXX unused PXOPT_LOOKUP_S64(det_id, config->args, "-sky_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+      psError(PS_ERR_UNKNOWN, false, "failed to open output file");
       return false;
   }
-  psMetadataItem *item = psMetadataLookup (input, "stackRun");
-  psAssert (item, "entry not in input?");
-  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-  psMetadataItem *entry = psListGet (item->data.list, 0);
-  assert (entry);
-  assert (entry->type == PS_DATA_METADATA);
-  stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md);
-  stackRunInsertObject (config->dbh, stackRun);
-
-  // fprintf (stdout, "---- stack run ----\n");
-  // psMetadataPrint (stderr, entry->data.md, 1);
-
-  for (int i = 0; i < numImportTables; i++) {
-    psMetadataItem *item = psMetadataLookup (input, tables[i]);
-    psAssert (item, "entry not in input?");
-    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-    switch (i) {
-      case 0:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md);
-          stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-
-      case 1:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md);
-          stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-    }
+
+  if (!pxExportVersion(config, f)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+    return false;
   }
-
-# endif
-  return true;
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "==");
+
+  ExportTable tables [] = {
+    {"skycalRun", "staticskytool_export_skycalrun.sql"},
+    {"skycalResult", "staticskytool_export_skycalresult.sql"},
+  };
+
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+      return false;
+    }
+
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
+      psFree(output);
+      return false;
+    }
+
+    if (clean) {
+        if (!strcmp(tables[i].tableName, "skycalRun")) {
+            if (!pxSetStateCleaned("skycalRun", "state", output)) {
+                psFree(output);
+                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+                return false;
+            }
+        }
+    }
+
+      // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+  }
+
+    fclose (f);
+
+    return true;
+}
+
+bool importskycalrunMode(pxConfig *config)
+{
+  return false;
 }
 
@@ -986 +1022 @@
 
 }
-
-# if (0)
-        // now we need to loop over all requested filters and check that each is found
-        // XXX is this needed?  haven't we required that we only match the requested filters
-        // and that N unique filters are found?
-
-        psVector *found = psVectorAlloc(inputs->n, PS_TYPE_U8);
-        psVectorInit (found, 0);
-
-        psMetadataItem *filter = NULL;
-        psListIterator *iter = psListIteratorAlloc (filters->data.list, PS_LIST_HEAD, false);
-        while ((filter = psListGetAndIncrement(iter))) {
-
-            bool foundOne = false;
-
-            psAssert (filter->type == PS_DATA_STR, "filter is not a string?");
-            for (int j = 0; !foundOne && (j < inputs->n); j++) {
-                if (found->data.U8[j]) continue;
-                psMetadata *inputRow = inputs->data[j]; // Row from select
-
-                // pull out the skycell_id, tess_id, filter
-                psString inFilter = psMetadataLookupStr(&status, row, "filter");
-                psAssert (status);
-
-                found->data.U8[j] = true;
-                foundOne = true;
-            }
-            if (!foundOne) {
-                // this required filter was not found in the inputs, skip the entry
-                skip();
-            }
-        }           
-# endif
 
 static bool defineskycalrunMode(pxConfig *config)

tags/ipp-20130712/ippTools/src/staticskytool.h

@@ -43 +43 @@
     STATICSKYTOOL_MODE_SKYCALRESULT,
     STATICSKYTOOL_MODE_REVERTSKYCALRESULT,
+    STATICSKYTOOL_MODE_EXPORTSKYCALRUN,
+    STATICSKYTOOL_MODE_IMPORTSKYCALRUN,
 } staticskytoolMode;
 

tags/ipp-20130712/ippTools/src/staticskytoolConfig.c

@@ -269 +269 @@
     psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
+    // -exportskycalrun
+    psMetadata *exportskycalrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "export this skycal ID (required)", 0);
+    psMetadataAddStr(exportskycalrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL);
+    psMetadataAddU64(exportskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(exportskycalrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false);
+
+    // -importrun
+    psMetadata *importskycalrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importskycalrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL);
+  
+
     psFree(now);
 
@@ -291 +303 @@
     PXOPT_ADD_MODE("-updateskycal",  "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs);
     PXOPT_ADD_MODE("-skycalresult",  "Get result of skycal run",  STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs);
+    PXOPT_ADD_MODE("-exportskycalrun",     "Export skycal run",   STATICSKYTOOL_MODE_EXPORTSKYCALRUN, exportskycalrunArgs);
+    PXOPT_ADD_MODE("-importskycalrun",     "Import skycal run",   STATICSKYTOOL_MODE_IMPORTSKYCALRUN, importskycalrunArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

tags/ipp-20130712/pstamp/scripts/pstamp_get_image_job.pl

@@ -65 +65 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $data = $mdcParser->parse(join "", (<INPUT>)) or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+my $data = $mdcParser->parse(join "", (<INPUT>)) 
+    or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+
 my $components = parse_md_list($data);
 my $n = scalar @$components;
@@ -83 +85 @@
     print STDERR "stage_id is $stage_id\n";
     print STDERR "path_base is $path_base\n";
-    print STDERR "path_base is $path_base\n";
     print STDERR "CAMERA is $camera\n";
     print STDERR "magicked is " . (defined $magicked ? $magicked : "undefined") . "\n";
@@ -89 +90 @@
 
 if (!$camera or !$path_base or !$component or !$stage_id or !$stage) {
-       my_die("failed to parse params from: $params_file", $PS_EXIT_UNKNOWN_ERROR);
+       my_die("One or more parameters are missing in: $params_file", $PS_EXIT_UNKNOWN_ERROR);
 }
 

tags/ipp-20130712/pstamp/scripts/pstamp_job_run.pl

@@ -57 +57 @@
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
 
-$options = 1 if !$options;
-
-# We don't need to muggle anymore. Ignore those options
-$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
+# ppstamp requires an input file
+$options = $PSTAMP_SELECT_IMAGE if !$options;
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -95 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
     my $stage = $params->{stage};
-
-    # XXX: should we do any other sanity checking?
-    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
+
+    if ($stage eq 'stack_summary') {
+
+        # remove options not supported by stack summary
+        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE 
+            | $PSTAMP_RESTORE_BACKGROUND);
+
+        # stackSummary outputs do not follow the usual IPP conventions for file names.
+        # The parser (actually Job.pm) has deferred handling this until here
+        update_stack_summary_filenames($params);
+
+    } elsif ($stage ne 'stack') {
+        # ignore options only supported by stack and stack_summary
+        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+    }
+   
 
     if ($stage eq "raw") {
@@ -181 +196 @@
     }
 
-    # my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-
-    my $command = "$ppstamp $outputBase $argString $fileArgs";
-    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
-    $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
-    $command .= " -stage $stage";
-    $command .= " -forheader $calibfile" if $calibfile;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
+    # otherwise we make copies of the input files to the outputs
+    my $use_ppstamp = ($stage ne 'stack_summary');
 
     my $exitStatus;
-    if (WIFEXITED($error_code)) {
-        $exitStatus = WEXITSTATUS($error_code);
-    } else {
-        print STDERR "ppstamp failed error_code: $error_code\n";
-        $exitStatus = $PS_EXIT_SYS_ERROR;
+    if ($use_ppstamp) {
+        my $command = "$ppstamp $outputBase $argString $fileArgs";
+        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
+        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
+        $command .= " -stage $stage";
+        $command .= " -forheader $calibfile" if $calibfile;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
+        } else {
+            print STDERR "ppstamp failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        # XXX: if stage is stack deal with EXP and NUM images if selected
+    } else {
+        $exitStatus = justCopyFiles($outputBase, \$options, $params);
     }
 
@@ -208 +230 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
-                           $PSTAMP_SELECT_MASK     => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
-                           $PSTAMP_SELECT_SOURCES  => "cmf",
-                           $PSTAMP_SELECT_JPEG     => "jpg");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits', 
+                           $PSTAMP_SELECT_MASK     => 'mk.fits',
+                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
+                           $PSTAMP_SELECT_SOURCES  => 'cmf',
+                           $PSTAMP_SELECT_JPEG     => 'jpg',
+                           $PSTAMP_SELECT_EXP      => 'exp.fits',
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -534 +564 @@
 }
 
+# stack_summary stage does not currently use proper file rules.
+# The parser defers handlint this to us..
+
+sub update_stack_summary_filenames {
+    my $params  = shift;
+
+    my $path_base = $params->{path_base};
+
+    $params->{image}  = $path_base . ".image.b1.fits";
+    $params->{mask}   = $path_base . ".mask.b1.fits";
+    $params->{weight} = $path_base . ".variance.b1.fits";
+    $params->{jpeg}   = $path_base . ".image.0.b1.jpeg";
+    $params->{exp}    = $path_base . ".exp.b1.fits";
+    $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
+}
+
+sub myCopy {
+    my ($dest, $src, $type, $dieOnFail) = @_;
+
+    my $result;
+    my $resolved = $ipprc->file_resolve($src);
+    if ($resolved and $ipprc->file_exists($resolved)) {
+        print "Copying $src to $dest\n" if $verbose;;
+        $result = copy($resolved, $dest);
+    } else {
+        my $msg = "Specified source $type image $src not found.";
+        if ($dieOnFail) {
+            $msg .= "\n";
+        } else {
+            $msg .= " ignoring\n";
+        }
+        carp $msg;
+        $result = 0;
+    }
+    
+    if (!$result and $dieOnFail) {
+        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
+    }
+    return $result;
+}
+
+sub justCopyFiles {
+    my ($outputBase, $r_options, $params) = @_;
+
+    print "Just copying files for $job_id.\n";
+
+    my $options = $$r_options;
+    if ($options & $PSTAMP_SELECT_IMAGE) {
+        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
+    }
+    if ($options & $PSTAMP_SELECT_MASK) {
+        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
+            $options &= ~$PSTAMP_SELECT_MASK;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
+            $options =  ~$PSTAMP_SELECT_VARIANCE;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_JPEG) {
+        if (!myCopy("$outputBase.jpg", $params->{jpeg}, 'jpeg', 0)) {
+            $options &= ~$PSTAMP_SELECT_JPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXP) {
+        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXP;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUM) {
+        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
+        }
+    }
+
+    $$r_options = $options;
+
+    print "Done with copy.\n";
+
+    return 0;
+}
+
 sub my_die
 {

tags/ipp-20130712/pstamp/scripts/pstampparse.pl

@@ -51 +51 @@
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76 +78 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116 +118 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175 +182 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212 +224 @@
 }
 
-if (!$rows) {
-    # we got called so a valid pstamp request header was found so we can only assume
-    # that the file is invalid
-    print STDERR "Invalid request file\n";
+my $nRows = $rows ? scalar @$rows : 0;
+print "\n$nRows rows read from request file\n";
+
+unless ($nRows) {
+    # pstamp_job_run was invoked so the request file must have contained a valid header
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
+    print STDERR "Invalid request file.\n";
     insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
     exit 0;
 }
 
-my $nRows = scalar @$rows;
-print "\n$nRows rows read from request file\n";
 
 
@@ -227 +242 @@
 my $imageList;
 my $stage;
+my $big_limit = 100;    # XXX: this should be in a configuration file some where
 foreach my $row (@$rows) {
 
     if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/)  and
         ($nRows > 200 or $num_jobs > 200) and $req_id and !$no_update) {
-
-        # this is a big request and it came from one of the "high priority" channels
-        # change the label to one that runs with lower priority
+        # This is a "big" request and it came from one of the "high priority" 
+        # channels and doesn't have a specific label assigned.
+        # Change the label to a value that its jobs run with lower priority.
         my $old_label = $label;
 
+        print "\nChanging label for big request from $old_label to $label\n";
         $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
-        print "\nChanging label for big request from $old_label to $label\n";
 
         my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
@@ -261 +277 @@
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
@@ -298 +316 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
+        # get_image jobs are quite expensive in terms of space so we are currently restricting them
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' 
+            or ($req_type eq 'byskycell' and $stage eq 'stack')
+            or ($stage eq 'stack_summary')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -307 +328 @@
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        if ($job_type eq 'get_image') {
+        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
             $row->{COMPONENT} = 'all';
         } else {
@@ -396 +417 @@
     }
 
-    if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
+    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
+        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -469 +490 @@
     my $start_locate = gettimeofday();
 
-    print "\nCalling new_locate_images for row: $rownum\n";
+    print "\nCalling locate_images_for_row for row: $rownum\n";
 
     $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
@@ -533 +554 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -544 +565 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -713 +704 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);
 
@@ -1175 +1180 @@
     }
     if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
-        ($img_type eq "stack") or ($img_type eq "diff")) {
+        ($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
         return 1;
     } else {

tags/ipp-20130712/pstamp/src/Makefile.am

@@ -1 @@
-bin_PROGRAMS = ppstamp pstamprequest pstampdump
+bin_PROGRAMS = ppstamp pstampdump
 
 include_HEADERS = \

tags/ipp-20130712/pstamp/src/ppstampMakeStamp.c

@@ -20 +20 @@
 static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance);
 static bool copySources(pmReadout *outputReadout, pmReadout *inputReadout, pmReadout *astromReadout, psRegion extractRegion);
+static bool imageHasValidPixels(psImage *image);
 
 // convert the input chip's transforms to the output
@@ -337 +338 @@
             status = false;
             break;
+        }
+        if (!imageHasValidPixels(outReadout->image)) {
+            return PSTAMP_NO_VALID_PIXELS;
         }
         if (readout->variance) {
@@ -885 +889 @@
     return true;
 }
+
+static bool
+imageHasValidPixels(psImage *image) {
+    // check F32 image and return true if any pixel is finite
+    if (image->type.type != PS_TYPE_F32) {
+        return true;
+    }
+    for (int y=0; y<image->numRows; y++) {
+        for (int x=0; x<image->numCols; x++) {
+            psF32 pixel = image->data.F32[y][x];
+            if (isfinite(pixel)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}

tags/ipp-20130712/pstamp/src/pstamp.h

@@ -15 +15 @@
         PSTAMP_DUP_REQUEST      = 20,
         PSTAMP_INVALID_REQUEST  = 21,
-        PSTAMP_UNKNOWN_PRODUCT  = 22,
+        PSTAMP_UNKNOWN_PROJECT  = 22,
         PSTAMP_NO_IMAGE_MATCH   = 23,
         PSTAMP_NOT_DESTREAKED   = 24,
@@ -21 +21 @@
         PSTAMP_GONE             = 26,
         PSTAMP_NO_JOBS_QUEUED   = 27,
-        PSTAMP_NO_OVERLAP       = 28
+        PSTAMP_NO_OVERLAP       = 28,
+        PSTAMP_NOT_AUTHORIZED   = 29,
+        PSTAMP_NO_VALID_PIXELS  = 30,
 } pstampJobErrors;
 
@@ -34 +36 @@
 #define PSTAMP_SELECT_BACKMDL       32
 #define PSTAMP_SELECT_JPEG          64
-// unused                           128
-// unused                           256
+#define PSTAMP_SELECT_EXP           128
+#define PSTAMP_SELECT_NUM           256
 #define PSTAMP_SELECT_UNCOMPRESSED  512
 #define PSTAMP_SELECT_INVERSE       1024
@@ -44 +46 @@
 
 #define PSTAMP_NO_WAIT_FOR_UPDATE   32768
+#ifdef notdef
 #define PSTAMP_REQUEST_UNCENSORED  0x10000
 #define PSTAMP_REQUIRE_UNCENSORED  0x20000
+#endif
 
 #define PSTAMP_CENTER_IN_PIXELS 1

tags/ipp-20130712/pstamp/test/gpc1/stacksummary.bycoord.txt

@@ -4 +4 @@
 CHANGEME 2 PROCESS null
 #  ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT
-1     359.767   -30.2009     100    100   2      stamp      455         gpc1      3PI         null     bycoord   stack_summary  null RINGS.V3 null   LAP.ThreePI.20130717%      r        0      0        null      0        0    | 
+1     359.767   -30.2009     100    100   2      stamp      1        gpc1      3PI         null     bycoord   stack_summary  null RINGS.V3 null   LAP.ThreePI.20130717%      r        0      0        null      0        0    | 

tags/ipp-20130712/pstamp/test/maketestreq

@@ -48 +48 @@
     'label=s'           => \$label,
     'email=s'           => \$email,
+    'username=s'        => \$email, # I always forget what the option is
 
     'dbname=s'           => \$dbname,