Changeset 35489

Timestamp:

May 2, 2013, 2:15:28 PM (13 years ago)

Author:

bills

Message:

Merge a whole bunch of changes from recent development.
Areas affected

Postage Stamp Server: support for version 2 of the request interface
ippTools/ippScripts/ippTasks: release tracking, lap groups (for staticsky queuing)
ippTools: integrate a couple of fixes from Heather which she has been running in the ippdvo user's build

Location:

tags/ipp-20130307

Files:

• DataStoreServer (modified) (1 prop)
• DataStoreServer/web/php (copied) (copied from trunk/DataStoreServer/web/php )
• DataStoreServer/web/php/getsmf.php (copied) (copied from trunk/DataStoreServer/web/php/getsmf.php )
• PS-IPP-PStamp (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (40 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (7 diffs)
• dbconfig (modified) (1 prop)
• dbconfig/changes.txt (modified) (2 diffs, 1 prop)
• dbconfig/lap.md (modified) (1 diff)
• ippScripts (modified) (1 prop)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/mergedvodb_merge.pl (modified) (1 diff)
• ippScripts/scripts/minidvodb_premerge.pl (modified) (5 diffs)
• ippScripts/scripts/queuestaticsky.pl (copied) (copied from trunk/ippScripts/scripts/queuestaticsky.pl )
• ippScripts/scripts/relgroup_exp_list.pl (modified) (6 diffs)
• ippScripts/scripts/whichimage (modified) (2 diffs)
• ippTasks (modified) (1 prop)
• ippTasks/lapgroup.pro (copied) (copied from trunk/ippTasks/lapgroup.pro )
• ippTasks/pstamp.pro (modified) (1 diff)
• ippTasks/release.pro (copied) (copied from trunk/ippTasks/release.pro )
• ippTasks/survey.pro (modified) (6 diffs)
• ippTools (modified) (1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (2 diffs)
• ippTools/share/addtool_find_processedexp_cam.sql (modified) (1 diff)
• ippTools/share/laptool_filtersforgroup.sql (copied) (copied from trunk/ippTools/share/laptool_filtersforgroup.sql )
• ippTools/share/laptool_pendinggroup.sql (copied) (copied from trunk/ippTools/share/laptool_pendinggroup.sql )
• ippTools/share/laptool_revertgroup.sql (copied) (copied from trunk/ippTools/share/laptool_revertgroup.sql )
• ippTools/share/pstamptool_completedreq.sql (modified) (1 diff)
• ippTools/share/pstamptool_getdependent.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (5 diffs, 1 prop)
• ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (modified) (1 diff)
• ippTools/share/releasetool_definerelstack.sql (modified) (2 diffs)
• ippTools/share/releasetool_definerelstack_with_skycal.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelexp.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelstack.sql (copied) (copied from trunk/ippTools/share/releasetool_listrelstack.sql )
• ippTools/share/releasetool_pendingrelgroup.sql (modified) (1 diff)
• ippTools/src (modified) (1 prop)
• ippTools/src/laptool.c (modified) (3 diffs)
• ippTools/src/laptool.h (modified) (1 diff)
• ippTools/src/laptoolConfig.c (modified) (3 diffs)
• ippTools/src/pstamptool.c (modified) (16 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (3 diffs)
• ippTools/src/releasetool.c (modified) (17 diffs)
• ippTools/src/releasetoolConfig.c (modified) (5 diffs)
• pstamp (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/psmkreq (modified) (14 diffs)
• pstamp/scripts/pstamp_finish.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_request_file (modified) (9 diffs)
• pstamp/scripts/pstamp_webrequest.pl (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (36 diffs)
• pstamp/src/ppstampParseCamera.c (modified) (3 diffs)
• pstamp/src/pstampdump.c (modified) (2 diffs)
• pstamp/test/gpc1 (copied) (copied from trunk/pstamp/test/gpc1 )
• pstamp/test/gpc1/chip.bycoord.norelease.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.bycoord.norelease.txt )
• pstamp/test/gpc1/chip.bycoord.preview.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.bycoord.preview.txt )
• pstamp/test/gpc1/chip.bycoord.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.bycoord.txt )
• pstamp/test/gpc1/chip.bydiff.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.bydiff.txt )
• pstamp/test/gpc1/chip.byexp.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.byexp.txt )
• pstamp/test/gpc1/chip.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/chip.byid.txt )
• pstamp/test/gpc1/diff.bydiff.txt (copied) (copied from trunk/pstamp/test/gpc1/diff.bydiff.txt )
• pstamp/test/gpc1/diff.byid.inverse.txt (copied) (copied from trunk/pstamp/test/gpc1/diff.byid.inverse.txt )
• pstamp/test/gpc1/diff.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/diff.byid.txt )
• pstamp/test/gpc1/extver.1.txt (copied) (copied from trunk/pstamp/test/gpc1/extver.1.txt )
• pstamp/test/gpc1/raw.bycoord.txt (copied) (copied from trunk/pstamp/test/gpc1/raw.bycoord.txt )
• pstamp/test/gpc1/raw.byexp.txt (copied) (copied from trunk/pstamp/test/gpc1/raw.byexp.txt )
• pstamp/test/gpc1/raw.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/raw.byid.txt )
• pstamp/test/gpc1/stack.bycoord.3pi.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.bycoord.3pi.txt )
• pstamp/test/gpc1/stack.bydiff.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.bydiff.txt )
• pstamp/test/gpc1/stack.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.byid.txt )
• pstamp/test/gpc1/stack.byskycell.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.byskycell.txt )
• pstamp/test/gpc1/stack.md08.txt (copied) (copied from trunk/pstamp/test/gpc1/stack.md08.txt )
• pstamp/test/gpc1/test_qub_ps_request_20130426_225546.tbl (copied) (copied from trunk/pstamp/test/gpc1/test_qub_ps_request_20130426_225546.tbl )
• pstamp/test/gpc1/tests_todo (copied) (copied from trunk/pstamp/test/gpc1/tests_todo )
• pstamp/test/gpc1/warp.bycoord.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.bycoord.txt )
• pstamp/test/gpc1/warp.bycoord.wrongskycell.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.bycoord.wrongskycell.txt )
• pstamp/test/gpc1/warp.bydiff.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.bydiff.txt )
• pstamp/test/gpc1/warp.byexp.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.byexp.txt )
• pstamp/test/gpc1/warp.byid.noverlap.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.byid.noverlap.txt )
• pstamp/test/gpc1/warp.byid.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.byid.txt )
• pstamp/test/gpc1/warp.byskycell.txt (copied) (copied from trunk/pstamp/test/gpc1/warp.byskycell.txt )
• pstamp/test/maketestreq (copied) (copied from trunk/pstamp/test/maketestreq )

tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -15 +15 @@
 our @EXPORT_OK = qw( 
                     locate_images
+                    locate_images_for_row
                     resolve_project
                     getCamRunByCamID
@@ -46 +47 @@
 my $last_project = "";
 
-my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
+my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool, $releasetool);
 my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
 
 my $default_data_groups = "";
+
+# old locate_images() function. Still used by dqueryparse
+# Postage stamp server now uses locate_images_for_row()
 
 sub locate_images {
@@ -56 +60 @@
     my $rowList  = shift;   # required
     my $req_type = shift;   # required
-    my $img_type = shift;   # required
+    my $stage = shift;   # required
     my $id       = shift;   # required unless req_type eq bycoord or byskycell
     my $tess_id  = shift;
@@ -78 +82 @@
            ($req_type ne "byskycell");
 
-    if (!$default_data_groups) {
+    if (!$data_group and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
@@ -106 +110 @@
         my $x = $row->{CENTER_X};
         my $y = $row->{CENTER_Y};
-        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
         return $results;
     }
 
-    if (($req_type eq "byid") and ($img_type eq "diff")) {
+    if (($req_type eq "byid") and ($stage eq "diff")) {
         # lookups of all of the information for diff images requires a two level lookup to
         # get the exposure information. Switching the req_type allows us to keep that code
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
         return $results;
     }
@@ -122 +126 @@
         # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
         # the image from the diffRun
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
         if (!$results) {
             return undef;
         }
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             # lookup_diff has done all of the work
             return $results;
@@ -132 +136 @@
 
         my $image = $results->[0];
-        if ($img_type eq "raw") {
+        if ($stage eq "raw") {
             $req_type = "byid";
             $id = $image->{exp_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "chip") {
+        } elsif ($stage eq "chip") {
             $req_type = "byid";
             $id = $image->{chip_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "warp") {
+        } elsif ($stage eq "warp") {
             $req_type = "byid";
             $id = $image->{warp_id};
@@ -148 +152 @@
             return undef if !$id;
             # fall through and lookup by warp_id
-        } elsif ($img_type eq "stack") {
+        } elsif ($stage eq "stack") {
             $req_type = "byid";
             $id = $image->{stack_id};
@@ -155 +159 @@
         } else {
             # This is checked this elsewhere?
-            print STDERR "Error: $img_type is an unknown image type\n";
+            print STDERR "Error: $stage is an unknown image type\n";
             return undef;
         }
     } elsif ($req_type eq "byskycell") {
-        if (($img_type eq "raw") or ($img_type eq "chip")) {
+        if (($stage eq "raw") or ($stage eq "chip")) {
             print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
             return undef;
@@ -169 +173 @@
     }
 
-    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
-        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+        0, 0,   # fwhm cuts are not applied here
+        undef, undef, undef, $verbose);
+
+    return $results;
+}
+
+sub locate_images_for_row {
+    my $ipprc    = shift; 
+    my $imagedb  = shift;
+    my $camera   = shift;
+    my $row      = shift;
+    my $verbose  = shift;
+
+    my $req_type  = $row->{REQ_TYPE};
+
+    # we die in response to bad data in request files
+    # The caller is responsible for updating the database
+    # pstampparse.pl error checks now so this shouldn't happen
+    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) 
+        if ($req_type ne "byid") and
+           ($req_type ne "byexp") and 
+           ($req_type ne "bycoord") and 
+           ($req_type ne "bydiff") and 
+           ($req_type ne "byskycell");
+
+    my $stage     = $row->{IMG_TYPE};
+    my $id        = $row->{ID};
+    my $component = $row->{COMPONENT};
+    my $tess_id   = $row->{TESS_ID};
+
+    my $filter    = $row->{REQFILT};
+    my $mjd_min   = $row->{MJD_MIN};
+    my $mjd_max   = $row->{MJD_MAX};
+    my $data_group = $row->{DATA_GROUP};
+    if (isnull($data_group) and !$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
+
+    my $rownum     = $row->{ROWNUM};
+    my $job_type   = $row->{JOB_TYPE};
+    my $option_mask= $row->{OPTION_MASK};
+
+    my $dateobs_begin;
+    my $dateobs_end;
+    if (!iszero($mjd_min)) {
+        $dateobs_begin = mjd_to_dateobs($mjd_min);
+    }
+    if (!iszero($mjd_max)) {
+        $dateobs_end = mjd_to_dateobs($mjd_max);
+    }
+    if (isnull($tess_id)) {
+        $tess_id = undef;
+    }
+    if (isnull($data_group)) {
+        $data_group = undef;
+    }
+    if (isnull($filter)) {
+        $filter = undef;
+    }
+
+    if ($req_type eq "bycoord") {
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, 0, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        return $results;
+    }
+
+    # for compatability with the lower level functions that haven't been converted to take a single row
+    my $rowList = [$row];
+
+    if (($req_type eq "byid") and ($stage eq "diff")) {
+        # lookups of all of the information for diff images requires a two level lookup to
+        # get the exposure information. Switching the req_type allows us to keep that code
+        # in one place
+        $req_type = "bydiff";
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
+        return $results;
+    }
+
+    my ($release_name, $survey, $default_tess_id) = get_release_info($row);
+    if (!$tess_id and $default_tess_id) {
+        $tess_id = $default_tess_id;
+    }
+
+    if ($req_type eq "bydiff") {
+        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
+        # the image from the diffRun
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
+        if (!$results) {
+            return undef;
+        }
+        if ($stage eq "diff") {
+            # lookup_diff has done all of the work
+            return $results;
+        } 
+
+        my $image = $results->[0];
+        if ($stage eq "raw") {
+            $req_type = "byid";
+            $id = $image->{exp_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "chip") {
+            $req_type = "byid";
+            $id = $image->{chip_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "warp") {
+            $req_type = "byid";
+            $id = $image->{warp_id};
+            $component = $image->{skycell_id};
+            return undef if !$id;
+            # fall through and lookup by warp_id
+        } elsif ($stage eq "stack") {
+            $req_type = "byid";
+            $id = $image->{stack_id};
+            return undef if !$id;
+            # fall though and lookup by stack_id
+        } else {
+            # This is checked this elsewhere?
+            print STDERR "Error: $stage is an unknown image type\n";
+            return undef;
+        }
+    } elsif ($req_type eq "byskycell") {
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
+            return undef;
+        }
+        if (!$tess_id or !$component) {
+            print STDERR "component and tess_id are required for REQ_TYPE byskycell\n";
+            return undef;
+        }
+    }
+
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
+        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+        getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}),
+        undef, $release_name, $survey,
+        $verbose);
 
     return $results;
@@ -195 +338 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $fwhm_min = shift;
+    my $fwhm_max = shift;
+    my $selectedAstrom = shift;
+    my $release_name = shift;
+    my $survey = shift;
     my $verbose  = shift;
 
@@ -219 +367 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    my $row = $rowList->[0];
+
     # all rows have the same center type
-    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
+    my $use_releasetool = 0;
+    my $release_args;
+    if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
+        if ($release_name or $survey) {
+            $use_releasetool = 1;
+            if ($release_name) {
+                $release_args = " -release_name $release_name";
+            } else {
+                $release_args = " -priority_order";
+            }
+            $release_args .= " -surveyName $survey" if $survey;
+        }
+    }
 
     my $component_args;
@@ -251 +415 @@
         # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
         # choose the chips containing the center pixel by calling selectComponets() below
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $class_id = $component;
-                $component_args = " -class_id $class_id";
-            }
-        } else {
-            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        } else {
+            $command = "$chiptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -processedimfile";
+                if ($component and $component ne 'all') {
+                    $class_id = $component;
+                    $component_args = " -class_id $class_id";
+                }
+            } else {
+                $command .= " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        }
         # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
         # With the mdc file it has all of the information that it needs.
@@ -275 +445 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($stage eq "warp") {
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$warptool -warped -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $skycell_id = $component;
-                $component_args = " -skycell_id $skycell_id";
-            }
-        } else {
-            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+    } elsif ($stage eq 'warp') {
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -listrelexp $release_args";
+            $skycell_id = $component;
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        } else {
+            $command = "$warptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -warped";
+                if ($component and $component ne 'all') {
+                    $skycell_id = $component;
+                    $component_args = " -skycell_id $skycell_id";
+                }
+            } else {
+                $command .= " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        }
         $image_name   = "PSWARP.OUTPUT";
         $mask_name    = "PSWARP.OUTPUT.MASK";
@@ -294 +471 @@
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($stage eq "diff") {
+        # XXX: is this path ever used for diff images anymore?
         if ($component or $use_imfile_id or !$skycenter) {
             $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb";
@@ -314 +492 @@
     } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $imagedb";
-        # only consider stacks in full state.
-        $command .= " -state full";
-        $id_opt = "-stack_id";
+        if ($use_releasetool) {
+            $command = "$releasetool -listrelstack -dbname $imagedb $release_args";
+            $base_name = 'stack_path_base';
+            my $stack_type = $row->{RUN_TYPE};
+            if (!isnull($stack_type)) {
+                if (lc($stack_type) eq 'notnightly') {
+                    $command .= " -stack_type deep -stack_type reference";
+                } else {
+                    $command .= " -stack_type $stack_type";
+                }
+            }
+        } else {
+            $command = "$stacktool -sumskyfile -dbname $imagedb";
+            # only consider stacks in full state.
+            $command .= " -state full";
+            $id_opt = "-stack_id";
+            $base_name = 'path_base';
+        }
         $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne  'all';
 
@@ -331 +523 @@
         # this is wrong but gets the right answer. Need to figure out how to find the
         # rule properly
-        #$cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        # $cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $psf_name    = "PPSTACK.TARGET.PSF";
-        $base_name   = "path_base";
     } else {
-        die "Unknown IMG_TYPe supplied: $stage";
+        die "Unknown IMG_TYPE supplied: $stage";
     }
 
@@ -371 +562 @@
         $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
     } elsif ($req_type ne "byid") {
-        $command .= " -mjd_obs_begin $mjd_min" if $mjd_min;
-        $command .= " -mjd_obs_end $mjd_max" if $mjd_max;
+        $command .= " -mjd_min $mjd_min" if $mjd_min;
+        $command .= " -mjd_max $mjd_max" if $mjd_max;
     }
 
@@ -379 +570 @@
     if (!isnull($data_group)) {
         $command .= " -data_group $data_group";
-    } elsif ($req_type eq 'byskycell') {
+    } elsif (!$use_releasetool and $req_type eq 'byskycell') {
+        # XXX: Why am I using default data_groups only for byskycell requests?
         $command .= $default_data_groups;
+    }
+
+    if ($use_releasetool) {
+        # fwhm arguments are currently only supported when using releasetool for lookups
+        $command .= " -fwhm_min $fwhm_min" if $fwhm_min;
+        $command .= " -fwhm_max $fwhm_max" if $fwhm_max;
     }
 
@@ -394 +592 @@
         $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
     }
+
     if ($choose_components) {
         # the list of "images" is actually a list of "Runs"
         # match the coords in the rows to the components in the runs
         # returns an actual list of images
-        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        if ($skycenter and $req_type ne 'byskycell') {
+            $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        } else {
+            $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose);
+        }
     } else {
-        # put index for each of the rows into all of the the images
+        # put index for each of the rows into all of the the selected images
         setRowRefs($rowList, $images);
     }
@@ -464 +667 @@
             # dquery wants these set
             $out->{magicked} = 0;
-            $out->{data_state} = $out->{state};
+            if ($use_releasetool) {
+                if ($image->{skycal_path_base}) {
+                    $out->{astrom} = $image->{skycal_path_base} . ".cmf";
+                }
+                $out->{data_state} = $out->{state} = $out->{stack_state};
+            } else {
+                # XXX: Consider looking up skycal results even if we are
+                # not using releasetool
+                $out->{data_state} = $out->{state};
+            }
         }
         $out->{stage_id} = $stage_id;
@@ -472 +684 @@
         my $mask_base;
         if ($want_astrom) {
-            if (! defined $out->{astrom}) {
+            if ($selectedAstrom) {
+                $out->{astrom} = $selectedAstrom;
+                $out->{cam_path_base} = $selectedAstrom;
+                if ($selectedAstrom =~ /\.smf$/) {
+                    $out->{cam_path_base} =~ s/\.smf$//;
+                } elsif ($selectedAstrom =~ /\.cmf$/) {
+                    $out->{cam_path_base} =~ s/\.cmf$//;
+                } else {
+                    die ("ERROR: don't know how to extract cam_path_base from $selectedAstrom\n");
+                }
+            } elsif (! defined $out->{astrom}) {
                 if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) {
                     print STDERR "failed to find astrometry for $stage $stage_id\n";
@@ -507 +729 @@
     my $byid     = shift;
     my $option_mask  = shift;
-    my $img_type = shift;
+    my $stage = shift;
     my $verbose = shift;
 
@@ -558 +780 @@
         }
 
+        # XXX: The logic below is somewhat messed up. See XXX: ... below
+        #
         # The standard way to do a diff is warp - stack 
         # so we interpret the requested image in that way
@@ -572 +796 @@
         }
         my $stack_id;
+        if ($image->{diff_mode} == 2) {
+            # XXX: .. stack_id wasn't getting set
+            $stack_id = $stack2;
+            $image->{stack_id} = $stack_id;
+        }
         if ($stack1 and $stack2) {
             # we have a stack - stack diff (well it might be stack - warp....)
             # but at any rate we only handle image type diff and stack
-            if (($img_type ne "diff") and ($img_type ne "stack")) {
-                print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n";
+            if (($stage ne "diff") and ($stage ne "stack")) {
+                print STDERR "lookup_diff: cannot lookup IMG_TYPE $stage bydiff from a stack stack diff run\n";
                 setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST);
                 # all images will be the same so we can stop
@@ -627 +856 @@
         }
 
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             my @imageList = ($image);
 
@@ -658 +887 @@
 sub lookup_bycoord {
     my $ipprc      = shift;
-    my $rowList    = shift;
+    my $row        = shift;
     my $imagedb    = shift;
-    my $img_type   = shift;
+    my $stage      = shift;
     my $tess_id    = shift;
     my $component  = shift;
@@ -666 +895 @@
     my $ra         = shift;
     my $dec        = shift;
-    my $dateobs_begin  = shift;
+    my $dateobs_begin  = shift; # these are used for single frame lookups
     my $dateobs_end    = shift;
     my $filter     = shift;
     my $data_group = shift;
     my $option_mask = shift;
-    my $mjd_min = shift;
-    my $mjd_max = shift;
+    my $mjd_min = shift;        # these are used for stack lookups
+    my $mjd_max = shift;        # these are used for stack lookups
     my $verbose    = shift;
 
+    my $fwhm_min = getValOrZero($row->{FWHM_MIN});
+    my $fwhm_max = getValOrZero($row->{FWHM_MAX});
+    
+
+    my ($release_name, $survey, $default_tess_id) = get_release_info($row);
+    if (!$tess_id and $default_tess_id) {
+        $tess_id = $default_tess_id;
+    }
+
+    my $rowList = [$row];
+
     my $results = ();
-    if (($img_type eq "raw") or ($img_type eq "chip")) {
-
-        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
-            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
+    if (($stage eq "raw") or ($stage eq "chip")) {
+
+        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $stage,
+            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose,
+            $release_name, $survey, $fwhm_min, $fwhm_max);
 
         if (!$chips or scalar @$chips == 0) {
@@ -686 +927 @@
             foreach my $chip (@$chips) {
                 next if $component and ($chip->{component} ne $component);
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id},
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+                    $fwhm_min, $fwhm_max,
+                    $chip->{astrom}, undef, undef,
+                    $verbose);
 
                 next if !$these_results;
@@ -697 +941 @@
     } else {
         # this should have been checked elsewhere
-        die "unexpected image type $img_type" if ($img_type ne "warp") 
-                                              and ($img_type ne "stack") and ($img_type ne "diff");
+        die "unexpected image type $stage" if ($stage ne "warp") 
+                                              and ($stage ne "stack") and ($stage ne "diff");
 
         my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
@@ -705 +949 @@
             setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH);
         } else {
+            # XXX: We are not applying the fwhm cuts
             foreach my $skycell (@$skycells) {
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                    $fwhm_min, $fwhm_max,
+                    undef, $release_name, $survey,
+                    $verbose);
 
                 next if !$these_results;
@@ -724 +972 @@
 
 sub lookup_by_cam_id_and_coords {
-    my $ipprc      = shift;
-    my $imagedb   = shift;
-    my $img_type   = shift;
-    my $ra         = shift;
-    my $dec        = shift;
-    my $need_magic = shift;
+    my $ipprc       = shift;
+    my $imagedb     = shift;
+    my $stage       = shift;
+    my $ra          = shift;
+    my $dec         = shift;
+    my $need_magic  = shift;
     my $dateobs_begin  = shift;
     my $dateobs_end = shift;
@@ -735 +983 @@
     my $data_group = shift;
     my $verbose    = shift;
-
-    my $camruns;
-
-    {
-        my $search_radius = 1.6; # XXX: this should be camera specific
-
+    my $release_name = shift;
+    my $survey      = shift;
+    my $fwhm_min    = shift;
+    my $fwhm_max    = shift;
+
+    my $search_radius = 1.6; # XXX: this should be camera specific
+
+    my $command;
+    my $using_camtool = 0;
+    if ($release_name or $survey) {
+        $command = "$releasetool -dbname $imagedb -listrelexp";
+        if ($survey) {
+            $command .= " -surveyName $survey";
+        }
+        if ($release_name) {
+            $command .= " -release_name $release_name";
+        } else {
+            $command .= " -priority_order";
+        }
+        if ($fwhm_min != 0) {
+            $command .= " -fwhm_min $fwhm_min";
+        }
+        if ($fwhm_max != 0) {
+            $command .= " -fwhm_max $fwhm_max";
+        }
+    } else {
+        $using_camtool = 1;
         my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
-           $command .= " -ra $ra -decl $dec -radius $search_radius";
-           $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
-           $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
-           $command .= " -filter $filter" if $filter;
            # NOTE: we are applying the data_group to the camera run.
            # If we're looking for chip stage images there is no guarentee that 
@@ -757 +1022 @@
            }
            $command .= " -destreaked" if $need_magic;
-
-        # run the tool and parse the output
-        $camruns = runToolAndParse($command, $verbose);
-    }
-    if (!$camruns) {
+    }
+    $command .= " -ra $ra -decl $dec -radius $search_radius";
+    $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+    $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
+    $command .= " -filter $filter" if $filter;
+
+    # run the tool and parse the output
+    my $results = runToolAndParse($command, $verbose);
+    if (!$results) {
         return undef;
     }
     my $components;
     my $last_exp_id = 0;
-    foreach my $camRun (@$camruns) {
-        my $cam_id = $camRun->{cam_id};
-        my $exp_id = $camRun->{exp_id};
-
+    foreach my $result (@$results) {
+        my $cam_id = $result->{cam_id};
+        my $exp_id = $result->{exp_id};
+
+        # go on if we already have a result for this exposure
         next if $exp_id eq $last_exp_id;
 
-        next if $camRun->{quality};
-        next if $camRun->{fault};
-
-        updateCamRunCache($camRun);
+        next if $result->{quality};
+
+        if ($using_camtool) {
+            next if $result->{fault};
+        } else {
+            $result->{path_base} = $result->{cam_path_base};
+        }
+        updateCamRunCache($result);
 
         $last_exp_id = $exp_id;
 
         # XXX Use file rule
-        my $astrom = $camRun->{path_base} . ".smf";
+        my $astrom = $result->{path_base} . ".smf";
         my $astrom_resolved = $ipprc->file_resolve($astrom);
         next if !$astrom_resolved;
@@ -834 +1108 @@
         }
 
-        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
+        print "cam_id $cam_id exp_id $result->{exp_id} chip_id $result->{chip_id} $class_id\n";
 
         # build the hash to return
@@ -842 +1116 @@
         # problem
         my $comp = {
-            exp_id    => $camRun->{exp_id},
-            exp_name  => $camRun->{exp_name},
-            chip_id   => $camRun->{chip_id},
-            cam_id    => $camRun->{cam_id},
+            exp_id    => $result->{exp_id},
+            exp_name  => $result->{exp_name},
+            chip_id   => $result->{chip_id},
+            cam_id    => $result->{cam_id},
             class_id  => $class_id,
-            component => $class_id
+            component => $class_id,
+            astrom    => $astrom
         };
-        if ($img_type eq "chip") {
-            $comp->{id} = $camRun->{chip_id};
-            $comp->{state} = $camRun->{chip_state};
-            $comp->{magicked} = $camRun->{chip_magicked};
-        } else {
-            $comp->{id} = $camRun->{exp_id};
+        if ($stage eq "chip") {
+            $comp->{id} = $result->{chip_id};
+            $comp->{state} = $result->{chip_state};
+            $comp->{magicked} = 0;
+        } else {
+            $comp->{id} = $result->{exp_id};
             $comp->{state} = 'full';
-            $comp->{magicked} = $camRun->{raw_magicked};
+            $comp->{magicked} = $0;
         }
         push @$components, $comp;
@@ -1072 +1347 @@
     my $val = shift;
     return (!defined($val) or ($val == 0));
+}
+
+sub getValOrZero {
+    my $val = shift;
+    return iszero($val) ? 0.0 : $val;
 }
 
@@ -1283 +1563 @@
 }
 
+sub selectComponentsByName {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $component  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+
+    $component = "" if $component and lc($component) eq 'all';
+
+    # all runs match all rows so they can share a common row_index
+    # XXX: I'm not sure we ever actually get here with more than one
+    # row
+    my $numRows = scalar @$rowList;
+    my $row_index = [];
+    for (my $i=0; $i < $numRows; $i++) {
+        push @$row_index, $i;
+    }
+    
+    # XXX: Why did I make this restriction? I added byskycell
+    if (($req_type eq "byid") or ($req_type eq "byexp") or ($req_type eq 'byskycell')) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            # now find the images for this run
+            #foreach my $c (keys %components) {
+            my $command;
+            my $stage_id;
+            if ($stage eq 'raw') {
+                $stage_id = $run->{exp_id};
+                $command = "$regtool -processedimfile -exp_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'chip') {
+                $stage_id = $run->{chip_id};
+                $command = "$chiptool -processedimfile -chip_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'warp') {
+                $stage_id = $run->{warp_id};
+                $command = "$warptool -warped -warp_id $stage_id";
+                $command .= " -skycell_id $component" if $component;
+            }
+            $command .= " -dbname $imagedb";
+            my $images = runToolAndParse($command, $verbose);
+            if (!defined $images) {
+                print "No  $component found for $stage ${stage}_id $stage_id\n";
+                next;
+            }
+            foreach my $image (@$images) {
+                $image->{row_index} = $row_index;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
 sub filterRuns {
     my $stage      = shift;
@@ -1409 +1753 @@
     $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" 
                                                                 and $missing_tools = 1);
+    $releasetool = can_run('releasetool') or (warn "Can't find releasetool" 
+                                                                and $missing_tools = 1);
     $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
     $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
@@ -1464 +1810 @@
 }
 
+sub get_release_info {
+    my $row = shift;
+
+    my $survey;
+    my $release_name = $row->{IPP_RELEASE};
+    if (isnull($release_name)) {
+        $release_name = "";
+        # no release check for survey
+        $survey = isnull($row->{SURVEY_NAME}) ? "" : $row->{SURVEY_NAME};
+    } else {
+        # if release is given leave survey empty
+        $survey = "";
+    }
+
+    my $tess_id;
+    if ($release_name) {
+        # hack to optimize bycoord and byskycell lookups for well known releases
+        # XXX: todo query database and return a list for the survey or release
+        if ($release_name eq '3PI.PV1') {
+            $tess_id = 'RINGS.V3';
+        }
+    }
+
+    return ($release_name, $survey, $tess_id);
+}
+
 sub my_die
 {

tags/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -44 +44 @@
                     $PSTAMP_DUP_REQUEST
                     $PSTAMP_INVALID_REQUEST
+                    $PSTAMP_UNKNOWN_PROJECT
                     $PSTAMP_UNKNOWN_PRODUCT
                     $PSTAMP_NO_IMAGE_MATCH
@@ -80 +81 @@
 our $PSTAMP_USE_IMFILE_ID      = 16384;
 our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
+
+# these bits will be repurposed
 our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
 our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
@@ -93 +96 @@
 our $PSTAMP_DUP_REQUEST      = 20;
 our $PSTAMP_INVALID_REQUEST  = 21;
-our $PSTAMP_UNKNOWN_PRODUCT  = 22;
+our $PSTAMP_UNKNOWN_PROJECT  = 22;
+our $PSTAMP_UNKNOWN_PRODUCT  = 22;  #this error code was mis-named it is left for compatabiliyt
 our $PSTAMP_NO_IMAGE_MATCH   = 23;
 our $PSTAMP_NOT_DESTREAKED   = 24;
 our $PSTAMP_NOT_AVAILABLE    = 25;
-our $PSTAMP_GONE             = 26;
+our $PSTAMP_GONE             = 26;  # this value is used in ippTools
 our $PSTAMP_NO_JOBS_QUEUED   = 27;
 our $PSTAMP_NO_OVERLAP       = 28;
@@ -131 +135 @@
 PSTAMP_DUP_REQUEST
 PSTAMP_INVALID_REQUEST
-PSTAMP_UNKNOWN_PRODUCT
+PSTAMP_UNKNOWN_PROJECT
 PSTAMP_NO_IMAGE_MATCH
 PSTAMP_NOT_DESTREAKED
@@ -139 +143 @@
 PSTAMP_NO_OVERLAP
 PSTAMP_NOT_AUTHORIZED
+PSTAMP_NOT_AUTHORIZED
 );
 
@@ -167 +172 @@
     my $fields_output;
     {
-        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL";
+        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -187 +192 @@
     if ($extver > 1) {
         $header{ACTION} = $action;
-        $header{USER} = $username;
         $header{EMAIL} = $email;
     } else {
         $header{ACTION} = $action = "PROCESS";
-        $header{USERNAME} = 'null';
         $header{EMAIL} = 'null';
     }
 
-    if ($action eq "LIST") {
-        return (\%header, undef);
-    } elsif ($action ne "PROCESS") {
-        die "unexpected request ACTION found: $action in $request_file_name";
+    if ($action ne "PROCESS" and $action ne 'PREVIEW') {
+        die "\nunexpected request ACTION found: $action in $request_file_name";
     }
 

tags/ipp-20130307/dbconfig/changes.txt

@@ -2388 +2388 @@
 CREATE TABLE lapGroup  (
     seq_id  BIGINT,
+    tess_id VARCHAR(64),
     projection_cell VARCHAR(64),
     state   VARCHAR(16),
+    label   VARCHAR(654),
+    registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
     fault   SMALLINT,
     PRIMARY KEY(seq_id, projection_cell),
@@ -2397 +2400 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
-UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP();
-
--- new Postage Stamp Request columns (NOTE these changes to not apply to gpc1
--- because the pstamp tables have been dropped from that database
+-- new Postage Stamp Request columns
+-- Note: These do not aply to the gpc1 database because the pstamp tables were
+-- deleted from that database.
 ALTER TABLE pstampRequest ADD COLUMN username VARCHAR(255) AFTER outdir;
 ALTER TABLE pstampRequest ADD COLUMN proj_id BIGINT AFTER username;
-ALTER TABLE pstampRequest ADD COLUMN registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER proj_id;
-
--- default to the gpc1 project (or is it ps1?)
+ALTER TABLE pstampRequest ADD COLUMN registered DATETIME AFTER proj_id;
+
+-- default to the gpc1 project
 UPDATE pstampRequest SET proj_id = 1;
 
-
-
+-- unique keys for the release tables
+
+alter table relStack  add unique key (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs)
+alter table relExp add UNIQUE KEY(rel_id, exp_id);
+alter table ippRelease add unique key (surveyID, release_name);
+
+UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP();

tags/ipp-20130307/dbconfig/lap.md

@@ -33 +33 @@
 lapGroup METADATA
     seq_id          S64 0
+    tess_id         STR 64
     projection_cell STR 64
     state           STR 64
+    label           STR 64
+    registered      TAI NULL 
     fault           S16 0
 end

tags/ipp-20130307/ippScripts/Build.PL

@@ -130 +130 @@
         scripts/regenerate_background.pl
         scripts/relgroup_exp_list.pl
+        scripts/queuestaticsky.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

tags/ipp-20130307/ippScripts/scripts/camera_exp.pl

@@ -363 +363 @@
     }
     # Construct FPA continuity corrected background images
-    {
+    if ($camera =~ /ISP/) {
+        print "Skipping FPA continuity corrected background images for ISP\n";
+    } else {
         my $command;
         $command = "$ppImage";

tags/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl

@@ -156 +156 @@
             $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path";
         } else {
-            $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path";
+            $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path";
         }
         print "\n$merge_command\n";

tags/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl

@@ -16 +16 @@
 my $dtime_relphot;
 my $dtime_script;
+my $dtime_delstar;
 
 use vars qw( $VERSION );
@@ -33 +34 @@
 my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1);
 my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1);
+my $delstar = can_run('delstar') or (warn "Can't find delstar" and $missing_tools = 1);
 my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
 my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1);
 my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1);
+
 my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1);
 
@@ -80 +83 @@
 unless ($no_op) {
     
-        #this is chopped into several parts: addstar, relphot
+        #this is chopped into several parts: delstar,addstar, relphot, relastro, dvoverify
+        #delstar - first step: are there duplicates, if so remove them
+        {
+            my $command  = "$delstar -update -dup-images ";
+            #$command .= " -D CAMERA $camera";
+            $command .= " -D CATDIR $minidvodb";
+            my $mjd_delstar_start = DateTime->now->mjd;   # MJD of starting script
+            print "\n$command\n";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform delstar: $error_code", $minidvodb_id, $error_code);
+            }
+            print $full_buf;
+            $dtime_delstar = 86400.0*(DateTime->now->mjd - $mjd_delstar_start);  
+            # MJD of starting script
+            print "delstar time $dtime_delstar\n";
+        }
+
+
+
         #addstar
         {
@@ -94 +118 @@
                 &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code);
             }
-            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);  $dtime_resort = $dtime_addstar;
-            # MJD of starting script
+            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);  
+            # MJD of starting script
             $dtime_resort = $dtime_addstar;
             print "addstar -resort time $dtime_addstar\n";
@@ -105 +129 @@
             # relphot only takes lower case gpc1
             my $relphot_camera = lc($camera);
-            my $command  = "$relphot -averages -update";
-            $command .= " -D CAMERA $relphot_camera";
-            $command .= " -D CATDIR $minidvodb";
+            #my $command  = "$relphot -averages -update";
+            #$command .= " -D CAMERA $relphot_camera";
+            #$command .= " -D CATDIR $minidvodb";
+            #this is a friday hack
+            my $command = "echo skipping relphot";
             print "$command\n";
             my $mjd_relphot_start = DateTime->now->mjd;   # MJD of starting script

tags/ipp-20130307/ippScripts/scripts/relgroup_exp_list.pl

@@ -16 +16 @@
 
 
-my ($group_id, $group_type, $lap_id, $release_name, $rebuild, $dbname, $no_update, $verbose);
+my ($group_id, $group_type, $lap_id, $group_name, $release_name, $rebuild, $dbname, $no_update, $verbose);
 
 GetOptions(
     'group_id=s'    =>      \$group_id,
+    'group_type=s'  =>      \$group_type,
     'lap_id=s'      =>      \$lap_id,
-    'group_type=s'  =>      \$group_type,
+    'group_name=s'  =>      \$group_name,
     'release_name=s' =>     \$release_name,
     'rebuild'       =>      \$rebuild,      # rebuild directory if it exists
@@ -36 +37 @@
            -exitval => 3) 
         unless $lap_id;
+    $group_name = "lap_$lap_id";
+} else {
+    pod2usage( -msg => "--group_name is required with --group_type $group_type",
+           -exitval => 3) 
+        unless $group_name;
 }
 
@@ -48 +54 @@
 
 # XXX: get this from site.config
-my $outputBase = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame/$release_name";
-
-my $outdir;
-if ($group_type eq 'lap') {
-    $outdir = "$outputBase/lap_$lap_id";
-    if (-e $outdir) {
-        if ($rebuild) {
-            my $rc = system "rm -rf $outdir";
-            if ($rc) {
-                my $status = $rc >> 8;
-                my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-            }
-        } else {
-            my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
-} else {
-    my_die("Not ready to do group_type $group_type\n", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+my $exposureListRoot = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame";
+
+my $outputBase = "$exposureListRoot/$release_name";
+
+if (!-e $outputBase) {
+    mkdir $outputBase or &my_die("Failed to create $outputBase", $group_id, $PS_EXIT_SYS_ERROR);
+}
+
+my $outdir = "$outputBase/$group_name";
+if (-e $outdir) {
+    if ($rebuild) {
+        my $rc = system "rm -rf $outdir";
+        if ($rc) {
+            my $status = $rc >> 8;
+            my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
 }
 
 mkdir $outdir or my_die ("failed to create $outdir", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-
-if (0) {
-    # XXX: old code from prototype for nightly lists
-        my $outroot;
-        my $oh;
-        my ($year, $month, $day);
-        if ($outroot) {
-        die "usage: $0 <year> <month> <day>\n" unless $year and $month and $day;
-            my $monthdir = sprintf "$outroot/%4d/%02d", $year, $month;
-            if (! -e $monthdir) {
-                die "can't find directory for $year-$month\n";
-            }
-            my $daydir = sprintf "$monthdir/%02d", $day;
-            if (! -e $daydir) {
-                mkdir $daydir or die "failed to create $daydir\n";
-            }
-            my $outfile = sprintf "$daydir/index.html", $day;
-            open $oh, ">$outfile" or die "failed to open $outfile";
-        } else {
-            $oh = *STDOUT;
-        }
-
-        my $ticks = timegm(0, 0, 0, $day, $month - 1, $year - 1900);
-
-        my $mjd = 40587 + ($ticks / 86400.);
-        my $date = sprintf "%04d-%02d-%02d", $year, $month, $day;
-}
 
 my $oh;
@@ -105 +85 @@
 open $oh2, ">$outfile2" or my_die("failed to open $outfile2", $group_id, $PS_EXIT_UNKNOWN_ERROR);
 
-my $title = "PS1 Exposure List for $release_name LAP run $lap_id";
+my $title = "PS1 Exposure List for";
+if ($group_type eq 'lap') {
+    $title .= " $release_name LAP run $lap_id";
+} else {
+    $title .= " $group_name";
+}
 
 my $head = "
@@ -125 +110 @@
 
 
-# find all exposures on given day that have a chip run
+# find all exposures in this relGroup
 my $query = "
 SELECT 
@@ -149 +134 @@
 $stmt->execute() or my_die("failed to execute query: $stmt->errstr\n", $group_id, $PS_EXIT_UNKNOWN_ERROR);
 
-printf STDERR "%4d exposures in group for LAP run $lap_id\n", $stmt->rows;
+printf STDERR "%4d exposures in group for $group_name\n", $stmt->rows;
 
 # $stmt->dump_results(1000);

tags/ipp-20130307/ippScripts/scripts/whichimage

@@ -94 +94 @@
 my $cmd = "dvoImagesAtCoords -chipcoords $coord_file";
 $cmd .= " -listchipcoords" if $listchipcoords;
+
+$tess_id = uc($tess_id) if $tess_id;
 
 my $status = 0;
@@ -125 +127 @@
             # next if $tess_name eq 'FIXNS';
             # next if $tess_name eq 'ALLSKY';
+            next if $tess_name eq 'RINGS.V0';
             push @tess_ids_to_check, $tess_name;
         }

tags/ipp-20130307/ippTasks/pstamp.pro

@@ -965 +965 @@
 
         # XXX: have the script set this up this
-        $MYLOGFILE="/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID"
+        $MYLOGFILE=/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID
         stdout $MYLOGFILE
         stderr $MYLOGFILE

tags/ipp-20130307/ippTasks/survey.pro

@@ -18 +18 @@
  book create SURVEY_STATICSKYSINGLE 
  book create SURVEY_SKYCAL
+ book create SURVEY_LAPGROUP
+ book create SURVEY_RELEXP
  $haveSurveyBooks = TRUE
 end
@@ -33 +35 @@
 $SURVEY_PUBLISH_DB = 0
 $SURVEY_SKYCAL_DB = 0
+$SURVEY_LAPGROUP_DB = 0
+$SURVEY_RELEXP_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
 $SURVEY_EXEC = 120
@@ -78 +82 @@
     active true
   end
+  task survey.lapgroup
+    active true
+  end
+  task survey.relexp
+    active true
+  end
 end
 
@@ -118 +128 @@
   end
   task survey.skycal
+    active false
+  end
+  task survey.lapgroup
+    active false
+  end
+  task survey.relexp
     active false
   end
@@ -494 +510 @@
 macro survey.show.skycal
   if ($0 != 1)
-    echo "USAGE: survey.show.skyacl"
+    echo "USAGE: survey.show.skycal"
     break
   end
   book listbook SURVEY_SKYCAL
+end
+
+macro survey.add.lapgroup
+  if ($0 != 3)
+    echo "USAGE: survey.add.lapgroup (label) (seq_id)"
+    break
+  end
+  book newpage SURVEY_LAPGROUP $1
+  book setword SURVEY_LAPGROUP $1 LABEL $1
+  book setword SURVEY_LAPGROUP $1 SEQ_ID $2
+  book setword SURVEY_LAPGROUP $1 STATE PENDING
+end
+
+macro survey.del.lapgroup
+  if ($0 != 2)
+    echo "USAGE: survey.del.lapgroup (label)"
+    break
+  end
+  book delpage SURVEY_LAPGROUP $1
+end
+
+macro survey.show.lapgroup
+  if ($0 != 1)
+    echo "USAGE: survey.show.lapgroup"
+    break
+  end
+  book listbook SURVEY_LAPGROUP
+end
+
+macro survey.add.relexp
+  if ($0 != 3)
+    echo "USAGE: survey.add.relexp (label) (releasename)"
+    break
+  end
+  book newpage SURVEY_RELEXP $1
+  book setword SURVEY_RELEXP $1 LABEL $1
+  book setword SURVEY_RELEXP $1 RELEASE_NAME $2
+  book setword SURVEY_RELEXP $1 STATE PENDING
+end
+
+macro survey.del.relexp
+  if ($0 != 2)
+    echo "USAGE: survey.del.relexp (label)"
+    break
+  end
+  book delpage SURVEY_RELEXP $1
+end
+
+macro survey.show.relexp
+  if ($0 != 1)
+    echo "USAGE: survey.show.relexp"
+    break
+  end
+  book listbook SURVEY_RELEXP
 end
 
@@ -1561 +1631 @@
   end
 end
+
+task survey.lapgroup
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.lapgroup.log
+  stderr $LOGDIR/survey.lapgroup.log
+
+  task.exec
+    book npages SURVEY_LAPGROUP -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_LAPGROUP $i -var label
+        book setword SURVEY_LAPGROUP $label STATE NEW
+      end
+      book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_LAPGROUP_DB ++
+      if ($SURVEY_LAPGROUP_DB >= $DB:n) set SURVEY_LAPGROUP_DB = 0
+    end
+
+    book setword SURVEY_LAPGROUP $label STATE DONE
+    book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID
+
+    # For now the list of filters is hardcoded here
+    $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_LAPGROUP_DB
+      option $DB:$SURVEY_LAPGROUP_DB
+    end
+    
+    # echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task survey.relexp
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.relexp.log
+  stderr $LOGDIR/survey.relexp.log
+
+  task.exec
+    book npages SURVEY_RELEXP -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_RELEXP 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_RELEXP $i -var label
+        book setword SURVEY_RELEXP $label STATE NEW
+      end
+      book getpage SURVEY_RELEXP 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_RELEXP_DB ++
+      if ($SURVEY_RELEXP_DB >= $DB:n) set SURVEY_RELEXP_DB = 0
+    end
+
+    book setword SURVEY_RELEXP $label STATE DONE
+    book getword SURVEY_RELEXP $label RELEASE_NAME -var RELEASE_NAME
+
+    $run = releasetool -definerelexp -label $label -release_name $RELEASE_NAME -set_state processed
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_RELEXP_DB
+      option $DB:$SURVEY_RELEXP_DB
+    end
+    
+    # echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

tags/ipp-20130307/ippTools/share/Makefile.am

@@ -457 +457 @@
         laptool_WSdiff_check.sql \
         laptool_stacks.sql \
+        laptool_pendinggroup.sql \
+        laptool_revertgroup.sql \
+        laptool_filtersforgroup.sql \
         vptool_find_rawexp.sql \
         vptool_pendingimfile.sql \
@@ -470 +473 @@
         releasetool_definerelstack.sql \
         releasetool_definerelstack_with_skycal.sql \
+        releasetool_listrelstack.sql \
         releasetool_definerelgroup_select_lap.sql \
         releasetool_definerelgroup_select_data_group.sql \

tags/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql

@@ -1 +1 @@
 SELECT
     addProcessedExp.*,
-    addRun.workdir
+    addRun.label,
+    addRun.workdir,
+    camRun.cam_id,
+    camRun.label as cam_label,
+    camRun.data_group as cam_data_group,
+    rawExp.exp_id,
+    exp_name
 FROM addProcessedExp
 JOIN addRun

tags/ipp-20130307/ippTools/share/pstamptool_completedreq.sql

@@ -8 +8 @@
         SELECT count(*) FROM pstampJob 
         WHERE pstampJob.req_id = pstampRequest.req_id
-                AND pstampJob.state != 'stop'
+                AND pstampJob.state != 'stop' 
+                AND pstampJob.state != 'cancel'
         ) = 0

tags/ipp-20130307/ippTools/share/pstamptool_getdependent.sql

@@ -1 +1 @@
 SELECT DISTINCT pstampDependent.*
 FROM pstampDependent
-WHERE pstampDependent.state = 'new'
+WHERE (pstampDependent.state = 'new' OR pstampDependent.state = 'hold')

tags/ipp-20130307/ippTools/share/pxadmin_create_tables.sql

@@ -1 @@
-CREATE TABLE dbversion (
+eREATE TABLE dbversion (
     schema_version VARCHAR(64),
     updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -2172 +2172 @@
     priority    INT,
     PRIMARY KEY(rel_id),
+    UNIQUE KEY(surveyID, release_name),
     KEY(release_name),
     KEY(release_state),
@@ -2196 +2197 @@
     PRIMARY KEY (relexp_id),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
+    UNIQUE KEY(rel_id, exp_id),
     KEY (state),
     KEY (fault),
@@ -2224 +2226 @@
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
+    UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
     KEY (stack_type),
     KEY (mjd_obs),
@@ -2258 +2260 @@
 CREATE TABLE lapGroup  (
     seq_id  BIGINT,
+    tess_id VARCHAR(64),
     projection_cell VARCHAR(64),
     state   VARCHAR(16),
+    label   VARCHAR(64),
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered
     fault   SMALLINT,
-    PRIMARY KEY(seq_id, projection_cell),
+    PRIMARY KEY(seq_id, tess_id, projection_cell),
     KEY(state),
     KEY(fault),

tags/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql

@@ -1 +1 @@
 SELECT relExp.relexp_id
-FROM relGroup JOIN lapRun USING(lap_id)
+FROM relGroup 
+    JOIN lapRun USING(lap_id)
+    JOIN lapExp USING(lap_id)
     JOIN relExp using(rel_id, exp_id)
 WHERE relExp.group_id = 0

tags/ipp-20130307/ippTools/share/releasetool_definerelstack.sql

@@ -6 +6 @@
     stackRun.tess_id,
     stackRun.filter,
-    0 AS mjd_obs,
+    stackSumSkyfile.mjd_obs,
     0 AS zpt_obs,
     0 AS zpt_stdev
@@ -12 +12 @@
 FROM ippRelease
 JOIN stackRun
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
-
+JOIN stackSumSkyfile using(stack_id)
+LEFT JOIN relStack AS previousRelStack 
+    ON previousRelStack.rel_id = ippRelease.rel_id
+    AND previousRelStack.tess_id = stackRun.tess_id 
+    AND previousRelStack.skycell_id = stackRun.skycell_id
+    AND previousRelStack.filter = stackRun.filter
+    -- JOIN hook %s
 WHERE previousRelStack.relstack_id IS NULL
+    AND stackRun.state ='full'
+    AND stackSumSkyfile.quality = 0
+    AND stackSumSkyfile.fault = 0

tags/ipp-20130307/ippTools/share/releasetool_definerelstack_with_skycal.sql

@@ -13 +13 @@
 JOIN stackRun
 JOIN stackSumSkyfile USING(stack_id)
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
 JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id
 JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id
+-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
+LEFT JOIN relStack AS previousRelStack 
+    ON ippRelease.rel_id = previousRelStack.rel_id 
+    AND stackRun.tess_id = previousRelStack.tess_id 
+    AND stackRun.skycell_id = previousRelStack.skycell_id 
+    AND stackRun.filter = previousRelStack.filter
+-- JOIN Hook %s
 
 WHERE previousRelStack.relstack_id IS NULL
+    AND skycalRun.state = 'full'

tags/ipp-20130307/ippTools/share/releasetool_listrelexp.sql

@@ -25 +25 @@
     warpRun.warp_id,
     warpRun.state as warp_state,
+    warpRun.tess_id,
     rawExp.filter,
     rawExp.dateobs,

tags/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql

@@ -1 @@
-SELECT relGroup.*
+SELECT 
+    relGroup.*,
+    ippRelease.release_name
 FROM relGroup
     JOIN ippRelease USING(rel_id)
 WHERE relGroup.state = 'new'
+    AND relGroup.fault = 0

tags/ipp-20130307/ippTools/src/laptool.c

@@ -34 +34 @@
 static bool inactiveexpMode(pxConfig *config);
 
+// Groups
+static bool definegroupMode(pxConfig *config);
+static bool pendinggroupMode(pxConfig *config);
+static bool filtersforgroupMode(pxConfig *config);
+static bool updategroupMode(pxConfig *config);
+static bool revertgroupMode(pxConfig *config);
+static bool listgroupMode(pxConfig *config);
+
 # define MODECASE(caseName, func) \
   case caseName: \
@@ -67 +75 @@
     
     MODECASE(LAPTOOL_MODE_INACTIVEEXP,   inactiveexpMode);
+
+    MODECASE(LAPTOOL_MODE_DEFINEGROUP,   definegroupMode);
+    MODECASE(LAPTOOL_MODE_PENDINGGROUP,  pendinggroupMode);
+    MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode);
+    MODECASE(LAPTOOL_MODE_UPDATEGROUP,   updategroupMode);
+    MODECASE(LAPTOOL_MODE_REVERTGROUP,   revertgroupMode);
+    MODECASE(LAPTOOL_MODE_LISTGROUP,     listgroupMode);
+
   default:
     psAbort("invalid option (this should not happen)");
@@ -900 +916 @@
   return(true);
 }
-
+// ---------------------------
+// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters
+
+static bool definegroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_S64(seq_id,          config->args, "-seq_id",    true, false);
+  PXOPT_LOOKUP_STR(label,           config->args, "-set_label", true, false);
+
+  PXOPT_LOOKUP_BOOL(pretend,        config->args, "-pretend",    false);
+  PXOPT_LOOKUP_BOOL(simple,         config->args, "-simple",    false);
+  PXOPT_LOOKUP_U64(limit,           config->args, "-limit",     false, false);
+
+  // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs)
+  psMetadataItem *item = psMetadataLookup(config->args, "-filter");
+  psAssert (item, "-filter argument not found in config->args");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter");
+  psAssert (item->data.list->n, "%s should at least have a place-holder", "filter");
+  psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data;
+  psAssert (entry, "%s should at least have a place-holder", "filter");
+  // end of checking
+
+  // Now if the ony entry is the place-holder then the user supplied no -filter arguments
+  // which are required
+  if (!entry->data.str) {
+    psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required");
+    return false;
+  }
+
+  int nFilters = item->data.list->n;
+
+  psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n");
+
+  // We construct a query joining completed lap runs with the same lapSequence.seq_id,
+  // tess_id, and projection_cell 
+
+  // select the lap_ids for the various filters. These are only used for debugging.
+  psStringAppend(&query, "lap_id_%d", 0);
+  for (int i = 1; i < nFilters; i++) {
+    psStringAppend(&query, ", lap_id_%d", i);
+  }
+
+
+  // sub query for each supplied filter
+  char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n"
+    "FROM lapRun\n"
+    "WHERE filter LIKE '%s'\n"
+    "   AND lapRun.seq_id = %"PRId64"\n"
+    "   AND (lapRun.state = 'done' or lapRun.state = 'full')\n"
+    " ) as lap_%d\n";
+
+  // loop over supplied filters and flesh out the query using the format above
+  psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true);
+  psMetadataItem *filterItem = NULL;
+  int i = -1;
+  while ((filterItem = psListGetAndIncrement(iter))) {
+    ++i;
+    if (i == 0) {
+      psStringAppend(&query, "\nFROM\n");
+    } else {
+      psStringAppend(&query, "\nJOIN\n");
+    }
+
+    psString filter = filterItem->data.str;
+    psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i);
+
+    if (i != 0) {
+      psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n");
+    }
+  }
+  // now join to lapGroup
+  psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n");
+
+  // we only want projection cells which do not already of an entry
+  psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n");
+  psStringAppend(&query, "\nORDER by projection_cell\n");
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, "\n %s", limitString);
+    psFree(limitString);
+  }
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+  }
+
+  if (!psArrayLength(output)) {
+      psTrace("laptool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+  }
+
+  if (pretend) {
+    if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+    return true;
+  }
+
+  if (!psDBTransaction(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psTime *now = psTimeGetNow(PS_TIME_TAI);
+
+  psString last_projection_cell = NULL;
+  for (long i = 0; i < psArrayLength(output); i++) {
+    psMetadata *row = output->data[i];
+
+    psString tess_id = psMetadataLookupStr(NULL, row, "tess_id");
+    psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell");
+    if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) {
+        // duplicate lap runs for a filter will generate multiple rows
+        // Since we care about projection_cells not lapRuns per se this is not problem.
+        // Skip any duplicates.
+        continue;
+    }
+    last_projection_cell = projection_cell;
+
+    lapGroupRow *group = lapGroupRowAlloc(
+                                  seq_id,
+                                  tess_id,
+                                  projection_cell,
+                                  "new",  // state
+                                  label,
+                                  now,    // registered
+                                  0       // fault
+                                  );
+    if (!group) {
+      psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object");
+      psFree(output);
+      psFree(now);
+      return(false);
+    }
+
+
+    if (!lapGroupInsertObject(config->dbh, group)) {
+      if (!psDBRollback(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+      }
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(output);
+      return(true);
+    }
+  }
+
+  // point of no return
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psFree(now);
+  psFree(output);
+
+  return(true);  
+}
+
+static bool pendinggroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "==");
+  
+  psString query = pxDataGet("laptool_pendinggroup.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  }
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, "\n %s", limitString);
+    psFree(limitString);
+  }
+  
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}
+static bool filtersforgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+  
+  psString query = pxDataGet("laptool_filtersforgroup.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " WHERE %s", whereClause);
+  psFree(whereClause);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}
+
+static bool updategroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  // check that all of the keys that define a lapGroup are supplied
+  PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false);
+  PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
+
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+
+  psMetadata *values = psMetadataAlloc();
+  PXOPT_COPY_STR(config->args, values, "-set_state", "state", "==");
+  PXOPT_COPY_STR(config->args, values, "-set_label", "label", "==");
+  PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "==");
+
+  if (!psListLength(values->list)) {
+    psFree(values);
+    psFree(where);
+    psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column");
+    return false;
+  }
+
+  long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values);
+  if (rows < 1) {
+    psFree(values);
+    psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell);
+    return false;
+  }
+  psFree(values);
+
+  return(true);
+}
+
+static bool revertgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psMetadata *where = psMetadataAlloc();
+  pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+
+  if (!psListLength(where->list)) {
+    psFree(where);
+    psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required");
+    return false;
+  }
+
+  psString query = pxDataGet("laptool_revertgroup.sql");
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " AND %s", whereClause);
+  psFree(whereClause);
+  psFree(where);
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psU64 affected = psDBAffectedRows(config->dbh);
+  psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected);
+
+  return true;
+}
+
+static bool listgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psError(PXTOOLS_ERR_SYS, true, "not yet implemented");
+  return false;
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-seq_name",   "name",   "LIKE");
+
+  psString query = pxDataGet("laptool_listsequence.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+  }
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}

tags/ipp-20130307/ippTools/src/laptool.h

@@ -20 +20 @@
   LAPTOOL_MODE_UPDATEEXP,
   LAPTOOL_MODE_DIFFCHECK,
-  LAPTOOL_MODE_INACTIVEEXP
+  LAPTOOL_MODE_INACTIVEEXP,
+  LAPTOOL_MODE_DEFINEGROUP,
+  LAPTOOL_MODE_PENDINGGROUP,
+  LAPTOOL_MODE_FILTERSFORGROUP,
+  LAPTOOL_MODE_UPDATEGROUP,
+  LAPTOOL_MODE_REVERTGROUP,
+  LAPTOOL_MODE_LISTGROUP,
 } laptoolMode;
 

tags/ipp-20130307/ippTools/src/laptoolConfig.c

@@ -97 +97 @@
   ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id",           "set quick stack sass_id", 0);
   ADD_OPT(S64, updaterunArgs, "-set_final_sass_id",           "set final stack sass_id", 0);
-  
   
   // -pendingexp
@@ -150 +149 @@
   ADD_OPT(U64, inactiveexpArgs, "-limit",                     "limit result set to N items", 0);
   
+  // -definegroup
+  psMetadata *definegroupArgs = psMetadataAlloc();
+  ADD_OPT(S64, definegroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
+  ADD_OPT(Str, definegroupArgs, "-set_label",                 "set label (required)", 0);
+
+  psMetadataAddStr(definegroupArgs,  PS_LIST_TAIL, 
+                                "-filter", PS_META_DUPLICATE_OK, 
+                                                              "search by filter (LIKE comparison, multiple OK)", NULL);
+  ADD_OPT(Bool,definegroupArgs, "-pretend",                   "do not actuallym modify the database", false);
+  ADD_OPT(Bool,definegroupArgs, "-simple",                    "use the simple output format", false);
+  ADD_OPT(U64, definegroupArgs, "-limit",                     "limit result set to N items", 0);
+
+  // -pendinggroup
+  psMetadata *pendinggroupArgs = psMetadataAlloc();
+  psMetadataAddStr(pendinggroupArgs,  PS_LIST_TAIL, 
+                                "-label", PS_META_DUPLICATE_OK, 
+                                                              "search by label (multiple OK)", NULL);
+  ADD_OPT(S64, pendinggroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
+
+  ADD_OPT(Bool,pendinggroupArgs, "-simple",                    "use the simple output format", false);
+  ADD_OPT(U64, pendinggroupArgs, "-limit",                     "limit result set to N items", 0);
+
+  // -updategroup
+  psMetadata *updategroupArgs = psMetadataAlloc();
+  ADD_OPT(S64, updategroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
+  ADD_OPT(Str, updategroupArgs, "-tess_id",                   "search by tess_id (required)", 0);
+  ADD_OPT(Str, updategroupArgs, "-projection_cell",           "search by projection_cell (required)", 0);
+  ADD_OPT(Str, updategroupArgs, "-set_state",                 "set state", 0);
+  ADD_OPT(S16, updategroupArgs, "-set_fault",                 "set fault code", INT16_MAX);
+  ADD_OPT(Str, updategroupArgs, "-set_label",                 "set label", 0);
+
+  // -revertgroup
+  psMetadata *revertgroupArgs = psMetadataAlloc();
+  psMetadataAddStr(revertgroupArgs,  PS_LIST_TAIL, 
+                                "-label", PS_META_DUPLICATE_OK, 
+                                                              "search by label (multiple OK)", NULL);
+  ADD_OPT(S64, revertgroupArgs, "-seq_id",                    "search by lap sequence ID", 0);
+  ADD_OPT(Str, revertgroupArgs, "-tess_id",                   "search by tess_id", 0);
+  ADD_OPT(Str, revertgroupArgs, "-projection_cell",           "search by projection_cell", 0);
+  ADD_OPT(S16, revertgroupArgs, "-fault",                     "fault code", INT16_MAX);
+
+  // -filtersforgroup
+  psMetadata *filtersforgroupArgs = psMetadataAlloc();
+  ADD_OPT(S64, filtersforgroupArgs, "-seq_id",                    "search by lap sequence ID (required)", 0);
+  ADD_OPT(Str, filtersforgroupArgs, "-tess_id",                   "search by tess_id (required)", 0);
+  ADD_OPT(Str, filtersforgroupArgs, "-projection_cell",           "search by projection_cell (required)", 0);
+  ADD_OPT(Bool,filtersforgroupArgs, "-simple",                    "use the simple output format", false);
+
   
   psMetadata *argSets = psMetadataAlloc();
@@ -166 +213 @@
   PXOPT_ADD_MODE("-diffcheck",               "", LAPTOOL_MODE_DIFFCHECK,        diffcheckArgs);
   PXOPT_ADD_MODE("-inactiveexp",             "", LAPTOOL_MODE_INACTIVEEXP,      inactiveexpArgs);
+  PXOPT_ADD_MODE("-definegroup",             "", LAPTOOL_MODE_DEFINEGROUP,      definegroupArgs);
+  PXOPT_ADD_MODE("-pendinggroup",            "", LAPTOOL_MODE_PENDINGGROUP,     pendinggroupArgs);
+  PXOPT_ADD_MODE("-filtersforgroup",         "", LAPTOOL_MODE_FILTERSFORGROUP,  filtersforgroupArgs);
+  PXOPT_ADD_MODE("-updategroup",             "", LAPTOOL_MODE_UPDATEGROUP,      updategroupArgs);
+  PXOPT_ADD_MODE("-revertgroup",             "", LAPTOOL_MODE_REVERTGROUP,      revertgroupArgs);
   
   if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

tags/ipp-20130307/ippTools/src/pstamptool.c

@@ -49 +49 @@
 static bool revertjobMode(pxConfig *config);
 
-# if (1)
-// these are unused functions -- since they are 'static', this raises an warning
-// XXX: They actually are used
 static bool addprojectMode(pxConfig *config);
 static bool projectMode(pxConfig *config);
@@ -63 +60 @@
 static bool listfileMode(pxConfig *config);
 static bool deletefileMode(pxConfig *config);
-# endif
 
 # define MODECASE(caseName, func) \
@@ -103 +99 @@
         MODECASE(PSTAMPTOOL_MODE_STOPDEPENDENTJOB, stopdependentjobMode);
         MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode);
-# if (1)
         MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode);
         MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode);
@@ -115 +110 @@
         MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode);
         MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode);
-# endif
+
         default:
             psAbort("invalid option (this should not happen)");
@@ -452 +447 @@
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
     PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!=");
-    PXOPT_COPY_STR(config->args, where, "-name", "name", "==");
+    PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-username", "username", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
 
     PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
@@ -458 +455 @@
 
     if (!psListLength(where->list)) {
-        psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied");
+        psError(PS_ERR_UNKNOWN, true, "search paramters are required");
         return false;
     }
@@ -490 +487 @@
     if (!psArrayLength(output)) {
         psTrace("pstamptool", PS_LOG_INFO, "request not found");
-        // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for
+        // This causes main to exit with PS_EXIT_DATA_ERROR which the pstamp scripts are looking for
         psError(PXTOOLS_ERR_CONFIG, true, "request not found");
         psFree(output);
-        // we return false so that the caller can determine that a request does not exist
+        // we return false so that the caller can easily determine that a request does not exist
         return false;
     }
@@ -593 +590 @@
     PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
-    PXOPT_COPY_STR(config->args, where, "-reqType",     "reqType", "==");
-    PXOPT_COPY_STR(config->args, where, "-name",     "name", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-reqType",    "reqType", "==");
+    PXOPT_COPY_STR(config->args, where, "-name",       "name", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-username",   "username", "==");
     PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">=");
     PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<=");
@@ -621 +619 @@
     if (outdir) {
         psStringAppend(&query, ", outdir = '%s'", outdir);
+    }
+    if (username) {
+        psStringAppend(&query, ", username = '%s'", username);
     }
     if (clearfault) {
@@ -1233 +1234 @@
     PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false);
     PXOPT_LOOKUP_BOOL(no_create,  config->args, "-no_create", false);
+    PXOPT_LOOKUP_BOOL(hold,       config->args, "-hold", false);
 
     psMetadata *where = psMetadataAlloc();
@@ -1240 +1242 @@
     PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
 
-    // start a transaction eraly so it will contain any row level locks
+    // start a transaction early so it will contain any row level locks
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1264 +1266 @@
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1271 +1276 @@
     if (!output) {
         psError(PS_ERR_UNKNOWN, false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1279 +1287 @@
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
         }
@@ -1284 +1295 @@
         if (fault > 0) {
             fprintf(stderr, "existing dependent has fault %d\n", fault);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            // return the fault to the client. This is used by the postage stamp parser
             exit (fault);
         }
+        bool commit = false;
+        // Check the state of the exisiting dependent. By query it is either
+        // new or hold
+        psString state = psMetadataLookupStr(NULL, dep, "state");
+        if (!hold && !strcmp(state, "hold")) {
+            // There is a dependent for this component but it's state is hold.
+            // This client needs one that will run.
+            // Update the state
+            psString updateQuery = NULL;
+            psStringAppend(&updateQuery, "UPDATE pstampDependent SET state = 'new' "
+                "\nWHERE stage = '%s' AND imagedb = '%s' AND stage_id = %"PRId64 " AND component = '%s'", 
+                    stage, imagedb, stage_id, component);
+
+            if (!p_psDBRunQuery(config->dbh, updateQuery)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(updateQuery);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+            // set flag to commit this change
+            commit = true;
+            psFree(updateQuery);
+        }
+        // print the dep_id for the user
         printf("%" PRId64 "\n", dep_id);
         psFree(output);
-        if (!psDBRollback(config->dbh)) {
+        // now either commit the change or rollback the transaction which releases the lock
+        if (commit) {
+            if (!psDBCommit(config->dbh)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
             psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+            }
+        } else if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
         }
         return true;
@@ -1307 +1359 @@
         config->dbh,
         0,              // dep_id
-        "new",          // state
+        hold ? "hold" : "new",          // state
         stage,
         stage_id,

tags/ipp-20130307/ippTools/src/pstamptoolConfig.c

@@ -90 +90 @@
     psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0,            "list by req_id", 0);
     psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0,              "list by name", NULL);
+    psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-username", 0,          "list by user name", NULL);
+    psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-state", 0,              "list by state", NULL);
     psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0,        "req_id to not list", 0);
     psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -107 +109 @@
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0,      "search by reqType", NULL);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,      "search by reqType (LIKE comparsion)", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,         "search by reqType (LIKE comparsion)", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-username", 0,     "search by username", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
     psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL);
@@ -220 +223 @@
     psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false);
     psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-outdir", 0,    "define output directory for dependent (required)", NULL);
+    psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-hold", 0, "if creating new dependent set it's state to hold", false);
     psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false);
 

tags/ipp-20130307/ippTools/src/releasetool.c

@@ -702 +702 @@
     PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",      "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",   "warpSkyfile.skycell_id", "==");
+    
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false);
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
@@ -715 +719 @@
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
+    }
+
+    if (skycell_id) {
+        psStringAppend(&query, "\nJOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id AND warpRun.tess_id = warpSkyfile.tess_id");
     }
 
@@ -741 +749 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC");
     }
 
@@ -787 +795 @@
     bool includeSkycal = false;
 
+    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "==");
     if (psListLength(where->list)) {
         includeSkycal = true;
@@ -795 +803 @@
 
     PXOPT_COPY_STR(config->args, where, "-label",      "stackRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-stack_id",   "stackRun.stack_id", "==");
 
@@ -832 +840 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     PXOPT_LOOKUP_BOOL(simple, config->args,  "-simple", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     // find the parameters of all the exposures that we want to add to the release
@@ -847 +856 @@
     }
 
+    psString joinConditions = psStringCopy("");
+    // Finish up the selectors for the JOIN to previousRelStack
+    // join to rows with our stack_type
+    psStringAppend(&joinConditions, "\nAND previousRelStack.stack_type = '%s'", stack_type);
+
+    if (!strcmp(stack_type, "nightly")) {
+        // detect nightly stack entries that already exist for this day
+        psStringAppend(&joinConditions, "\nAND previousRelStack.mjd_obs = floor(stackSumSkyfile.mjd_obs)");
+    }
+
+    // Add in the where conditions
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-    psStringAppend(&query, " AND %s", whereClause);
+    psStringAppend(&query, "\nAND %s", whereClause);
     psFree(whereClause);
 
+    if (!strcmp(stack_type, "nightly")) {
+        // avoid stacks with NAN mjd_obs. These are old skycells that were lost due to system failure
+        // prior to the time that the mjd_obs was extracted from the headers and added to stackSumSkyfile
+        psStringAppend(&query, "\nAND stackSumSkyfile.mjd_obs IS NOT NULL");
+    }
+
     psFree(where);
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, joinConditions)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        psFree(joinConditions);
         return false;
     }
     psFree(query);
+    psFree(joinConditions);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -968 +1002 @@
     psMetadata *where = psMetadataAlloc();
 
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",    "relStack.stack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",   "relStack.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
     PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
     pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
-    PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
-    PXOPT_COPY_STR(config->args, where, "-filter",      "rawExp.filter", "LIKE");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">=");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",     "relExp.chip_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-cam_id",      "relExp.cam_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-warp_id",     "warpRun.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-state",       "relStack.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "relStack.filter", "LIKE");
+    PXOPT_COPY_F32(config->args, where, "-mjd_min",    "stackSumSkyfile.mjd_obs", ">=");
+    PXOPT_COPY_F32(config->args, where, "-mjd_max",    "stackSumSkyfile.mjd_obs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",     "relStack.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "relStack.skycell_id", "LIKE");
+//    PXOPT_COPY_STR(config->args, where, "-stack_type",  "relStack.stack_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-stack_data_group",  "stackRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
     PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+    pxskycellAddWhere(config, where);
+
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "IFNULL(skycalResult.fwhm_major, 999)", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "IFNULL(skycalResult.fwhm_major, 0)", "<=");
 
     PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
@@ -993 +1030 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("releasetool_listrelexp.sql");
+    pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "==");
+
+    psString query = pxDataGet("releasetool_listrelstack.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    psString where2 = NULL;
-    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
-        psError(psErrorCodeLast(), false, "pxspaceAddWhere failed");
         return false;
     }
@@ -1009 +1042 @@
         psStringAppend(&query, "\nWHERE %s", whereClause);
         psFree(whereClause);
-    } else if (where2) {
-        psStringAppend(&query, "\nWHERE ");
     } else {
         psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
@@ -1017 +1048 @@
     }
 
-    if (where2) {
-        psStringAppend(&query, "\n%s", where2);
-        psFree(where2);
-    }
-
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC");
     }
 
@@ -1051 +1077 @@
     }
 
-    if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "relStack", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
         psFree(output);
@@ -1270 +1296 @@
         }
         if (!psArrayLength(exposures)) {
+            fprintf(stderr, "no exposures found for lap_id %" PRId64 "\n", lap_id);
+#ifdef notdef
+            continue;
             psFree(now);
             psFree(output);
@@ -1278 +1307 @@
             }
             return false;
+#endif
         }
 
@@ -1301 +1331 @@
         psFree(exposures);
 
-        // set state of relGroup to 'new'
+        // update the state of relGroup to 'new'
         if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1325 +1355 @@
 static bool updaterelgroupMode(pxConfig *config)
 {
-    psError(PS_ERR_UNKNOWN, true, "not yet implemented");
-    return false;
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S32(group_id, config->args,      "-group_id", false, false);
+    PXOPT_LOOKUP_STR(group_name, config->args,    "-group_name",  false, false);
+    if (!group_id && !group_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either group_id or group_name is required\n");
+        return false;
+    }
+
+    PXOPT_LOOKUP_STR(state,    config->args,    "-set_state",  false, false);
+    PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false);
+    PXOPT_LOOKUP_STR(new_label, config->args,   "-set_label", false, false);
+    PXOPT_LOOKUP_S16(fault,    config->args,    "-set_fault",  false, false);
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault",  false);
+
+    if (!state && !exp_list_path && !new_label && !fault && !clearfault) {
+        psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n");
+        return false;
+    }
+
+    if (fault && clearfault) {
+        psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n");
+        return false;
+    }
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-group_id", "relGroup.group_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-group_name", "relGroup.group_name", "==");
+    // XXX TODO if label is enabled (for changing label or state for a block) we should
+    // disallow setting some parameters such as exp_list_path
+    //    PXOPT_COPY_S64(config->args, where, "-label", "relGroup.label", "==");
+    // make sure that we have enough parameters to identify the relGroup to change
+    if (!psListLength(where->list)){
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    psString sep = "";
+    psString comma = ", ";
+    psString query = psStringCopy("UPDATE relGroup SET");
+    if (state) {
+        psStringAppend(&query, "%s relGroup.state = '%s'", sep, state);
+        sep = comma;
+    }
+    if (exp_list_path) {
+        psStringAppend(&query, "%s relGroup.exp_list_path = '%s'", sep, exp_list_path);
+        sep = comma;
+    }
+    if (new_label) {
+        psStringAppend(&query, "%s relGroup.label = '%s'", sep, new_label);
+        sep = comma;
+    }
+    if (fault) {
+        psStringAppend(&query, "%s relGroup.fault = %d", sep, fault);
+        sep = comma;
+    }
+    if (clearfault) {
+        psStringAppend(&query, "%s relGroup.fault = 0", sep);
+        sep = comma;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected);
+
+
+    psFree(query);
+
+    return true;
 }
 
@@ -1410 +1518 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
     PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalResult.fwhm_major", "<=");
     PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");

tags/ipp-20130307/ippTools/src/releasetoolConfig.c

@@ -164 +164 @@
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL);
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-tess_id", 0,    "warpRun.tess_id", NULL);
 
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
@@ -195 +197 @@
     psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_zpt_stdev", 0,  "define zero point stdev", NAN);
     psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_fwhm_major", 0, "define fwhm_major", NAN);
-    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state (required)", NULL);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state", NULL);
     psMetadataAddS16(definerelstackArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define fault", 0);
 
-    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false);
     psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+    psMetadataAddU64(definerelstackArgs, PS_LIST_TAIL,  "-limit",   0, "limit result set to N items", 0);
+
+    // -listrelstack
+    psMetadata *listrelstackArgs = psMetadataAlloc();
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-relstack_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-stack_id", 0,   "select by stack ID", 0);
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-skycal_id", 0,   "select by skycal ID", 0);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-state", 0,        "select by released stack state", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-filter", 0,       "select by filter name (LIKE comparison)", NULL);
+    // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_begin", 0,"search for exposures by time (>=)", NULL);
+    // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_end", 0,  "search for exposures by time (<=)", NULL);
+
+    pxskycellAddArguments(listrelstackArgs);
+
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_min", 0, "search by measured seeing (>=)", NAN);
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN);
+
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_min", 0, "search by MJD seeing (>=)", NAN);
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_max", 0, "search by MJD (<=)", NAN);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group (LIKE comparison)", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
+    psMetadataAddS64(listrelstackArgs,  PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0);
+
+    psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-priority_order",   0, "order by release priority", false);
+
+    psMetadataAddU64(listrelstackArgs,  PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
+
 
     // -definerelgroup
@@ -217 +257 @@
         // parameters of the relGroup
     psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0,  "define group_type (required)", NULL);
-    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0,  "define group_name (required)", NULL);
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0,  "define group_name", NULL);
     psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL);
     psMetadataAddStr(definerelgroupArgs,  PS_LIST_TAIL, "-set_state", 0,      "define state", NULL);
@@ -240 +280 @@
     psMetadata *updaterelgroupArgs = psMetadataAlloc();
 
-        // set the target release
-    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID (required)", 0);
+    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID",  0);
+    psMetadataAddStr(updaterelgroupArgs,  PS_LIST_TAIL, "-group_name", 0, "select by group name", 0);
 
         // parameters of the relGroup
@@ -273 +313 @@
 
     PXOPT_ADD_MODE("-definerelstack",     "define a released stack",    RELEASETOOL_MODE_DEFINERELSTACK,  definerelstackArgs);
+    PXOPT_ADD_MODE("-listrelstack",       "list released stacks",      RELEASETOOL_MODE_LISTRELSTACK,    listrelstackArgs);
 
     PXOPT_ADD_MODE("-definerelgroup",     "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP,  definerelgroupArgs);

tags/ipp-20130307/pstamp/scripts/psmkreq

@@ -1 +1 @@
 #!/bin/env perl
 ###
-### pstamprequest
+### psmkreq
 ###
-###     Program to make a postage stamp request table for a set of coordinates
+###     Program to make a postage stamp request table for a set of coordinates using supplied command
+###     line options and defaults
 ###
 
@@ -28 +29 @@
 my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
 
-my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse);
-my ($unconvolved, $use_imfile_id, $no_wait);
+my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse);
+my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait);
+
+# new header keywords for version 2
+my $action = 'PROCESS';
+my $email = 'null';
 
 my $default_size = 100;
@@ -52 +57 @@
 my $comment;
 
+# new request specification columns for version 2
+my $survey_name ='null';
+my $release_name = 'null';
+my $run_type ='null';
+my $fwhm_min = 0;
+my $fwhm_max = 0;
+
+
 my $missing_tools;
 my $pstamp_request_file  = can_run('pstamp_request_file')  or (warn "Can't find required program pstamp_request_file"  and $missing_tools = 1);
@@ -62 +75 @@
 GetOptions(
     'list=s'            => \$list,          # list of coordinates if undef ra and dec or x and y are required
-    'ra=s'              => \$ra,            # 
+    'ra=s'              => \$ra,
     'dec=s'             => \$dec,
     'x=s'               => \$x,
@@ -87 +100 @@
     'comment=s'         => \$comment,
 
+    'survey=s'          => \$survey_name,
+    'release=s'         => \$release_name,
+    'run_type=s'        => \$run_type,
+    'fwhm_min=s'        => \$fwhm_min,
+    'fwhm_max=s'        => \$fwhm_max,
+
     'option_mask=i'     => \$option_mask,
     'image'             => \$image,
     'mask'              => \$mask,
+    'jpeg'              => \$jpeg,
     'variance'          => \$variance,
     'cmf'               => \$cmf,
@@ -96 +116 @@
     'inverse'           => \$inverse,
     'unconvolved'       => \$unconvolved,
+    'convolved'         => \$convolved,
+    'uncompressed'      => \$uncompressed,
     'use_imfile_id'     => \$use_imfile_id,
     'do_not_wait'       => \$no_wait,
+
+    'action=s'          => \$action,
+    'email=s'           => \$email,
 
     'verbose'           => \$verbose,
@@ -156 +181 @@
 }
 
+if ($stage eq 'stack') {
+    # default is to do convolved stack
+    unless (defined $option_mask || $convolved) {
+        $unconvolved = 1;
+    }
+}
+
 checkFilter($filter, 'null', $filter)  if $filter;
 checkMJD($mjd_min, 0, "") if $mjd_min;
@@ -167 +199 @@
         $option_mask |= $PSTAMP_SELECT_MASK     if $mask;
         $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance;
+        $option_mask |= $PSTAMP_SELECT_JPEG     if $jpeg;
+
+        # if no image was requested make a stamp of the image
         $option_mask = $PSTAMP_SELECT_IMAGE    if $option_mask == 0;
 
-        # if no image was requested make a stamp of the image
 
         $option_mask |= $PSTAMP_SELECT_CMF      if $cmf;
@@ -207 +241 @@
 } else {
     $rows = [];
-    push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max);
+    push @$rows, buildRow("", $comment, 1, $x, $y, $filter, $mjd_min, $mjd_max);
 }
 
 my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps);
-print $tdf "$req_name 1\n";
+my $line = "$req_name 2 $action $email";
+print "$line\n" if $verbose;
+print $tdf "$line\n";
 my $rownum = 0;
 foreach my $row (@$rows) {
     $rownum++;
     my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height"
-        . " $coord_mask $job_type $option_mask $project $req_type"
+        . " $coord_mask $job_type $option_mask $project $survey_name $release_name $req_type"
         . " $stage $id $tess_id $component $data_group"
-        . " $row->{filter} $row->{mjd_min} $row->{mjd_max}";
+        . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $run_type $fwhm_min $fwhm_max";
 
     if ($row->{comment} and $row->{comment} ne '') {
@@ -235 +271 @@
     $command .= " --output $output" if $output;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+        run(command => $command, verbose => $verbose);
     unless ($success) {
         print STDERR @$stderr_buf;
@@ -274 +310 @@
 
     my $row = {};
-    $row->{ra}      = checkRA($vals[0], $linenumber);
-    $row->{dec}     = checkDEC($vals[1], $linenumber);
-    $row->{filter}  = checkFilter($vals[2], $filter, $linenumber);
-    $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);;
-    $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);;
+    $row->{rownum}  = checkRownum($vals[0], $linenumber);
+    $row->{ra}      = checkRA($vals[1], $linenumber);
+    $row->{dec}     = checkDEC($vals[2], $linenumber);
+    $row->{filter}  = checkFilter($vals[3], $filter, $linenumber);
+    $row->{mjd_min} = checkMJD($vals[4], $mjd_min, $linenumber);;
+    $row->{mjd_max} = checkMJD($vals[5], $mjd_max, $linenumber);;
     $row->{comment} = $comment;
 
@@ -305 +342 @@
     my $linenumber = shift;
 
-    my $result;
+   my $result;
     if ($c =~ /\:/) {
         # sexagesmial format not valid for pixel coordinates
@@ -329 +366 @@
 }
 
+sub checkRownum {
+    my $rownum = shift;
+    my $linenumber = shift;
+    die "rownum can not be null at $linenumber\n" unless defined $rownum;
+
+    # XXX: For now just check that rownum is defined and a single word
+    # extend this. Should we require a number?
+    my @words = split " ", $rownum;
+    die "$rownum is not a vaild rownum at $linenumber\n" unless (scalar @words) == 1;
+}
+
 sub checkRA {
     my $ra = shift;
@@ -335 +383 @@
 
     if ($ra =~ /\:/) {
+        # assume RA is in hours:minutes:seconds
         return $checked * 360. / 24.;
     } else {

tags/ipp-20130307/pstamp/scripts/pstamp_finish.pl

@@ -107 +107 @@
     my ($header, $rows) = read_request_file($req_file);
 
-    my $action = $header->{ACTION};
-
-    if (!$header or (($action eq 'PROCESS') and !$rows)) {
+    if (!$header or !$rows) {
         # Since a request got queued, the request file must have been readable at some point 
         my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
@@ -128 +126 @@
 
     my $request_fault = 0;
-    if ($action eq 'PROCESS') {
+    {
         # The results table definition file
         my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
@@ -238 +236 @@
                 } else {
                     my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
-                        if $fault eq $PSTAMP_SUCCESS;
+                        if $job->{state} eq 'stop' and $fault eq $PSTAMP_SUCCESS;
                     print STDERR "no reglist file for job $job_id\n" if $verbose;
                     print $tdf "$rownum|$fault|$error_string|0|$job_id|";
@@ -273 +271 @@
             }
         }
-    } elsif ($action ne 'LIST') {
-        my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
-    } else {
-        # pstampparse did all of the work
     }
 

tags/ipp-20130307/pstamp/scripts/pstamp_job_run.pl

@@ -113 +113 @@
 
     if ($stage eq "raw") {
+        # zap options that don't apply to raw stage
         $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
@@ -136 +137 @@
         $argString .= " -astrom $params->{astrom}";
         push @file_list, $params->{astrom};
+    }
+
+    if ($options & $PSTAMP_SELECT_SOURCES) {
+        # Extract sources from astrometry file if provided. This will be the smf for chip stage
+        # or the skycal cmf for stacks
+        if ($params->{astrom}) {
+            $argString .= " -write_cmf";
+            if ($stage eq 'stack') {
+                # Set psphot recipe to STACKPHOT so that the extended source paramters will
+                # be copied from the cmf file.
+                $argString .= " -recipe PSPHOT STACKPHOT"
+            }
+        } elsif ($params->{cmf}) {
+            $argString .= " -write_cmf";
+            push @file_list, $params->{cmf};
+        } else {
+            print "Could not find suitable sources file will not write cmf\n";
+        }
     }
 
@@ -254 +273 @@
                            $PSTAMP_SELECT_MASK     => "mk.fits",
                            $PSTAMP_SELECT_VARIANCE => "wt.fits",
+                           $PSTAMP_SELECT_SOURCES  => "cmf",
                            $PSTAMP_SELECT_JPEG     => "jpg");
 
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
 
         foreach my $key (keys (%extensions)) {
@@ -400 +420 @@
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $cmf_file;
-        if ($stage ne 'chip') {
-            # we don't ship chip stage cmf files because they may not be censored
-            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
-        }
+#        if ($stage ne 'chip') {
+#            # we don't ship chip stage cmf files because they may not be censored
+#            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+#        }
 
         my $outdir = dirname($output_base);

tags/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl

@@ -158 +158 @@
 my $request_fault = $PSTAMP_INVALID_REQUEST;
 
+# default action is to process the request after parsing. This can be overridden by
+# PREVIEW mode for pstamp requests
+my $action = 'PROCESS';
+
 if (-r $uri) {
     # run the appropriate parse command to parse the queue the jobs for this request
     # first check the extension header to find the EXTNAME
-    $request_type = find_request_type($uri);
+    $request_type = find_request_type($uri, \$action);
 
     if ($request_type) {
@@ -240 +244 @@
 
     if ($success) {
-        $newState = 'run';
+        # XXX: This bit of the postage stamp request API has slipped in here because we need to control
+        # the new state of the request
+        if ($action eq 'PROCESS') {
+            $newState = 'run';
+        } elsif ($action eq 'PREVIEW') {
+            $newState = 'parsed';
+        } else {
+            print STDERR "WARNING Ignoring unexpected value for ACTION found in request header: $action\n";
+        }
     } else {
         $fault = $error_code >> 8;
@@ -275 +287 @@
     # find the EXTNAME in the input fits table
     my $file_name = shift;
-    my $out = `echo $file_name | fields -x 0 EXTNAME`;
+    my $r_action = shift;
+
+    my $out = `echo $file_name | fields -x 0 EXTNAME ACTION`;
 
     if ($out) {
         # output from fields is filename value
-        my ($dummy, $extname) = split " ", $out;
+        my ($dummy, $extname, $action) = split " ", $out;
+
+        # Set the action if it is defined in the request header
+        # XXX:consider doing this only if extname is PS1_PS_REQUSET
+        $$r_action = $action if ($action);
 
         return $extname;

tags/ipp-20130307/pstamp/scripts/pstamp_request_file

@@ -18 +18 @@
      $output,                   # Name of output table
      $req_name, 
+     $email, 
      $help
      );
@@ -25 +26 @@
            'output|o=s'   => \$output,
            'req_name|r=s' => \$req_name,
-           'help|h'         => \$help,
+           'email=s'      => \$email,
+           'help|h'       => \$help,
 ) or pod2usage( 2 );
 
@@ -54 +56 @@
                     value => undef
         },
-        { name => 'USERNAME',
-                    writetype => TSTRING,
-                    comment => 'username for request (optional)',
-                    value => undef
-        },
+#        { name => 'USERNAME',
+#                    writetype => TSTRING,
+#                    comment => 'username for request (optional)',
+#                    value => undef
+#        },
         { name => 'EMAIL',
                     writetype => TSTRING,
@@ -65 +67 @@
         },
 ];
+
+my $email_column_num = 3;
 
 # Specification of columns to write
@@ -97 +101 @@
         { name => 'MJD_MIN',    type => 'D',   writetype => TDOUBLE },
         { name => 'MJD_MAX',    type => 'D',   writetype => TDOUBLE },
+        # new in version 2
         { name => 'RUN_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'FWHM_MIN',   type => 'D',   writetype => TDOUBLE },
+        { name => 'FWHM_MAX',   type => 'D',   writetype => TDOUBLE },
 
         { name => 'COMMENT ',   type => '64A', writetype => TSTRING },
@@ -162 +169 @@
     $req_name = $header->[0]->{value};
 }
+if ($email) {
+    $header->[$email_column_num]->{value} = $email;
+}
+
 
 die "no request name defined" unless defined $req_name;
@@ -299 +310 @@
 
             $$r_extver = $vals[1];
-            if ($extver > 1) {
+            if ($$r_extver > 1) {
                 die "number of header columns in input $nvals does not equal expected number of header words $nhead"
                     if (@vals != @$header);
@@ -323 +334 @@
     if ($$r_extver == 1) {
         # pop off the colData arrays to account for the unused columns
+        pop @colData;
+        pop @colData;
         pop @colData;
         pop @colData;
@@ -346 +359 @@
         }
         # trim leading whitespace from comment
-        $comment =~ s/^\s+//;
+        $comment =~ s/^\s+// if $comment;
         if ($comment) {
             if ($comment =~ /#/) {

tags/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl

@@ -71 +71 @@
 }
 
-# make a request file
+# make the request file in a sub directory of the current directory
 my $cur_dir = getcwd();
 
 #print STDERR "cur_dir is $cur_dir\n";
 
+# put file in directory for the current date
 my $datestr = strftime "%Y/%m/%d", gmtime;
 my $datedir = "$cur_dir/webreq/$datestr";

tags/ipp-20130307/pstamp/scripts/pstampparse.pl

@@ -27 +27 @@
 my $outdir;
 my $product;
-my $label;
+my $label = "";
 my $save_temps;
 my $no_update;
@@ -57 +57 @@
     die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
+} else {
+    $req_id = 0;
+    $outdir = "nowhere";
+    $product = "dummy";
 }
 
@@ -83 +87 @@
 $pstamptool .= " -dbserver $dbserver" if $dbserver;
 
-# list_job is a deugging mode
-$no_update = 1 if $mode eq "list_job";
+# If $mode is not queue_job we are using a debugging mode
+# do not update the database
+$no_update = 1 if $mode ne "queue_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 #
-# get the data from the extension header
+# Read the keywords from the extension header
 #
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    # fields doesn't return zero when it succeeds
-    #unless ($success) {
-    #    print STDERR @$stderr_buf;
-    #}
+
+    # note fields doesn't return zero when it succeeds.
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
-# This program shouldn't have been run if the request file is bogus.
-# No need to notify the client
-my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
-my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
-
+# pstamp_parser_run.pl would not have run this program unless the request file was ok
+my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname;
+my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver >= 2) {
+    # We have a version 2 file. Require that the new keywords be supplied. 
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    # XXX check for "valid" $email
+} else {
+    # for version 1 file the action is process and email is not used
+    $action = 'PROCESS';
+    $email = 'null';
+}
+
+print "Request Header Keywords EXTVER: $extver REQ_NAME: $req_name ACTION: $action EMAIL: $email\n";
 
 # check for duplicate request name
@@ -161 +179 @@
     # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
@@ -174 +193 @@
 
 #
-# now convert the request table to an array of metadata config docs
+# now convert the request table to an array of metadatas
 #
 
@@ -191 +210 @@
     my $dtime_request_file = gettimeofday() - $start_request_file;
     print "Time to read and parse request file: $dtime_request_file\n";
-
-}
-
-#
-# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
-# in the sense that their selection parameters will yield the same "Runs".
-# Process the groups of rows together to reduce lookup time and to potentially make multiple
-# stamps from the same ppstamp process.
-#
-my @rowList;
+}
+
+my $nRows = scalar @$rows;
+print "\n$nRows rows read from request file\n";
+
+
 my $num_jobs = 0;
 my $imageList;
 my $stage;
-my $need_magic;
 foreach my $row (@$rows) {
-    # santiy check the paramaters
+
+    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
+        # this is a big request and it came from the upload page and doesn't have a specific label assigned
+        # change it to the generic one that has lower with lower priority
+        $label = 'WEB.BIG';
+        print "\nChanging label for big WEB.UP request to $label\n";
+
+        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+
+    # validate the paramaters
     if (!checkRow($row)) {
         # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
@@ -216 +245 @@
     $row->{error_code} = 0;
 
-    if (scalar @rowList == 0) {
-        push @rowList, $row;
-        next;
-    }
-
-    my $firstRow = $rowList[0];
-    if (same_images_of_interest($firstRow, $row)) {
-        # add this row to the list and move on
-        push @rowList, $row;
-        next;
-    }
-
-    # the images of interest for this new row doesn't match the list. 
-    # process the list ...
-    $num_jobs += processRows($action, \@rowList);
-
-    # and reset the list to contain just the new row
-    @rowList = ($row);
-}
-
-# out of rows process the list
-if (scalar @rowList > 0) {
-    $num_jobs += processRows($action, \@rowList);
+    $num_jobs += processRow($action, $row);
 }
 
@@ -252 +259 @@
     my $row = shift;
 
+    # check validity of the paramters in a request specification
     # If we encounter an error for a particular row add a job with the proper fault code.
+    # also adjust some paramterers like filter and set defaults for some others
 
     my $stage = $row->{IMG_TYPE};
@@ -284 +293 @@
     }
 
-
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
@@ -314 +322 @@
         return 0;
     }
+
+    my $fwhm_min = $row->{FWHM_MIN};
+    if (!defined $fwhm_min) {
+        $row->{FWHM_MIN} = 0;
+    } elsif (!validNumber($fwhm_min)) {
+        print STDERR "$fwhm_min is not a valid FWHM_MIN\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $fwhm_max = $row->{FWHM_MAX};
+    if (!defined $fwhm_max) {
+        $row->{FWHM_MAX} = 0;
+    } elsif (!validNumber($fwhm_max)) {
+        print STDERR "$fwhm_max is not a valid FWHM_MAX\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
     my $data_group = $row->{DATA_GROUP};
     if (!defined $data_group) {
@@ -334 +360 @@
 
     my $wholefile = 0;
-    if (!$skycenter && $row->{CENTER_X} == 0 && $row->{CENTER_Y} == 0) {
+    if (($row->{WIDTH} == 0 && $row->{HEIGHT} == 0) ||
+       (!$skycenter && $row->{CENTER_X} == 0 && $row->{CENTER_Y} == 0)) {
         # Secret code for returning the whole file
         $wholefile = 1;
@@ -362 +389 @@
         return 0;
     }
-
-    # $mode list_uri is a debugging mode (it may used by the http interface)
-    # if this happens just croak
-   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
 
     if ($req_type eq "bycoord") {
@@ -384 +406 @@
     }
 
+
     return 1;
 }
@@ -392 +415 @@
 
     $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRow {
+    my $action  = shift;
+    my $row     = shift;
+
+    my $num_jobs = 0;
+
+    my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PROJECT);
+        $num_jobs++;
+        return $num_jobs;
+    }
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+
+    my $req_type    = $row->{REQ_TYPE};
+    my $rownum      = $row->{ROWNUM};
+
+    # Since user can get unmagicked data "by coordinate" requests can go back in time
+    # to dredge unusable data from the "dark days"...
+    if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) {
+        # ... so unless the user sets mjd_min clamp it to 2009-04-01
+        # XXX: This value should live in the pstampProject table not be hardcoded here
+        $row->{MJD_MIN} = 54922;
+    }
+
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $row->{ID} = "" if !$row->{ID};
+    }
+    
+    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+    # request specification. An array reference is returned.
+    my $start_locate = gettimeofday();
+
+    print "\nCalling new_locate_images for row: $rownum\n";
+
+    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+
+    my $dtime_locate = gettimeofday() - $start_locate;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
+
+    my @rowList = ($row);
+    $num_jobs += queueJobs($action, \@rowList, $imageList);
+
+    # if this row slipped through without a job being added add one. 
+    if ($row->{job_num} == 0) {
+        print "row $row->{ROWNUM} produced no jobs\n";
+        print STDERR "row $row->{ROWNUM} produced no jobs\n";
+        my $error_code = $row->{error_code};
+        $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+        insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+    }
+
     return $num_jobs;
 }
@@ -401 +485 @@
 
     if ($action eq 'LIST') {
+        # LIST is not allowed by caller. Can't get here.
         return list_targets($rowList);
     }
@@ -420 +505 @@
     }
     my $req_type  = $row->{REQ_TYPE};
-    $stage        = $row->{IMG_TYPE};
+    my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
     my $component = $row->{COMPONENT};
@@ -436 +521 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    $need_magic    = $proj_hash->{need_magic};
+    my $need_magic    = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -496 +581 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -506 +591 @@
     $row->{need_magic} = $need_magic;
 
-    $num_jobs += queueJobs($mode, \@rowList, $imageList);
+    $num_jobs += queueJobs($action, $rowList, $imageList);
 
     # if a row slipped through with no jobs add a faulted one
-    foreach my $row (@rowList) {
+    foreach my $row (@$rowList) {
         if ($row->{job_num} == 0) {
             print "row $row->{ROWNUM} produced no jobs\n";
@@ -528 +613 @@
     my $image = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $rownum = $row->{ROWNUM};
@@ -544 +629 @@
     my $h = $row->{HEIGHT};
     my $coord_mask = $row->{COORD_MASK};
+
     my $wholeFile = 0; 
-    if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-        # Center of 0, 0 in pixel coordinates is interpreted to mean
-        # return the entire file
-        if ($x == 0 && $y == 0) {
-            $roi_string = "-wholefile";
-            $wholeFile = 1;
+    # For historical reasons there are two ways to specify that the entire file be returned 
+    # rather than a postage stamp ...
+    if ($w == 0 and $h == 0) {
+        # ... The right way: width and height both zero ...
+        $wholeFile = 1;
+    } else {
+        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+            if ($x == 0 && $y == 0) {
+                # ... and pixel coordinate center of 0, 0
+                # I made this one up without thinking through the API clearly.
+                # allowing width and height to do it works much better
+                $wholeFile = 1;
+            } else {
+                $roi_string = "-pixcenter $x $y";
+            }
         } else {
-            $roi_string = "-pixcenter $x $y";
-        }
+            $roi_string = "-skycenter $x $y";
+        }
+    }
+
+    if ($wholeFile) {
+        $roi_string = "-wholefile";
+    } elsif ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+            $roi_string .= " -pixrange $w $h";
     } else {
-        $roi_string = "-skycenter $x $y";
-    }
-    if (!$wholeFile) {
-        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-            $roi_string .= " -pixrange $w $h";
-        } else {
             $roi_string .= " -arcrange $w $h";
-        }
     }
 
@@ -611 +705 @@
     write_params($output_base, $image);
 
-    my $newState = "run";
+    my $newState = $action eq 'PROCESS' ? "run" : "parsed";
     my $fault = 0;
     my $dep_id;
 
-    queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+    queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
     my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
@@ -644 +738 @@
 sub queueJobs
 {
-    my $mode = shift;
+    my $action = shift;
     my $rowList = shift;
     my $imageList = shift;
 
-    my $firstRow = $rowList[0];
+    my $firstRow = $rowList->[0];
     my $stage    = $firstRow->{IMG_TYPE};
     my $job_type = $firstRow->{JOB_TYPE};
@@ -656 +750 @@
 
     if ($mode eq "list_uri") {
+        $num_jobs = $imageList ? scalar @$imageList : 0;
+        print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n";
         foreach my $image (@$imageList) {
             print "$image->{image}\n";
+            ++$firstRow->{job_num};
         }
     } elsif ($job_type eq "get_image") {
@@ -664 +761 @@
         my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
 
-        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $action);
 
     } else {
@@ -692 +789 @@
                 my $row = $rowList->[$i];
 
-                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $action);
             }
         }
@@ -706 +803 @@
     my $stage = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $num_jobs = 0;
@@ -753 +850 @@
         write_params($output_base, $image);
 
-        my $newState = "run";
+        my $newState = $action eq 'PROCESS' ? "run" : "parsed";
         my $fault = 0;
         my $dep_id;
 
-        queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+        queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
         $num_jobs++;
@@ -899 +996 @@
 sub get_dependent 
 {
-    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
+    my ($action, $r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
     # chipRun's can be in full state if destreaking is necessary
@@ -918 +1015 @@
     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
-
-    # compute rlabel for the run.
-    # XXX: This bit of policy shouldn't be buried so deeply in the code
-    # For now use one that implies 'postage stamp server' 'update' 'request_label"
-    my $rlabel = "ps_ud_" . $label if $label;
-    $command .= " -rlabel $rlabel" if $rlabel;
+    $command .= ' -hold' if $action eq 'PREVIEW';
+
+    if ($label) {
+        # compute rlabel for the run.
+        # XXX: This bit of policy shouldn't be buried so deeply in the code
+        # For now use one that implies 'postage stamp server' 'update' 'request_label"
+        my $rlabel = "ps_ud_" . $label;
+        $command .= " -rlabel $rlabel";
+    }
 
     if (!$no_update) {
@@ -931 +1031 @@
             my $fault = $error_code >> 8;
             print STDERR "$command failed with fault $fault\n";
-            if ($fault < 10) {
+            if ($fault < $PSTAMP_FIRST_ERROR_CODE) {
                 # pstamptool returns an error if an existing depenent is faulted
-                # Set the object to not available even if the fault < 10
+                # Set the object to not available even if the fault < $PSTAMP_FIRST_ERROR_CODE
                 # which is nominally a recoverable error in order to keep
                 # the request from faulting (which can be very expensive if
@@ -939 +1039 @@
                 $fault = $PSTAMP_NOT_AVAILABLE
             }
-            if ($fault >= 10) {
+            if ($fault >= $PSTAMP_FIRST_ERROR_CODE) {
                 $$r_dep_id = 0;
                 $$r_fault = $fault;
@@ -967 +1067 @@
 
 sub queueUpdatesIfNeeded {
+    my $action = shift;
     my $stage = shift;
     my $image = shift;
@@ -973 +1074 @@
     my $r_fault = shift;
     my $r_dep_id = shift;
+
+    my $need_magic = 0;
 
     if ($stage ne 'raw') {
@@ -1016 +1119 @@
                     # set up to queue an update run
                     my $require_magic = ($need_magic or $image->{magicked});
-                    get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
+                    get_dependent($action, \$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
                         $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
                 }
@@ -1070 +1173 @@
     my $fault = shift;
 
-    carp $msg;
+    print STDERR $msg;
 
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary

tags/ipp-20130307/pstamp/src/ppstampParseCamera.c

@@ -95 +95 @@
     }
         
-
-    // XXX: create a filerule for PPSTAMP.ASTROM
     pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
     if (!status) {
@@ -103 +101 @@
     }
     if (astrom) {
-        psLogMsg ("ppstamp", 3, "using supplied astrometry\n");
+        psLogMsg ("ppstamp", 3, "Using supplied astrometry.\n");
     } else {
-        psLogMsg ("ppstamp", 3, "using header astrometry\n");
+        psLogMsg ("ppstamp", 3, "Using header astrometry.\n");
     }
-
-#ifdef notyet
-    // add recipe options supplied on command line
-    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, RECIPE_NAME);
-#endif
 
     // Set up the output target
@@ -138 +131 @@
             }
         } else {
-            psLogMsg ("ppstamp", PS_LOG_INFO, "output sources file requested but no input supplied. Will try the supplied astrometry file.\n");
+            psLogMsg ("ppstamp", PS_LOG_INFO, "Output sources file requested but no -sources supplied.\n");
+            psLogMsg ("ppstamp", PS_LOG_INFO, "Will use the sources in the supplied astrometry file.\n");
         }
         

tags/ipp-20130307/pstamp/src/pstampdump.c

@@ -94 +94 @@
         if (!strcmp(extname, "PS1_PS_REQUEST")) {
             psString extver = psMetadataLookupStr(NULL, header, "EXTVER");
+            psString action = psMetadataLookupStr(NULL, header, "ACTION");
+            psString email = psMetadataLookupStr(NULL, header, "EMAIL");
             if (!extver) {
                 // work around bug in MOPS request files
@@ -105 +107 @@
                 }
             }
-            printf("%s %s %s\n", extname, extver, req_name);
+            if (!strcmp(extver, "1")) {
+                printf("%s %s %s\n", extname, extver, req_name);
+            } else {
+                if (!action) {
+                    psErrorStackPrint(stderr, "failed to find action in fits header of: %s\n", fileName);
+                    return PS_EXIT_DATA_ERROR;
+                }
+                if (!email) {
+                    psErrorStackPrint(stderr, "failed to find action in fits header of: %s\n", email);
+                    return PS_EXIT_DATA_ERROR;
+                }
+                printf("%s %s %s %s %s\n", extname, extver, req_name, action, email);
+            }
         } else if (!strcmp(extname, "PS1_PS_RESULTS")) {
             psS64 req_id = psMetadataLookupS64(NULL, header, "REQ_ID");