Changeset 34466 for branches/eam_branches/ipp-20120905

Timestamp:

Sep 25, 2012, 10:05:21 AM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20120905

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (6 diffs)
• ippMonitor (modified) (1 prop)
• ippMonitor/Makefile.in (modified) (3 diffs)
• ippMonitor/def/simplePlotCamImage.d (modified) (3 diffs)
• ippMonitor/def/simplePlotSkycalImage.d (copied) (copied from trunk/ippMonitor/def/simplePlotSkycalImage.d )
• ippMonitor/def/simplePlotStackImage.d (copied) (copied from trunk/ippMonitor/def/simplePlotStackImage.d )
• ippMonitor/def/simplePlotStaticskyImage.d (copied) (copied from trunk/ippMonitor/def/simplePlotStaticskyImage.d )
• ippMonitor/raw/ipp.plots.dat (modified) (1 diff)
• ippMonitor/raw/simplePlotcam.php (modified) (8 diffs)
• ippMonitor/raw/skycellplot.php (copied) (copied from trunk/ippMonitor/raw/skycellplot.php )
• ippMonitor/scripts/skycellplot.dvo (copied) (copied from trunk/ippMonitor/scripts/skycellplot.dvo )
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippToPsps (modified) (1 prop)
• ippToPsps/jython/objectbatch.py (modified) (4 diffs)
• ippToPsps/jython/scratchdb.py (modified) (1 diff)
• ippToPsps/jython/stackbatch.py (modified) (8 diffs)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/difftool_definestackstack_part0.sql (modified) (1 diff)
• ippTools/share/difftool_definestackstack_part1.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 prop)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/difftoolConfig.c (modified) (1 diff)
• ippTools/src/magictool.c (modified) (1 prop)
• ippconfig (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppTranslate/src/ppMopsWrite.c (modified) (7 diffs)
• psphot (modified) (1 prop)
• psphot/src (modified) (1 prop)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/dquery_finish.pl (modified) (2 diffs)
• pstamp/scripts/psmkreq (modified) (9 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_finish.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_request_file (modified) (8 diffs)
• pstamp/scripts/pstamp_targetlist_file.pl (copied) (copied from trunk/pstamp/scripts/pstamp_targetlist_file.pl )
• pstamp/scripts/pstampparse.pl (modified) (6 diffs)
• psvideophot (modified) (1 prop)

branches/eam_branches/ipp-20120905/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -25 +25 @@
                     $PSTAMP_SELECT_WEIGHT
                     $PSTAMP_SELECT_CMF
+                    $PSTAMP_SELECT_SOURCES
                     $PSTAMP_SELECT_PSF
                     $PSTAMP_SELECT_BACKMDL
@@ -32 +33 @@
                     $PSTAMP_SELECT_UNCONV
                     $PSTAMP_RESTORE_BACKGROUND
+                    $PSTAMP_USE_IMFILE_ID
+                    $PSTAMP_NO_WAIT_FOR_UPDATE
                     $PSTAMP_REQUEST_UNCENSORED
                     $PSTAMP_REQUIRE_UNCENSORED
-                    $PSTAMP_USE_IMFILE_ID
-                    $PSTAMP_NO_WAIT_FOR_UPDATE
                     $PSTAMP_SUCCESS
                     $PSTAMP_FIRST_ERROR_CODE
@@ -59 +60 @@
 our $PSTAMP_RANGE_IN_PIXELS  = 2;
 
+# Definition of the bits in OPTION_MASK
 our $PSTAMP_SELECT_IMAGE     = 1;
 our $PSTAMP_SELECT_MASK      = 2;
 our $PSTAMP_SELECT_VARIANCE  = 4;
 our $PSTAMP_SELECT_WEIGHT    = 4;
+our $PSTAMP_SELECT_SOURCES   = 8;
 our $PSTAMP_SELECT_CMF       = 8;
 our $PSTAMP_SELECT_PSF       = 16;
 our $PSTAMP_SELECT_BACKMDL   = 32;
 our $PSTAMP_SELECT_JPEG      = 64;
+# unused 128
+# unused 256
 our $PSTAMP_SELECT_UNCOMPRESSED = 512;
-our $PSTAMP_SELECT_INVERSE   = 1024;
-our $PSTAMP_SELECT_UNCONV    = 2048;
-our $PSTAMP_RESTORE_BACKGROUND = 4096;
-
-our $PSTAMP_USE_IMFILE_ID    = 16384;
-
+our $PSTAMP_SELECT_INVERSE      = 1024;
+our $PSTAMP_SELECT_UNCONV       = 2048;
+our $PSTAMP_RESTORE_BACKGROUND  = 4096;
+# unused 8192
+
+our $PSTAMP_USE_IMFILE_ID      = 16384;
 our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
-our $PSTAMP_REQUEST_UNCENSORED = 1 << 16;
-our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17;
+our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
+our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
 
 # job and result codes
@@ -162 +167 @@
     my $fields_output;
     {
-        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         $fields_output = join "", @$stdout_buf;
     }
-    my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+    my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
 
     # make sure the file contains what we are expecting
@@ -174 +179 @@
                     if !$extname or ($extname ne "PS1_PS_REQUEST");
     die "REQ_NAME not found in $request_file_name"  if (!$req_name);
-    die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
+    die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "2" and $extver ne "1");
 
     my %header;
@@ -180 +185 @@
     $header{EXTVER}   = $extver;
     $header{EXTNAME}  = $extname;
+    if ($extver > 1) {
+        $header{ACTION} = $action;
+        $header{USER} = $username;
+        $header{EMAIL} = $email;
+    } else {
+        $header{ACTION} = $action = "PROCESS";
+        $header{USERNAME} = 'null';
+        $header{EMAIL} = 'null';
+    }
+
+    if ($action eq "LIST") {
+        return (\%header, undef);
+    } elsif ($action ne "PROCESS") {
+        die "unexpected request ACTION found: $action in $request_file_name";
+    }
+
 
     #

branches/eam_branches/ipp-20120905/ippMonitor/Makefile.in

@@ -14 +14 @@
 PROGRAMS = \
 $(DESTBIN)/skyplot.dvo \
+$(DESTBIN)/skycellplot.dvo \
 $(DESTBIN)/czartool_getServerStatus.pl \
 $(DESTBIN)/czartool_revert.pl \
@@ -57 +58 @@
 $(DESTWWW)/simplePlotraw.php \
 $(DESTWWW)/skyplot.php \
+$(DESTWWW)/skycellplot.php \
 $(DESTWWW)/getimage.php \
 $(DESTWWW)/ipp.czartool.dat \
@@ -211 +213 @@
 $(DESTWWW)/simplePlotChipImage.php \
 $(DESTWWW)/simplePlotCamImage.php \
+$(DESTWWW)/simplePlotStackImage.php \
+$(DESTWWW)/simplePlotStaticskyImage.php \
+$(DESTWWW)/simplePlotSkycalImage.php \
 $(DESTWWW)/histogramBackgroundImage.php \
 $(DESTWWW)/histogramCamProcessedExpImage.php \

branches/eam_branches/ipp-20120905/ippMonitor/def/simplePlotCamImage.d

@@ -1 @@
-TABLE camRun, chipRun, rawExp
+TABLE camProcessedExp, camRun, chipRun, rawExp
 TITLE New sky plot - cam
 FILE  simplePlotCamImage.php
@@ -7 +7 @@
 WHERE chipRun.exp_id = rawExp.exp_id
 WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
 
 #    field                size  format  name           show     link to         extras
@@ -24 +25 @@
 FIELD rawExp.bg,                  5, %.2f,   backgnd,         restrict
 FIELD rawExp.bg_stdev,            5, %.2f,   stdev,           restrict
+FIELD camProcessedExp.fwhm_major, 5, %.2f,   FWHM_major,      restrict      
 
 # What to plot

branches/eam_branches/ipp-20120905/ippMonitor/raw/ipp.plots.dat

@@ -5 +5 @@
 menulink  | menuselect      | link    | Simple plot - chip                  | simplePlotChipImage.php
 menulink  | menuselect      | link    | Simple plot - cam                   | simplePlotCamImage.php
+menulink  | menuselect      | link    | Simple plot - stack                 | simplePlotStackImage.php
+menulink  | menuselect      | link    | Simple plot - staticsky             | simplePlotStaticskyImage.php
+menulink  | menuselect      | link    | Simple plot - skycal             | simplePlotSkycalImage.php
 
 menutop   | menutop      | plain   |                                   | 

branches/eam_branches/ipp-20120905/ippMonitor/raw/simplePlotcam.php

@@ -28 +28 @@
 // define restrictiosn to the queries
 // ** TABLE RESTRICTIONS **
-$WHERE = "WHERE camRun.state != 'new' AND chipRun.exp_id = rawExp.exp_id AND camRun.chip_id = chipRun.chip_id";
+$WHERE = "WHERE camRun.state != 'new' AND chipRun.exp_id = rawExp.exp_id AND camRun.chip_id = chipRun.chip_id AND camProcessedExp.cam_id = camRun.cam_id";
 $WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0);
 $WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'string', 1.0);
@@ -62 +62 @@
 $WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'max', 1.0);
 $WHERE = check_restrict ('rawExp.comment', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'max', 1.0);
 $WHERE = check_ordering ('', $WHERE);
 
@@ -73 +76 @@
 
 // get the result table count
-$sql = "SELECT count(*) FROM camRun, chipRun, rawExp $WHERE";
+$sql = "SELECT count(*) FROM camProcessedExp, camRun, chipRun, rawExp $WHERE";
 
 $qry = $db->query($sql);
 if (dberror($qry)) {
-  echo "<b>error reading camRun, chipRun, rawExp table count</b><br>\n";
+  echo "<b>error reading camProcessedExp, camRun, chipRun, rawExp table count</b><br>\n";
   echo "<br><small><b> count query : $sql </b></small><br>\n";
   menu_end();
 }
 if (!$qry->fetchInto($row)) {
-  echo "<b>error reading camRun, chipRun, rawExp table count</b><br>\n";
+  echo "<b>error reading camProcessedExp, camRun, chipRun, rawExp table count</b><br>\n";
   echo "<br><small><b> count query : $sql </b></small><br>\n";
   menu_end();
@@ -128 +131 @@
 $buttonLink = button_restrict_max ('rawExp.bg_stdev', $buttonLink);
 $buttonLink = button_restrict_string ('rawExp.comment', $buttonLink);
-
+$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major', $buttonLink);
+$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major', $buttonLink);
+$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major', $buttonLink);
 // set up the table
 echo "<table class=list>\n";
@@ -181 +186 @@
 write_header_cell ("list", "stdev    ");
 write_header_cell ("list", "Date/Time");
+write_header_cell ("list", "Cam FWHM_major");
 echo "</tr>\n";
 
@@ -193 +199 @@
 write_query_row ('rawExp.bg_stdev', 5, 'min');
 write_query_row ('rawExp.dateobs', 19, 'min');
+write_query_row ('camProcessedExp.fwhm_major', 5, 'min');
 echo "</tr><tr><td>&le;</td>\n";
 echo "<td> &nbsp; </td>\n";
@@ -201 +208 @@
 write_query_row ('rawExp.bg_stdev', 5, 'max');
 write_query_row ('rawExp.dateobs', 19, 'max');
+write_query_row ('camProcessedExp.fwhm_major', 5, 'max');
 echo "</tr>\n";
 
@@ -228 +236 @@
 
 // query the database
-$sql = "SELECT rawExp.ra,rawExp.decl,rawExp.filter FROM camRun, chipRun, rawExp $WHERE";
+$sql = "SELECT rawExp.ra,rawExp.decl,rawExp.filter FROM camProcessedExp, camRun, chipRun, rawExp $WHERE";
 
 $qry = $db->query($sql);

branches/eam_branches/ipp-20120905/ippToPsps/jython/objectbatch.py

@@ -124 +124 @@
                    ," + filter[1] + "StackPSFFlux = FLUX_PSF \
                    ," + filter[1] + "StackPSFFluxErr = FLUX_PSF_ERR \
-                   ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) -8.9 \
+                   ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \
                    ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \
                    ," + filter[1] + "20pct = MAG_20/1000 \
                    ," + filter[1] + "80pct = MAG_80/1000 \
-                   "
+                   ," + filter[1] + "Flags = FLAGS "  
+                   
             
             self.scratchDb.execute(sql)
@@ -141 +142 @@
                    WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut)
             self.scratchDb.execute(sql)
+
+        self.logger.infoPair("Setting to NULL all x20pct =", "32.767" ) 
+        for filter in filters:
+            sql  = "UPDATE Object \
+                   SET " + filter[1] + "20pct = null \
+                   WHERE " + filter[1] + "20pct > 32.766"
+            self.scratchDb.execute(sql)
+        self.logger.infoPair("Setting to NULL all x80pct =", "32.767" )
+        for filter in filters:
+            sql  = "UPDATE Object \
+                    SET " + filter[1] + "80pct = null \
+                    WHERE " + filter[1] + "80pct > 32.766"
+            self.scratchDb.execute(sql)
+
+
+
 
     '''
@@ -187 +204 @@
                ,decErr \
                ,nDetections \
+               ,extra \
+               ,sgSep \
                ,dataRelease \
                ) \
@@ -200 +219 @@
                ,DEC_ERR \
                ,NMEASURE \
+               ,PSF_QF_PERF \
+               ,STARGAL_SEP \
                , " + str(self.config.dataRelease) + "\
                FROM " + cptTableName

branches/eam_branches/ipp-20120905/ippToPsps/jython/scratchdb.py

@@ -391 +391 @@
         except: 
             self.logger.error("Unable to create DVO detection database table")
-
+        self.makeColumnUnique("dvoDetection", "objID") 
 
     '''

branches/eam_branches/ipp-20120905/ippToPsps/jython/stackbatch.py

@@ -374 +374 @@
                ,PSF_THETA \
                ,psfLikelihood(EXT_NSIGMA) \
-               ,FLAGS << 32 | FLAGS2 \
+               ,FLAGS2 << 32 | FLAGS \
                ,PSF_QF \
                ,MOMENTS_XX \
@@ -389 +389 @@
                ,KRON_FLUX / " + str(self.expTime) + " \
                ,KRON_FLUX_ERR / " + str(self.expTime) + " \
-               , NULL \
+               , MOMENTS_R1 * 2.5 \
                , NULL \
                ,N_FRAMES \
@@ -395 +395 @@
                ," + self.historyModNum + " \
                FROM SkyChip_psf"
-
-        self.scratchDb.execute(sql)
+        self.scratchDb.execute(sql)
+        
         #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid
         #we drop those...
@@ -402 +402 @@
         self.scratchDb.execute(sql)
         self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
-
+        
         self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
+        
         self.scratchDb.updateFilterID("StackDetection", self.filter)
+        
         self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease))
+        
         self.scratchDb.updateAllRows("StackDetection", "primaryF", "0")
+        
         self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0")
+        
         self.updateStackMetaID("StackDetection")
+        
         self.updateStackTypeID("StackDetection")
+        
         self.updateDvoIDs("StackDetection")
+        
         sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
+        
         self.scratchDb.execute(sql)
   
@@ -422 +431 @@
                    AND a.ippDetectID = b.IPP_IDET  \
                    AND b.PSF_INST_FLUX_SIG IS NOT NULL"
+            
             self.scratchDb.execute(sql)
             
+        
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
+        
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
+        
         sql="DELETE FROM StackDetection where objID = 0"
-        self.scratchDb.execute(sql)
-        self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
-
+        
+        self.scratchDb.execute(sql)
+        self.logger.infoPair("Delleting", "entries with StackDetection.objID = 0")
+        
 
     '''
@@ -600 +614 @@
 
         self.logger.debug("Altering PSPS tables")
+        # heather uncommented the unique line -- well that no worky
         #self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
@@ -624 +639 @@
     '''
     def updateDvoIDs(self, table):
-
+        self.logger.infoPair("getting","imageID")
         imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+        self.logger.infoPair("obtained","imageID")
         self.logger.debug("Updating table '" + table + "' with DVO IDs...")
         sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
@@ -633 +649 @@
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.imageID = " + str(imageID)
-        self.scratchDb.execute(sql)
-
+        
+        self.scratchDb.execute(sql)
+        
     '''
     Does the processing, i.e. pulling stuff from IPP tables into PSPS tables

branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part0.sql

@@ -39 +39 @@
           WHERE 1
           @STACK1_QUERY@ -- input condition
-) AS diffExp ON diffExp.stack1 = stackRun.stack_id AND diffExp.stack2 = template.max_stack_id
+) AS diffExp ON diffExp.stack1 = stackRun.stack_id 
 WHERE
         stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0

branches/eam_branches/ipp-20120905/ippTools/share/difftool_definestackstack_part1.sql

@@ -40 +40 @@
           WHERE 1
           @STACK1_QUERY@ -- input constraint
-) AS diffExp ON diffExp.stack1 = stackRun.stack_id AND diffExp.stack2 = template.max_stack_id
+) AS diffExp ON diffExp.stack1 = stackRun.stack_id 
 WHERE
         stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0

branches/eam_branches/ipp-20120905/ippTools/src/difftoolConfig.c

@@ -351 +351 @@
     psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
     psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false);
-    psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
+    psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template and same template label", false);
     psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);

branches/eam_branches/ipp-20120905/ppTranslate/src/ppMopsWrite.c

@@ -38 +38 @@
 
   // Get these header words from the first input non null input
+  // but for SEEING which is the median of the (non-NULL) detections
+  // seeings
   ppMopsDetections *det = NULL;
-  for (int d = 0; d < detections->n && det == NULL; d++) {
-    det = detections->data[d];
+  int validSeeingCount = 0;
+  for (int d = 0; d < detections->n; d++) {
+    if (detections->data[d] != NULL) {
+      if (det == NULL) {
+        det = detections->data[d];
+      }
+      if (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) {
+        validSeeingCount += 1;
+      }
+    }
   }
   if (det != NULL) {
+    //Get the SEEING median
+    psVector* seeingValues = psVectorAlloc(validSeeingCount, PS_TYPE_F32);
+    psVectorInit(seeingValues, NAN);
+    validSeeingCount = 0;
+    for (int d = 0; d < detections->n; d++) {
+      if ( (detections->data[d] != NULL) && (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) ){
+        seeingValues->data.F32[validSeeingCount++] = ((ppMopsDetections*) detections->data[d])->seeing;
+      }
+    }
+    psStats *seeingStats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN);
+    psVectorStats(seeingStats, seeingValues, NULL, NULL, 0);
     psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd);
     psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight);
@@ -52 +73 @@
     psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter);
     psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass);
-    psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", det->seeing);
+    psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", seeingStats->sampleMedian);
   } else {
     psWarning("no inputs with surviving detections. output header will be incomplete");
@@ -112 +133 @@
       // Write data of version 2 (see ICD)
       psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET",         PS_DATA_U32, "IPP detection identifier index",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX",    PS_DATA_F32, "PSF fit instrumental magnitude",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG",           PS_DATA_F32, "magnitude in standard aperture",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW",       PS_DATA_F32, "magnitude in real aperture",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS",    PS_DATA_F32, "radius used for aperture mags",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX",          PS_DATA_F32, "instrumental flux in standard aperture",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX_SIG",      PS_DATA_F32, "aperture flux error",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG",      PS_DATA_F32, "PSF Magnitude using supplied calibration",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG",  PS_DATA_F32, "measured scatter of zero point calibration",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "SKY",              PS_DATA_F32, "Sky level",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA",        PS_DATA_F32, "Sigma of sky level",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF_PERFECT",   PS_DATA_F32, "PSF coverage/quality factor (poor)",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1",       PS_DATA_F32, "first radial moment",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH",       PS_DATA_F32, "half radial moment",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX",        PS_DATA_F32, "Kron Flux (in 2.5 R1)",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR",    PS_DATA_F32, "Kron Flux Error",
-                     NAN);
-      psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER",  PS_DATA_F32, "Kron Flux (in 1.0 R1)", 
-                     NAN);
+                     NAN);
+      psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER",  PS_DATA_F32, "Kron Flux (in 1.0 R1)",
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER",  PS_DATA_F32, "Kron Flux (in 4.0 R1)",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_P",         PS_DATA_F32, "distance to positive match source",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_P",        PS_DATA_F32, "signal-to-noise of pos match src",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_M",         PS_DATA_F32, "distance to negative match source",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_M",        PS_DATA_F32, "signal-to-noise of neg match src",
-                     NAN);
+                     NAN);
       psMetadataAdd (row, PS_LIST_TAIL, "FLAGS2",           PS_DATA_U32, "psphot analysis flags (group 2)",
-                     0);
-      psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES",         PS_DATA_U16, "Number of frames overlapping source center", 
-                     0);
-      psMetadataAdd (row, PS_LIST_TAIL, "PADDING",          PS_DATA_S16, "padding", 
-                     0);
+                     0);
+      psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES",         PS_DATA_U16, "Number of frames overlapping source center",
+                     0);
+      psMetadataAdd (row, PS_LIST_TAIL, "PADDING",          PS_DATA_S16, "padding",
+                     0);
       if (args->version == 3) {
-        // TODO: Write data of version 3 (see ICD)
+        // TODO: Write data of version 3 (see ICD)
       }
     }
@@ -179 +200 @@
   } else {
 
-#define addColumn(_outName, _inName, _convertTo32)                      \
+#define addColumn(_outName, _inName, _convertTo32)                      \
     if (!addOutputColumn(table, detections, total, _outName, _inName, _convertTo32)) { \
       psError(PS_ERR_UNKNOWN, false, "Failed to add column %s", _outName); \
-      return false;                                                     \
+      return false;                                                     \
     }
 
@@ -198 +219 @@
     addColumn("EXT_SIGNIFICANCE", "EXT_NSIGMA", 0);
     addColumn("PSF_MAJOR", NULL, 0);
-    addColumn("PSF_MINOR", NULL, 0);     
-    addColumn("PSF_THETA", NULL, 0);    
-    addColumn("PSF_QUALITY", "PSF_QF", 0); 
-    addColumn("PSF_NPIX", NULL, 1);        
+    addColumn("PSF_MINOR", NULL, 0);
+    addColumn("PSF_THETA", NULL, 0);
+    addColumn("PSF_QUALITY", "PSF_QF", 0);
+    addColumn("PSF_NPIX", NULL, 1);
     addColumn("MOMENTS_XX", NULL, 0);
     addColumn("MOMENTS_XY", NULL, 0);
@@ -287 +308 @@
       ppMopsDetections *det = detections->data[i];
       if (!det || det->num == 0) {
-        // no detections survived for this input
-        continue;
+        // no detections survived for this input
+        continue;
       }
       psVector *in = NULL;
       in = psMetadataLookupVector(NULL, det->table, inColumnName);
       if (!in) {
-        psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is true)", inColumnName);
-        return false;
+        psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is true)", inColumnName);
+        return false;
       }
       if (in->type.type != PS_TYPE_S64 && in->type.type != PS_TYPE_U64) {
-        psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d",
-                inColumnName, in->type.type);
-        return false;
+        psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d",
+                inColumnName, in->type.type);
+        return false;
       }
       if (out == NULL) {
-        // First time through set up the output vector and the copy parameters
-        if (in->type.type == PS_TYPE_S64) {
-          out = psVectorAlloc(outputSize, PS_TYPE_S32);
-        } else {
-          out = psVectorAlloc(outputSize, PS_TYPE_U32);
-        }
+        // First time through set up the output vector and the copy parameters
+        if (in->type.type == PS_TYPE_S64) {
+          out = psVectorAlloc(outputSize, PS_TYPE_S32);
+        } else {
+          out = psVectorAlloc(outputSize, PS_TYPE_U32);
+        }
       }
       for (long d=0; d < det->num; d++) {
-        if (in->type.type == PS_TYPE_S64) {
-          out->data.S32[next++] = in->data.S64[d];
-        } else {
-          out->data.U32[next++] = in->data.U64[d];
-        }
+        if (in->type.type == PS_TYPE_S64) {
+          out->data.S32[next++] = in->data.S64[d];
+        } else {
+          out->data.U32[next++] = in->data.U64[d];
+        }
       }
     }
@@ -323 +344 @@
       ppMopsDetections *det = detections->data[i];
       if (!det || det->num == 0) {
-        // no detections survived for this input
-        continue;
+        // no detections survived for this input
+        continue;
       }
       psVector *in = NULL;
       in = psMetadataLookupVector(NULL, det->table, inColumnName);
       if (!in) {
-        psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is false)", inColumnName);
-        out = psVectorAlloc(outputSize, PS_TYPE_F32);
-        psVectorInit(out, NAN);
-        psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out);
-        return false;
+        psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is false)", inColumnName);
+        out = psVectorAlloc(outputSize, PS_TYPE_F32);
+        psVectorInit(out, NAN);
+        psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out);
+        return false;
       }
       if (out == NULL) {
-        // First time through set up the output vector and the copy parameters
-        out = psVectorAlloc(outputSize, in->type.type);
-        next = (void *) out->data.U8;
-        switch (in->type.type) {
-        case PS_TYPE_S8:
-          elementSize = sizeof(psS8);
-          break;
-        case PS_TYPE_U8:
-          elementSize = sizeof(psU8);
-          break;
-        case PS_TYPE_S16:
-          elementSize = sizeof(psS16);
-          break;
-        case PS_TYPE_U16:
-          elementSize = sizeof(psU16);
-          break;
-        case PS_TYPE_S32:
-          elementSize = sizeof(psS32);
-          break;
-        case PS_TYPE_U32:
-          elementSize = sizeof(psU32);
-          break;
-        case PS_TYPE_S64:
-          elementSize = sizeof(psS64);
-          break;
-        case PS_TYPE_U64:
-          elementSize = sizeof(psU64);
-          break;
-        case PS_TYPE_F32:
-          elementSize = sizeof(psF32);
-          break;
-        case PS_TYPE_F64:
-          elementSize = sizeof(psF64);
-          break;
-        default:
-          psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type);
-          return false;
-        }
-      }
-      // We are doing nasty things here so we can use memcpy. 
+        // First time through set up the output vector and the copy parameters
+        out = psVectorAlloc(outputSize, in->type.type);
+        next = (void *) out->data.U8;
+        switch (in->type.type) {
+        case PS_TYPE_S8:
+          elementSize = sizeof(psS8);
+          break;
+        case PS_TYPE_U8:
+          elementSize = sizeof(psU8);
+          break;
+        case PS_TYPE_S16:
+          elementSize = sizeof(psS16);
+          break;
+        case PS_TYPE_U16:
+          elementSize = sizeof(psU16);
+          break;
+        case PS_TYPE_S32:
+          elementSize = sizeof(psS32);
+          break;
+        case PS_TYPE_U32:
+          elementSize = sizeof(psU32);
+          break;
+        case PS_TYPE_S64:
+          elementSize = sizeof(psS64);
+          break;
+        case PS_TYPE_U64:
+          elementSize = sizeof(psU64);
+          break;
+        case PS_TYPE_F32:
+          elementSize = sizeof(psF32);
+          break;
+        case PS_TYPE_F64:
+          elementSize = sizeof(psF64);
+          break;
+        default:
+          psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type);
+          return false;
+        }
+      }
+      // We are doing nasty things here so we can use memcpy.
       // It would be safer to do a proper loop over the elements.
       long toCopy = det->num * elementSize;

branches/eam_branches/ipp-20120905/pstamp/scripts/dquery_finish.pl

@@ -128 +128 @@
     my $job_id = $job->{job_id};
 
+    if ($job->{fault}) {
+        # how do we get this information back to the user?
+        print STDERR "job: $job_id faulted with $job->{fault}\n";
+        next;
+    }
     my $outputBase = $job->{outputBase};
     my $results = "${outputBase}results.txt";
@@ -166 +171 @@
     my $results = shift;
 
-    open IN, "<$results_file" or my_die("failed to open $results_file", $PS_EXIT_UNKNOWN_ERROR);
+    open IN, "<$results_file" or my_die("failed to open $results_file\n", $PS_EXIT_UNKNOWN_ERROR);
 
     foreach my $line (<IN>) {

branches/eam_branches/ipp-20120905/pstamp/scripts/psmkreq

@@ -52 +52 @@
 my $comment;
 
+my $survey_name = 'null';
+my $release = 'null';
+my $stack_type = 'null';
+my $bgrestore;
+my $jpeg;
+my $nocompress;
+
 my $missing_tools;
 my $pstamp_request_file  = can_run('pstamp_request_file')  or (warn "Can't find required program pstamp_request_file"  and $missing_tools = 1);
@@ -68 +75 @@
     'width=i'           => \$width,
     'height=i'          => \$height,
-    'pixcenter'         => \$pixcenter,
+# pixcenter is deprecated
+#    'pixcenter'         => \$pixcenter,
     'arcseconds'        => \$arcseconds,
     'coord_mask=i'      => \$coord_mask,
@@ -87 +95 @@
     'comment=s'         => \$comment,
 
+    # new columns for version 2
+    'survey=s'          => \$survey_name,
+    'release=s'         => \$release,
+    'stack_type'        => \$stack_type,
+
     'option_mask=i'     => \$option_mask,
     'image'             => \$image,
     'mask'              => \$mask,
     'variance'          => \$variance,
-    'cmf'               => \$cmf,
+    'sources'           => \$cmf,
     'psf'               => \$psf,
     'backmdl'           => \$backmdl,
@@ -99 +112 @@
     'do_not_wait'       => \$no_wait,
 
+    # new options for version 2
+    'no_compress'       => \$nocompress,
+    'bgrestore'         => \$bgrestore,
+
     'verbose'           => \$verbose,
     'save-temps'        => \$save_temps,
@@ -110 +127 @@
 
 pod2usage( -msg => "Invalid job_type: $job_type", -exitval =>1 )
-        unless ($job_type eq 'stamp' or $job_type eq 'get_image');
-
-if ($job_type eq 'stamp') {
+        unless ($job_type eq 'stamp' or $job_type eq 'get_image' or $job_type eq 'listtargets');
+
+if (($job_type eq 'stamp') or ($job_type eq 'listtargets')) {
     if (defined $list) {
         pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 )
@@ -152 +169 @@
 $id = 0 if !$id;
 
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
-    die "$stage is not a valid value for stage\n";
+
+unless (($job_type eq 'listtargets' and $stage eq 'exposure') or $stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+    die "$stage is not a valid stage for $job_type jobs\n";
 }
 
@@ -207 +225 @@
 } else {
     $rows = [];
-    push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max);
+    push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max, $survey_name, $release, $stack_type);
 }
 
 my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps);
-print $tdf "$req_name 1\n";
+print $tdf "$req_name 2\n";
 my $rownum = 0;
 foreach my $row (@$rows) {
     $rownum++;
     my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height"
-        . " $coord_mask $job_type $option_mask $project $req_type"
+        . " $coord_mask $job_type $option_mask $project $row->{survey_name} $row->{release} $req_type"
         . " $stage $id $tess_id $component $data_group"
-        . " $row->{filter} $row->{mjd_min} $row->{mjd_max}";
+        . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $row->{stack_type}";
 
     if ($row->{comment} and $row->{comment} ne '') {
@@ -230 +248 @@
 }
 close $tdf;
+
+system "cat $table_def_name";
+#exit 0;
 
 {
@@ -277 +298 @@
     $row->{dec}     = checkDEC($vals[1], $linenumber);
     $row->{filter}  = checkFilter($vals[2], $filter, $linenumber);
-    $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);;
-    $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);;
+    $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);
+    $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);
+    $row->{survey_name}  = $vals[5];
+    $row->{release} = $vals[6];
+    $row->{stack_type} = $vals[7];
     $row->{comment} = $comment;
 

branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_checkdependent.pl

@@ -137 +137 @@
         print "skipping $command\n";
     }
-} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) {
+} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'goto_cleaned') or ($it->{state} eq 'update')) {
     #       For warp and diff stages we need to call the 'queue_update' subroutines even if the 
     #       data_state is update in order to check the state of inputs in earlier stages in the pipeline
@@ -260 +260 @@
             # caller will fault the jobs
             return $error_code;
+        } elsif ($chip->{state} eq 'goto_cleaned') {
+
+            if ($chip->{data_state} eq 'full') {
+                # cleanup has been queued, but has not yet finished. (cleanup pantasks is probably stopped)
+                # Since this chip is in full go ahead and set the run to state update and change the label.
+                # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks
+                # queue things may get confused but we can live with that
+                my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+                $command .= " -set_label $rlabel" if $rlabel;
+
+                if (!$no_update) {
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                                run(command => $command, verbose => $verbose);
+                    unless ($success) {
+                        my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                    }
+                } else {
+                    print "skipping $command\n";
+                }
+                $queued_update = 1;
+            }
         } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
 
@@ -344 +365 @@
     my $skycell = $metadata;
 
+    if ($skycell->{state} eq 'goto_cleaned' and $skycell->{data_state} eq 'full') {
+        # cleanup has been queued, but hasn't been done. it's "safe" to set the state back to full
+        # XXX: seem comments in check_states_chip
+        my $command = "$warptool -updaterun -set_state update -warp_id $warp_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+        return 0;
+    }
+
     # get the list of input chips for this skycell
     my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
@@ -428 +467 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
+
+    if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') {
+        print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n";
+        my $command = "$difftool -updaterun -set_state update -diff_id $diff_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+        return 0;
+    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_finish.pl

@@ -107 +107 @@
     my ($header, $rows) = read_request_file($req_file);
 
-    if (!$header or !$rows) {
-        # Since a request got queued, the request file must have been readable at some
-        # point 
+    my $action = $header->{ACTION};
+
+    if (!$header or (($action eq 'PROCESS') and !$rows)) {
+        # Since a request got queued, the request file must have been readable at some point 
         my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
     }
@@ -126 +127 @@
     }
 
-    # The results table definition file
-    my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
-
-    # data for the header
-    print $tdf "$req_name|$req_id|\n";
-
-    # get the list of jobs generated for this request
-    my @jobs;
-    {
-        my $command = "$pstamptool -listjob -req_id $req_id";
-        $command   .= " -dbname $dbname" if $dbname;
-        $command   .= " -dbserver $dbserver" if $dbserver;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            die("Unable to perform $command error code: $error_code");
-        }
-        my $output = join "", @$stdout_buf;
-        if (!$output) {
-            # This should not happen. A fake job should have been entered
-            my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
-        } else {
-            my $jobs = parse_md_fast($mdcParser, $output);
-
-            @jobs = @$jobs;
-        }
-    }
-
-    my $exp_info;
-    foreach my $job (@jobs) {
-        my $job_id = $job->{job_id};
-        my $job_type = $job->{jobType};
-        my $rownum = $job->{rownum};
-        my $fault = $job->{fault};
-        my $exp_id = $job->{exp_id};
-
-        my $error_string = get_error_string($fault);
-
-        if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
-            # this request had a duplicate request name yet the parser didn't give
-            # it an "ERROR style name.
-            my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
-        }
-        my ($row, $req_info, $project) = get_request_info($rows, $rownum);
-
-        my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-        my $image_db = $proj_hash->{dbname};
-        if (!$image_db and !$fault) {
-            # if project isn't resolvable, the paser should have faulted this job
-            my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
-        }
-
-        my $job_params = get_job_parameters($job);
-        my $stage = "";
-        if ($job_params) {
-            $stage = $job_params->{stage};
-        }
-
-        if ($stage ne 'stack') {
-            # get the metadata for the exposure (if any i.e. stack) 
-            # returns an appropriate string if !$exp_id
-            $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
-        } else {
-            my $filter = $job_params->{filter};
-            $filter = "0" if !$filter;
-            my $mjd_obs = $job_params->{mjd_obs};
-            $mjd_obs = "0" if !$mjd_obs;
-            $exp_info = "$mjd_obs|0|0|$filter|0|0";
-        }
-
-        if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
-            my $jreglist = "$outdir/reglist$job_id";
-            if (open JRL, "<$jreglist") {;
-                # process the reglist file to get the list of files produced by this job
-                foreach my $line (<JRL>) {
-
-                    # XXX: we are getting many cases where the size and/or md5sum calculated by
-                    # the job has changed by the time the request_finish has run
-                    # Don't
-                    # add line to the requests's reglist
-                    # ....
-#                    print $rlf $line;
-
-
-                    chomp $line;
-                    my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
-                    # ... instead let dsreg compute the paramters by leaving them blank
-                    print $rlf "$img_name|||$filetype|\n";
-
-                    # add line to the table definition file
-                    print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
-
-                    # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # first assume that the image is compressed and check the first extension.
-                    # If not found check the PHU. If that doesn't work just set them to zero.
-                    # XXX do this more cleanly
-                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
-                    if (!defined $ra_deg) {
-                        (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+    my $request_fault = 0;
+    if ($action eq 'PROCESS') {
+        # The results table definition file
+        my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
+        # data for the header
+        print $tdf "$req_name|$req_id|\n";
+        # get the list of jobs generated for this request
+        my @jobs;
+        {
+            my $command = "$pstamptool -listjob -req_id $req_id";
+            $command   .= " -dbname $dbname" if $dbname;
+            $command   .= " -dbserver $dbserver" if $dbserver;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                die("Unable to perform $command error code: $error_code");
+            }
+            my $output = join "", @$stdout_buf;
+            if (!$output) {
+                # This should not happen. A fake job should have been entered
+                my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
+            } else {
+                my $jobs = parse_md_fast($mdcParser, $output);
+
+                @jobs = @$jobs;
+            }
+        }
+
+        my $exp_info;
+        foreach my $job (@jobs) {
+            my $job_id = $job->{job_id};
+            my $job_type = $job->{jobType};
+            my $rownum = $job->{rownum};
+            my $fault = $job->{fault};
+            my $exp_id = $job->{exp_id};
+
+            my $error_string = get_error_string($fault);
+
+            if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
+                # this request had a duplicate request name yet the parser didn't give
+                # it an "ERROR style name.
+                my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
+            }
+            my ($row, $req_info, $project) = get_request_info($rows, $rownum);
+
+            my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+            my $image_db = $proj_hash->{dbname};
+            if (!$image_db and !$fault) {
+                # if project isn't resolvable, the paser should have faulted this job
+                my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
+            }
+
+            my $job_params = get_job_parameters($job);
+            my $stage = "";
+            if ($job_params) {
+                $stage = $job_params->{stage};
+            }
+
+            if ($stage ne 'stack') {
+                # get the metadata for the exposure (if any i.e. stack) 
+                # returns an appropriate string if !$exp_id
+                $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
+            } else {
+                my $filter = $job_params->{filter};
+                $filter = "0" if !$filter;
+                my $mjd_obs = $job_params->{mjd_obs};
+                $mjd_obs = "0" if !$mjd_obs;
+                $exp_info = "$mjd_obs|0|0|$filter|0|0";
+            }
+
+            if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
+                my $jreglist = "$outdir/reglist$job_id";
+                if (open JRL, "<$jreglist") {;
+                    # process the reglist file to get the list of files produced by this job
+                    foreach my $line (<JRL>) {
+
+                        # XXX: we are getting many cases where the size and/or md5sum calculated by
+                        # the job has changed by the time the request_finish has run
+                        # Don't
+                        # add line to the requests's reglist
+                        # ....
+    #                    print $rlf $line;
+
+
+                        chomp $line;
+                        my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
+                        # ... instead let dsreg compute the paramters by leaving them blank
+                        print $rlf "$img_name|||$filetype|\n";
+
+                        # add line to the table definition file
+                        print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
+
+                        # ra_deg and dec_deg are the coordinates of center of the stamp
+                        # first assume that the image is compressed and check the first extension.
+                        # If not found check the PHU. If that doesn't work just set them to zero.
+                        # XXX do this more cleanly
+                        my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+                        if (!defined $ra_deg) {
+                            (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+                        }
+                        $ra_deg = 0.0 if (!$ra_deg);
+                        $dec_deg = 0.0 if (!$dec_deg);
+                        print $tdf "$ra_deg|$dec_deg|";
+
+                        print $tdf "$exp_info|";
+                        print $tdf "$req_info|";
+                        print $tdf "\n";
                     }
-                    $ra_deg = 0.0 if (!$ra_deg);
-                    $dec_deg = 0.0 if (!$dec_deg);
-                    print $tdf "$ra_deg|$dec_deg|";
-
+                    close JRL;
+                } else {
+                    my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
+                        if $fault eq $PSTAMP_SUCCESS;
+                    print STDERR "no reglist file for job $job_id\n" if $verbose;
+                    print $tdf "$rownum|$fault|$error_string|0|$job_id|";
+                    print $tdf "0|0|";       # center of (non-existent) stamp
                     print $tdf "$exp_info|";
                     print $tdf "$req_info|";
                     print $tdf "\n";
                 }
-                close JRL;
             } else {
-                my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
-                    if $fault eq $PSTAMP_SUCCESS;
-                print STDERR "no reglist file for job $job_id\n" if $verbose;
-                print $tdf "$rownum|$fault|$error_string|0|$job_id|";
-                print $tdf "0|0|";       # center of (non-existent) stamp
-                print $tdf "$exp_info|";
-                print $tdf "$req_info|";
-                print $tdf "\n";
-            }
-        } else {
-            # XXX do list jobs
-            # we can probably arange things to use the code as above and skip the fileset registration
-            print STDERR "Unknown jobType: $job_type";
-            next;
-        }
-    }
-    close $rlf;
-    close $tdf;
-    my $request_fault = 0;
-    # make the results file
-    {
-        my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            print STDERR "Unable to perform $command error code: $error_code\n";
-            $request_fault = $error_code >> 8;
-        } else {
-            # dump a textual representation
-            my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+                # XXX do list jobs
+                # we can probably arange things to use the code as above and skip the fileset registration
+                print STDERR "Unknown jobType: $job_type";
+                next;
+            }
+        }
+        close $tdf;
+        # make the results file
+        {
+            my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
             unless ($success) {
+                print STDERR "Unable to perform $command error code: $error_code\n";
                 $request_fault = $error_code >> 8;
-                my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
-            }
-        }
-    }
+            } else {
+                # dump a textual representation
+                my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $request_fault = $error_code >> 8;
+                    my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
+                }
+            }
+        }
+    } elsif ($action ne 'LIST') {
+        my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
+    } else {
+        # pstampparse did all of the work
+    }
+
+    close $rlf;
+
     if (!$request_fault) {
         # register the fileset

branches/eam_branches/ipp-20120905/pstamp/scripts/pstamp_request_file

@@ -49 +49 @@
                     value => undef
         },
+        { name => 'ACTION',
+                    writetype => TSTRING,
+                    comment => 'Action to perform: LIST or PROCESS',
+                    value => undef
+        },
+        { name => 'USERNAME',
+                    writetype => TSTRING,
+                    comment => 'username for request (optional)',
+                    value => undef
+        },
+        { name => 'EMAIL',
+                    writetype => TSTRING,
+                    comment => 'email for submittor (optional)',
+                    value => undef
+        },
 ];
 
 # Specification of columns to write
 my $columns = [ 
+        { name => 'ROWNUM',     type => 'J',   writetype => TULONG }, 
+
+        { name => 'CENTER_X',   type => 'D',  writetype => TDOUBLE },
+        { name => 'CENTER_Y',   type => 'D',  writetype => TDOUBLE },
+        { name => 'WIDTH',      type => 'D',  writetype => TDOUBLE },
+        { name => 'HEIGHT',     type => 'D',  writetype => TDOUBLE },
+        # 2 bits in COORD_MASK indicate what units of roi coords are
+        { name => 'COORD_MASK', type => 'J',  writetype => TULONG },
+
+        { name => 'JOB_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'OPTION_MASK',type => 'J',   writetype => TULONG },
+
+        # image selection parameters
+        { name => 'PROJECT',    type => '16A', writetype => TSTRING },
+        # new in version 2
+        { name => 'SURVEY_NAME',type => '16A', writetype => TSTRING },
+        { name => 'IPP_RELEASE', type => '16A', writetype => TSTRING },
+        # end of new in version 2
+        { name => 'REQ_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'IMG_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'ID',         type => '16A', writetype => TSTRING },           
+        { name => 'TESS_ID',    type => '64A', writetype => TSTRING },
+        { name => 'COMPONENT',  type => '64A', writetype => TSTRING },
+
+        { name => 'DATA_GROUP ',type => '64A', writetype => TSTRING },
+
+        { name => 'REQFILT',    type => '16A', writetype => TSTRING },
+        { name => 'MJD_MIN',    type => 'D',   writetype => TDOUBLE },
+        { name => 'MJD_MAX',    type => 'D',   writetype => TDOUBLE },
+        { name => 'RUN_TYPE',   type => '16A', writetype => TSTRING },
+
+        { name => 'COMMENT ',   type => '64A', writetype => TSTRING },
+];
+my $columns_v1 = [ 
         { name => 'ROWNUM',     type => 'J',   writetype => TULONG }, 
 
@@ -89 +138 @@
 }
 
+# build array of arrays for the column data. Note if the input file is version 1 then
+# some of thes will get dropped
 my @colData;
 foreach (@$columns) {
@@ -96 +147 @@
 
 my $minimum_cols = 6;
-my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols); 
+my $extver = 2;
+my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols, \$extver); 
 if (!$numRows) {
     print STDERR "no data in $input\n";
@@ -115 +167 @@
 $output = $req_name . ".fits" if !$output;
 
-my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);
+my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $extver > 1 ? $columns : $columns_v1, $header);
 
 exit $status;
@@ -230 +282 @@
     my $header  = shift;    # rerence to array of header keyword descriptions
     my $minimum_required_vals = shift;
+    my $r_extver = shift;   # reference to extension version
 
     my $line_num = 0;
@@ -243 +296 @@
             my @vals = split /$sep/, $line;
             my $nvals = @vals;
-            die "number of header columns in input $nvals does not equal expected number of header words $nhead"
+            die "two few columns $nvals in header definition. Require at least 2" if @vals < 2;
+
+            $$r_extver = $vals[1];
+            if ($extver > 1) {
+                die "number of header columns in input $nvals does not equal expected number of header words $nhead"
                     if (@vals != @$header);
-
-            for (my $i=0; $i < @$header; $i++) {
-                $header->[$i]->{value} = $vals[$i];
-            }
-
-            last; # only one header line
-        }
+            } else {
+                # drop header keywords not used in version 1
+                for (my $i = @$header - 1; $i >= $nvals; $i--) {
+                    pop @$header;
+                }
+            }
+
+            for (my $i=0; $i < $nvals; $i++) {
+                my $val = $vals[$i];
+                if (defined $val) {
+                    $header->[$i]->{value} = $val;
+                } else {
+                    $header->[$i]->{value} = 'null';
+                }
+            }
+
+            last; # one line defines the header
+        }
+    }
+    if ($$r_extver == 1) {
+        # pop off the colData arrays to account for the unused columns
+        pop @colData;
+        pop @colData;
+        pop @colData;
     }
 
@@ -271 +345 @@
             exit 1;
         }
+        # trim leading whitespace from comment
+        $comment =~ s/^\s+//;
         if ($comment) {
             if ($comment =~ /#/) {
@@ -381 +457 @@
     print "If --output is omitted the output file name is set to REQ_NAME.fits\n\n";
     print "Header 1 Line.  Format:\n\n";
-    print "  REQ_NAME EXTVER\n\n";
+    print "REQ_NAME EXTVER ACTION USERNAME EMAIL\n\n";
     print "REQUEST specification (1 or more lines). Format:\n\n";
-    print "  ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX | COMMENT\n\n";
+    print "ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE | COMMENT\n\n";
 
     exit 0;

branches/eam_branches/ipp-20120905/pstamp/scripts/pstampparse.pl

@@ -93 +93 @@
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -102 +102 @@
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
@@ -109 +109 @@
 my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
 my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
 
 
@@ -230 +230 @@
     # the images of interest for this new row doesn't match the list. 
     # process the list ...
-    $num_jobs += processRows(\@rowList);
+    $num_jobs += processRows($action, \@rowList);
 
     # and reset the list to contain just the new row
@@ -238 +238 @@
 # out of rows process the list
 if (scalar @rowList > 0) {
-    $num_jobs += processRows(\@rowList);
-}
-
-if (($mode eq "queue_job") and ($num_jobs eq 0)) {
+    $num_jobs += processRows($action, \@rowList);
+}
+
+if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
     print STDERR "no jobs created for $req_name\n" if $verbose;
     insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
@@ -387 +387 @@
 }
 
-sub processRows {
+sub list_targets {
     my $rowList = shift;
     my $num_jobs = 0;
+
+    $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRows {
+    my $action = shift;
+    my $rowList = shift;
+    my $num_jobs = 0;
+
+    if ($action eq 'LIST') {
+        return list_targets($rowList);
+    }
 
     # all rows in the list are compatible