Changeset 39224

Timestamp:

Dec 3, 2015, 4:14:24 PM (11 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20151113

Files:

• . (modified) (1 prop)
• Nebulous/nebclient (modified) (1 prop)
• Ohana (modified) (1 prop)
• Ohana/src/libohana/test (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (6 diffs)
• ippScripts/scripts/addstar_multi_run.pl (modified) (4 diffs)
• ippScripts/scripts/addstar_run.pl (modified) (2 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (1 diff)
• ippTasks/addstar.pro (modified) (1 diff)
• ippToPsps (modified) (1 prop)
• ippToPsps/config/bundle.pl (copied) (copied from trunk/ippToPsps/config/bundle.pl )
• ippToPsps/config/schema_browser.vot (modified) (1 diff)
• ippToPsps/config/tables.IN.vot (modified) (1 diff)
• ippToPsps/jython/batch.py (modified) (1 diff)
• ippToPsps/jython/cleanup.py (modified) (1 diff)
• ippToPsps/jython/diffbatch.py (modified) (5 diffs)
• ippToPsps/jython/dvo.py (modified) (8 diffs)
• ippToPsps/jython/dvoobjects.py (modified) (1 diff)
• ippToPsps/jython/forcedgalaxybatch.py (modified) (5 diffs)
• ippToPsps/jython/gpc1db.py (modified) (8 diffs)
• ippToPsps/jython/ipptopspsdb.py (modified) (8 diffs)
• ippToPsps/jython/loader.py (modified) (1 diff)
• ippToPsps/jython/queue.py (modified) (3 diffs)
• ippToPsps/jython/scratchdb.py (modified) (2 diffs)
• ippToPsps/jython/skychunk.py (modified) (1 diff)
• ippToPsps/test (modified) (1 prop)
• ippToPsps/test/fulltest.sh (modified) (1 diff)
• ippToPsps/test/gpc1schema.sql (modified) (1 diff)
• ippToPsps/test/mkgpc1data.dvo (modified) (1 diff)
• ippToPsps/test/sascheck.v0.sql (deleted)
• ippToPsps/test/sascheck.v1.sql (deleted)
• ippToPsps/test/sascheck.v2.sql (deleted)
• ippToPsps/test/sascheck.v3.sql (deleted)
• ippToPsps/test/sascheck.v4.sql (deleted)
• ippToPsps/test/sascheck.v5.sql (deleted)
• ippToPsps/test/sascheck.v6.sql (deleted)
• ippToPsps/test/sascheck.v7.sql (deleted)
• ippToPsps/test/sascheck.v8.sql (deleted)
• ippToPsps/test/sascheck.v9.sql (deleted)
• ippToPsps/test/testscripts (copied) (copied from trunk/ippToPsps/test/testscripts )
• ippTools/src/pstamptool.c (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ippTools/src/pxtools.h (modified) (1 diff)
• ippconfig/recipes/fitstypes.mdc (modified) (1 diff)
• ippconfig/tc3/format_ps2.config (modified) (1 diff)
• ippconfig/tc3/format_tc1.config (modified) (6 diffs)
• ppNoiseMap (modified) (1 prop)
• ppSim (modified) (1 prop)
• psLib (modified) (1 prop)
• psModules (modified) (1 prop)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 diff)
• tools/bills/adddeltacolumns (modified) (4 diffs)
• tools/bills/comparep2detections (modified) (1 diff)
• tools/bills/comparestackdetections (modified) (6 diffs)
• tools/bills/concatsmf (modified) (1 diff)
• tools/bills/joinstackcmf (modified) (4 diffs)
• tools/dvodb_calib/HDFV2calib/run.dvolens (modified) (1 diff)
• tools/dvodb_calib/HDFV2calib/run.relphot (modified) (1 diff)
• tools/dvodb_calib/SAS40calib (copied) (copied from trunk/tools/dvodb_calib/SAS40calib )
• tools/releasecalib/scripts/extractinfo (modified) (1 diff)

branches/eam_branches/ipp-20151113/Nebulous/nebclient

@@ -20 +20 @@
 nebclient.pc
 stamp-h1
+test-driver

@@ -20 +20 @@
 nebclient.pc
 stamp-h1
+test-driver

branches/eam_branches/ipp-20151113/Ohana/src/libohana/test

@@ -7 +7 @@
 string
 typetest
+sprintf_floats

@@ -7 +7 @@
 string
 typetest
+sprintf_floats

branches/eam_branches/ipp-20151113/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -746 +746 @@
         my $mask_base;
         if ($want_astrom) {
-            if ($selectedAstrom) {
+            if ($out->{cam_path_base}) {
+                # XXX Cheating here
+                my $astromSource = 'PSASTRO.OUTPUT';
+                $out->{astrom} = $ipprc->filename($astromSource, $out->{cam_path_base});
+            } elsif ($selectedAstrom) {
                 $out->{astrom} = $selectedAstrom;
                 $out->{cam_path_base} = $selectedAstrom;
@@ -763 +767 @@
                 }
             }
+            # XXX: cheating here
             # XXX: do this right by looking at the recipe
             $mask_base = $out->{cam_path_base};
@@ -1471 +1476 @@
     if (!$astromSource) {
         if ($camera eq "GPC1") {
-            # CHEATER !
+            # XXX: CHEATER !
             $astromSource = "PSASTRO.OUTPUT";
         } else {
@@ -1489 +1494 @@
     }
 
-    # XXX: Is this code correct if ASTROM.SOURCE ne "PSASTRO.OUTPUT"
+    # XXX: Is this code correct if ASTROM.SOURCE ne "PSASTRO.OUTPUT" ?
     my $astromFile = $ipprc->filename($astromSource, $camRoot);
     if (!$astromFile) {
@@ -1939 +1944 @@
     my $now = gettimeofday();
     my $dtime_tool = $now - $start_tool;
-    print "Time to run $program: $dtime_tool\n";
+    # XXX: shouldn't this be if $verbose?
+    print "Time to run $program: $dtime_tool\n" ; # if $verbose;
 
     my $buf = join "", @$stdout_buf;
@@ -1953 +1959 @@
 
     my $dtime_parse = gettimeofday() - $start_parse;
-    print "Time to parse results from $program: $dtime_parse\n";
+    # XXX: shouldn't this be if $verbose?
+    print "Time to parse results from $program: $dtime_parse\n" ; # if $verbose;
 
     return $results;

branches/eam_branches/ipp-20151113/ippScripts/scripts/addstar_multi_run.pl

@@ -39 +39 @@
 my $minidvodb_path;
 
-my ( $label,$camera, $stage, $stage_id, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
+my ( $label,$camera, $stage, $stage_id, $use_diff_inv, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
      $no_op, $redirect, $save_temps);
 GetOptions(
@@ -46 +46 @@
     'stage|s=s'        => \$stage, # Camera
     'stage_id|w=s'   => \$stage_id,
+    'use_diff_inv'   => \$use_diff_inv, #use inv images for diff cmf? 
     'dbname|d=s'        => \$dbname, # Database name
     'outroot|w=s'       => \$outroot, # output file base name
@@ -137 +138 @@
             print "there are $num_components\n";
             foreach my $comp  (@$components) {
-            my  $mparsed = $comp->{stageroot};
+                my  $mparsed = $comp->{stageroot};
             
-            if (!defined($mparsed)) {
-                &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
-            } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
-            print "found a value for stageroot:$mparsed\n";
-            my $fpaObjects = $mparsed.'.cmf';
-            my $realFile = $ipprc->file_resolve($fpaObjects);
-            if (!defined($realFile)) {
-                &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
+                if (!defined($mparsed)) {
+                    &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
+                } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
+                print "found a value for stageroot:$mparsed\n";
+                my $fpaObjects = $mparsed.'.cmf';
+                my $realFile = $ipprc->file_resolve($fpaObjects);
+                if (!defined($realFile)) {
+                    &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
+                }
+                printf TEMPLIST "$realFile = $fpaObjects\n";
+                if ($stage =~ /diff/) { # hardwired  for now$use_diff_inv) {
+                    print "finding the inv.cmf files\n";
+                    my $fpaObjectsInv = $mparsed.'.inv.cmf';
+                    my $realFileInv = $ipprc->file_resolve($fpaObjectsInv);
+                    if (!defined($realFileInv)) {
+                        &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
+                    }
+                    printf TEMPLIST "$realFileInv = $fpaObjectsInv\n";
+                }
+                
             }
-            printf TEMPLIST "$realFile = $fpaObjects\n";
-            }
-    }
-
-}
-
+
+    }
+}
 close(TEMPLIST);
 print "saved $temp_file here\n";
@@ -238 +248 @@
             if ($stage =~ /diff/) {
                 $command .= " -accept-astrom";
+                if ($use_diff_inv) {
+                    $command .= " -diff-inv";
+                }
             }
+            
             if ($stage =~ /fullforce/) {
                 $command .= " -accept-astrom -xrad"; 

branches/eam_branches/ipp-20151113/ippScripts/scripts/addstar_run.pl

@@ -388 +388 @@
                     my $ffSummary = $metadata->[0];
                     my $filter = $ffSummary->{filter};
-                    my $stack_id = $ffSummary->{stack_id};
+                    my $stage_id = $ffSummary->{stage_id};
                     
                     $filter =~ s/\.00000//;
@@ -399 +399 @@
                 $command .= " $realFile ";
                 $command .= " -p $photcode ";
-                $command .= " -image-id $stack_id ";
+                $command .= " -image-id $stage_id ";
             }
                     

branches/eam_branches/ipp-20151113/ippScripts/scripts/warp_skycell.pl

@@ -223 +223 @@
 my $wrote_astrom = 0;
 foreach my $imfile (@$imfiles) {
-    my $image = $imfile->{uri}; # Image name
+    my $image = $ipprc->filename ("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); # Image name
     my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
 

branches/eam_branches/ipp-20151113/ippTasks/addstar.pro

@@ -483 +483 @@
     end
     if ("$STAGE" == "diff")
-      $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
+      #$run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
+        $run = $run --use_diff_inv --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 
     end
     if ("$STAGE" == "fullforce")

branches/eam_branches/ipp-20151113/ippToPsps

@@ -14 +14 @@
 install-sh
 Doxyfile
+compile

@@ -14 +14 @@
 install-sh
 Doxyfile
+compile

branches/eam_branches/ipp-20151113/ippToPsps/config/schema_browser.vot

@@ -123 +123 @@
         <TABLE name="ForcedMeanObject"> </TABLE>
      </RESOURCE>
+
+     <RESOURCE name="ForcedGalaxyModelView">
+        <TABLE name="ObjectThin"> </TABLE>
+        <TABLE name="ForcedGalaxyShape"> </TABLE>
+     </RESOURCE>
+
   </RESOURCE>
 

branches/eam_branches/ipp-20151113/ippToPsps/config/tables.IN.vot

@@ -862 +862 @@
 
      <TABLE name="ForcedGalaxyShapeFlags">
-      <DESCRIPTION>Contains information flag values that define ForcedGalaxyShape chisq surface fit failuers.  </DESCRIPTION>
+      <DESCRIPTION>Contains information flag values that define ForcedGalaxyShape chisq surface fit failures.</DESCRIPTION>
       <PARAM arraysize="1" datatype="char" ucd="meta.bib.author" name="Author" value="PSPS"></PARAM>
       <FIELD name="name" arraysize="100" datatype="char">

branches/eam_branches/ipp-20151113/ippToPsps/jython/batch.py

@@ -104 +104 @@
 
         # create DVO tables (scratch database)
-        self.scratchDb.createDvoTables()
+        ### XXX self.scratchDb.createDvoTables() <-- remove this
 
         # dump stuff to the log

branches/eam_branches/ipp-20151113/ippToPsps/jython/cleanup.py

@@ -50 +50 @@
             self.clean("FW")
             self.clean("FO")
+            self.clean("FG")
 
             self.checkClientStatus()

branches/eam_branches/ipp-20151113/ippToPsps/jython/diffbatch.py

@@ -141 +141 @@
 
            # look up the path base once
-           pathBase = gpc1Db.getDiffStagePathBase(skychunk.dvoLabel, int(diffSkyFileID))
+           pathBase = gpc1Db.getDiffStagePathBase(int(diffSkyFileID))
 
            # select the two (possible) cmf files (.cmf and .inv.cmf)
@@ -336 +336 @@
         else:
             fluxScaleStr = "-1"
-            detIDoffset = self.posImageDetCount
+            detIDoffset = self.posImageDetCount[num]
 
         if isPosFile or self.haveNegFits[num]:
@@ -723 +723 @@
         self.haveSrcPos = {}
         self.haveNegFits = {}
-        self.posImageDetCount = 0
+        self.posImageDetCount = {}
 
         for num in self.number:
@@ -742 +742 @@
             ## if we have a negative image (inv.cmf), load that as well:
             self.haveNegFits[num] = 0
+            self.posImageDetCount[num] = 0
             if not self.negFits[num] == None:
                 fileNameNeg = self.negFits[num].getPath()
@@ -759 +760 @@
 
                 posTableName = "SkyChip_psf_" + str(num)
-                self.posImageDetCount = self.scratchDb.getRowCount(posTableName)
+                # self.posImageDetCount[num] = self.scratchDb.getMaxIPP_IDET(posTableName)
+                self.posImageDetCount[num] = self.scratchDb.getRowCount(posTableName)
 
             ## use stilts to get a list of the tables from the header

branches/eam_branches/ipp-20151113/ippToPsps/jython/dvo.py

@@ -72 +72 @@
             #     response = raw_input("(y/n) ")
             #     if response == "y":
-            self.resetAllTables()
+            self.resetAllTables() ## called if we need to change DVOs
 
     '''
@@ -95 +95 @@
         self.correctDvo = True
         if not self.loadSkyTable():
+            self.correctDvo = False
+            return
+        if not self.loadPhotcodes():
+            self.correctDvo = False
+            return
+        if not self.loadImages():
             self.correctDvo = False
             return
@@ -298 +304 @@
             return False        
 
-        # update the two meta tables, Images and SkyTable
-        self.loadSkyTable()
+        # update the SkyTable
+        self.loadSkyTable() ## in setSkyArea
 
         # set bounding-box coords
@@ -508 +514 @@
     def sync(self):
 
-        # go no further if we've already partly ingested a different DVO
-        if not self.correctDvo: return False
-
-        # check we have something to do
-        if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
-            self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
-            return True
-
-        # make sure we have an up-to-date Images table
-        self.loadImages()
-
-        # purge stuff
-        self.purgeRegions(self.regionsToPurge)
-
-        totalRegions = len(self.regionsToIngest)
-        if totalRegions > 0: self.logger.infoSeparator()
-
-        # now loop through all regions and ingest to MySQL
-        ingestedRegionsCount = 0
-        regionCount = 1
-        for region in self.regionsToIngest:
-
-           self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
-           if self.ingestRegion(region): ingestedRegionsCount += 1
-           regionCount += 1
-
-        self.logger.infoSeparator()
-        self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
-
-        # update lists after attempted sync
-        self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
-        self.printSummary()
+        print "---- sync is disabled ----"
+        os._exit(3)
+
+        ### XXX # go no further if we've already partly ingested a different DVO
+        ### XXX if not self.correctDvo: return False
+        ### XXX 
+        ### XXX # check we have something to do
+        ### XXX if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
+        ### XXX     self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
+        ### XXX     return True
+        ### XXX 
+        ### XXX # make sure we have an up-to-date Images table
+        ### XXX self.loadImages()
+        ### XXX 
+        ### XXX # purge stuff
+        ### XXX self.purgeRegions(self.regionsToPurge)
+        ### XXX 
+        ### XXX totalRegions = len(self.regionsToIngest)
+        ### XXX if totalRegions > 0: self.logger.infoSeparator()
+        ### XXX 
+        ### XXX # now loop through all regions and ingest to MySQL
+        ### XXX ingestedRegionsCount = 0
+        ### XXX regionCount = 1
+        ### XXX for region in self.regionsToIngest:
+        ### XXX 
+        ### XXX    self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
+        ### XXX    if self.ingestRegion(region): ingestedRegionsCount += 1
+        ### XXX    regionCount += 1
+        ### XXX 
+        ### XXX self.logger.infoSeparator()
+        ### XXX self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
+        ### XXX 
+        ### XXX # update lists after attempted sync
+        ### XXX self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
+        ### XXX self.printSummary()
 
         return True
@@ -667 +676 @@
     includes purging detections outside sky area
     '''
-    def nativeIngestDetections(self, boxId, boxDim):
+    def nativeIngestDetections(self, boxId, boxDim, myBatchType):
 
         # clear the 'ingested' field for all boxes owned by this host
@@ -688 +697 @@
 
         # make sure we have an up-to-date Images table
-        self.loadImages()
+        self.loadImages() # this is in nativeIngestDetections (remove?)
 
         # the box dimensions are the area used to select the items of
@@ -695 +704 @@
         # to grab all detections / objects which are contributed by an
         # item with the center in the given box
-        dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC']))
-        dD_border = self.skychunk.BORDER
-        
-        #This is a terrible hack but I'm tired of it taking a decade to ingest the FW skycells.
-
-        #if self.batchType == "ST":
-        #    dR_border = .4 / math.cos(math.radians(boxDim['DEC']))
-        #    dD_border = .4
-        #if self.batchType == "FW":
-        #    dR_border = .4 / math.cos(math.radians(boxDim['DEC']))
-        #    dD_border = .4
-
-
+
+        # P2 and DF need to use a full GPC1 exposure radius:
+        myBorder = self.skychunk.BORDER_GPC1
+
+        # ST and FW need to use just a skycell
+        if myBatchType == "ST" :
+            myBorder = self.skychunk.BORDER_SKYCELL
+        if myBatchType == "FW" :
+            myBorder = self.skychunk.BORDER_SKYCELL
+
+        dR_border = myBorder / math.cos(math.radians(boxDim['DEC']))
+        dD_border = myBorder
 
         minRA  = boxDim['minRA']  - dR_border
@@ -788 +796 @@
             else:
                 cmd += " -photcode-start 12000 -photcode-end 12500"
-        ## if (useFW == 1):
-        ##     if (self.config.camera == "simtest"):
-        ##         print "need photcodes for non gpc1 camera"
-        ##     else:
-        ##         print "need photcode for warp for gpc1 camera, default is no selection, should work, could be slow"
+
+        if (useST + useP2 + useDF + useFW > 1):
+            self.logger.errorPair("multiple batch types in a run", "NOW ILLEGAL")
+            os._exit(5)
                 
         ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?

branches/eam_branches/ipp-20151113/ippToPsps/jython/dvoobjects.py

@@ -28 +28 @@
         # declare DVO file types of interest
         self.ingestFileTypes = ['cpt', 'cps']
-        self.loadPhotcodes()
+        self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches
 
     '''

branches/eam_branches/ipp-20151113/ippToPsps/jython/forcedgalaxybatch.py

@@ -72 +72 @@
 
     '''
+    Generate ippObjID and make it an index 
+    '''
+    def addIppObjID (self, tableName):
+        sql = "ALTER TABLE " + tableName + " ADD COLUMN ippObjID BIGINT"
+        self.scratchDb.execute(sql)
+        
+        sql = "CREATE INDEX ippObjID_index ON " + tableName + " ( ippObjID ) ";
+        self.scratchDb.execute(sql)
+
+        sql = "UPDATE " + tableName + " set ippObjID = (OBJ_ID + (CAT_ID << 32))"
+        self.scratchDb.execute(sql)
+
+    '''
     Overriden from batch base class to import directly from DVO for forced galaxy shape parameters
     '''
@@ -101 +114 @@
             self.logger.infoPair("0 rows for ","cpq table")
             return False
+
+        # add ippObjID columns and index them:
+        self.addIppObjID (cptTableName)
+        self.addIppObjID (cpqTableName)
+
         return True
 
@@ -178 +196 @@
             # sqlLine.group("a." + filter[1] + "GalDecErr",       "b.DEC_ERR")
             
-            sql = sqlLine.makeEquals("WHERE a.ippObjID = (b.OBJ_ID + (b.CAT_ID << 32)) AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode))
+            sql = sqlLine.makeEquals("WHERE a.ippObjID = b.ippObjID AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode))
             
             self.logger.debugPair("cpq sql: ", sql)
             
             self.scratchDb.execute(sql)
+
+            ## # check on the status as we go:
+            ## sql = "select count(*) as N, galModelType, nFilter, count(*)*nFilter as S from ForcedGalaxyShape group by galModelType, nFilter"
+            ## rs = self.scratchDb.executeQuery(sql)  
+            ## 
+            ## while rs.next():
+            ##     N = rs.getInt(1)
+            ##     M = rs.getInt(2)
+            ##     F = rs.getInt(3)
+            ##     S = rs.getInt(4)
+            ##     
+            ##     print "N: %d, TYPE: %d, nFilter: %d, Sum: %d" % (N, M, F, S)
 
     '''
@@ -207 +237 @@
 
             sqlLine.group("objID",             "EXT_ID")
-            sqlLine.group("ippObjID",          "OBJ_ID + (CAT_ID << 32)")
+            sqlLine.group("ippObjID",          "ippObjID")
             sqlLine.group("surveyID",          str(self.surveyID))
             sqlLine.group("randomForcedGalID", "RAND("+str(self.batchID)+")")
@@ -215 +245 @@
             sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName)
 
-            print "--- sql: " + sql
+            # print "--- sql: " + sql
 
             self.scratchDb.execute(sql)

branches/eam_branches/ipp-20151113/ippToPsps/jython/gpc1db.py

@@ -79 +79 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full' \
-                   AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
-                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \
+                   AND decl >= RADIANS(" + str(minDec) + ") AND decl < RADIANS(" + str(maxDec) + ") \
+                   AND ra >= RADIANS(" + str(minRA) + ") AND ra < RADIANS(" + str(maxRA) + ") \
                    AND dateobs >= '" + tstart + "'\
                    AND dateobs <= '" + tend  + "'"
@@ -103 +103 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full' \
-                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
-                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
+                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
                    AND mjd_obs >= (to_days('" + tstart + "')-678941) \
                    AND mjd_obs <= (to_days('" + tend + "') - 678941) "
@@ -121 +121 @@
                    JOIN mergedvodbProcessed using (merge_id) \
                    JOIN diffInputSkyfile \
-                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
-                   AND stage = 'diff') \
+                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \
                    JOIN skycell using (skycell_id, tess_id) \
                    JOIN warpRun on (warp1 = warp_id \
@@ -137 +136 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full' \
-                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
-                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
+                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
                    AND dateobs >= '" + tstart + "'\
                    AND dateobs <= '" + tend  + "'\
@@ -164 +163 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full' \
-                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
-                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
+                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
                    AND mjd_obs >= (to_days('" + tstart + "')-678941) \
                    AND mjd_obs <= (to_days('" + tend + "') - 678941) \
@@ -308 +307 @@
                    JOIN mergedvodbProcessed using (merge_id) \
                    JOIN diffInputSkyfile \
-                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
-                   AND stage = 'diff') \
+                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \
                    JOIN skycell using (skycell_id, tess_id) \
                    JOIN warpRun on (warp1 = warp_id \
@@ -513 +511 @@
     get diff stage cmf name
     '''
-    def getDiffStagePathBase(self, dvoDb, diffSkyfileID):
+    def getDiffStagePathBase(self, diffSkyfileID):
+        sql = "SELECT DISTINCT diffSkyfile.path_base \
+               FROM diffInputSkyfile \
+               JOIN diffSkyfile using (diff_id, skycell_id) \
+               WHERE diff_skyfile_id = " + str(diffSkyfileID)
+
+        rs = self.executeQuery(sql)
+
+        if not rs.first():
+            self.logger.errorPair("missing cmf entry diff_skyfile_id:", str(diff_skyfile_id))
+            # self.logger.errorPair("no diff cmfs found for diff_skyfile_id = %d" %diff_skyfile_id)
+            return None
+
+        pathBase = rs.getString(1)
+        return pathBase
+
+    '''
+    get diff stage cmf name
+    XXX EAM 20151119 : this function does too much work: you have a diffSkyfileID, that matches to a unique path_base
+    '''
+    def getDiffStagePathBaseOldVersion(self, dvoDb, diffSkyfileID):
         print "got here, diffstagecmf"
         sql = "SELECT DISTINCT diffSkyfile.path_base \
                FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \
                JOIN addRun using (minidvodb_name) \
-               JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \
-               AND diff_id = stage_id AND stage = 'diff') \
-               JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \
+               JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 AND diff_id = stage_id AND stage = 'diff') \
+               JOIN diffSkyfile using (diff_id, skycell_id) \
                WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \
                AND minidvodbRun.state = 'merged' \
@@ -561 +578 @@
     '''
     get diff stage cmf name
+    XXX EAM 20151119 This function is broken AND it does too much work
     '''
     def getDiffStageCmf(self, dvoDb, diffSkyfileID):

branches/eam_branches/ipp-20151113/ippToPsps/jython/ipptopspsdb.py

@@ -324 +324 @@
                AND processed = 1 \
                AND loaded_to_datastore = 1 \
-               AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()"
+               AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()"
 
         total = -1
@@ -421 +421 @@
                AND timestamp > '" + self.skychunk.epoch + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
-               AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()"
+               AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()"
 
         try:
@@ -545 +545 @@
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
                AND batch_type != 'IN' \
-               AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()"
+               AND timestamp >= now() - INTERVAL 12 HOUR AND timestamp < now()"
 
         try:
@@ -1158 +1158 @@
 
         self.skychunk.boxSize = rs.getDouble(10)
-        # self.skychunk.halfBox = self.skychunk.boxSize/2.0
-        # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
 
         self.skychunk.basePath = rs.getString(11)
@@ -1240 +1238 @@
         sql = "SELECT id FROM box \
                WHERE skychunk  = '" + self.skychunk.name + "' \
-               AND ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
-               AND dec_center BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
+               AND ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
+               AND dec_center >= " + str(minDEC) + " AND dec_center < " + str(maxDEC)
 
                # AND box_side = " + str(self.skychunk.boxSize)
@@ -1424 +1422 @@
                FROM box \
                JOIN pending ON (id = box_id) \
-               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+               WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
                AND batch_type = '" + batchType + "' \
                AND skychunk = '" + self.skychunk.name + "' \
@@ -1503 +1501 @@
                FROM box \
                JOIN pending ON (id = box_id) \
-               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+               WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
                AND skychunk = '" + self.skychunk.name + "' \
                ORDER BY dec_center DESC"
@@ -1712 +1710 @@
         sql = "SELECT DISTINCT stage_id \
                FROM all_pending \
-               WHERE ra_bore BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
-               AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
+               WHERE ra_bore >= " + str(minRA) + " AND ra_bore < " + str(maxRA) + " \
+               AND dec_bore >= " + str(minDEC) + " AND dec_bore < " + str(maxDEC)
 
         try:

branches/eam_branches/ipp-20151113/ippToPsps/jython/loader.py

@@ -172 +172 @@
                     if (batchType == "P2" or batchType == "ST" or batchType =="DF" or batchType == "FW"):
                         if self.config.resetDvo or not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
-                            if not self.dvo.nativeIngestDetections(boxId, boxDim):
+                            if not self.dvo.nativeIngestDetections(boxId, boxDim, batchType):
                                 self.logger.error("Unable to ingest detections with nativeIngest")
                                 return False

branches/eam_branches/ipp-20151113/ippToPsps/jython/queue.py

@@ -146 +146 @@
 
                     # XXX not sure why this is not done with a for-loop...
+                    # answer: because python does not have a good float-style for-loop
                     while dec <= self.skychunk.maxDec:
                         while ra <= self.skychunk.maxRa:
@@ -158 +159 @@
             
                            ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC)
-    
+
                            self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
                                        ra, 
@@ -165 +166 @@
                                        len(ids)))
                 
-                           if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
-                           # print "inserted pending"
+                           if len(ids) > 0: 
+                               ## for myID in ids:
+                               ##     print "id: %d : %10.6f - %10.6f | %10.6f - %10.6f" % (myID, minRA, maxRA, minDEC, maxDEC)
+
+                               self.ippToPspsDb.insertPending(box_id, batchType, ids)
+                               # print "inserted pending"
+
                            ra = ra + 2*dR
                            # print "new ra: ", ra

branches/eam_branches/ipp-20151113/ippToPsps/jython/scratchdb.py

@@ -406 +406 @@
 
         return size
+
+    '''
+    Returns a row count for this table
+    '''
+    def getMaxIPP_IDET(self, table):
+
+        sql = "SELECT MAX(IPP_IDET) FROM " + table
+        rs = self.executeQuery(sql)  
+
+        if not rs.first():
+            self.logger.exception("Could not count rows for table: '" + table + "'")
+            return 0
+
+        return rs.getInt(1)
 
     '''
@@ -655 +669 @@
     def resetAllDvoTables(self):
 
-       # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
-       self.logger.infoPair("Dropping table", self.dvoImagesTable)
-       self.dropTable(self.dvoImagesTable) 
-       self.logger.infoPair("Dropping table", self.dvoSkyTable)
-       self.dropTable(self.dvoSkyTable) 
-       self.logger.infoPair("Dropping table", self.dvoDoneTable)
-       self.dropTable(self.dvoDoneTable) 
-
-       # blow away existing dvoDetection table & re-crate
-       if not self.resetDvoDetectionTable():
-           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
-           return False
-
-       # create dvoDone table
-       self.logger.infoPair("Creating table", self.dvoDoneTable)
-       sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
-       try: self.execute(sql)
-       except: 
-           self.logger.errorPair("Unable to create table", self.dvoDoneTable)
-           return False
-
-       self.changeEngineToInnoDB(self.dvoDoneTable)
-
-       return True
+        # re-create the definition tables
+        self.createDvoTables()
+
+        # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
+        self.logger.infoPair("Dropping table", self.dvoImagesTable)
+        self.dropTable(self.dvoImagesTable) 
+        self.logger.infoPair("Dropping table", self.dvoSkyTable)
+        self.dropTable(self.dvoSkyTable) 
+        self.logger.infoPair("Dropping table", self.dvoDoneTable)
+        self.dropTable(self.dvoDoneTable) 
+
+        # blow away existing dvoDetection table & re-crate
+        if not self.resetDvoDetectionTable():
+            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
+            return False
+
+        # create dvoDone table
+        self.logger.infoPair("Creating table", self.dvoDoneTable)
+        sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
+        try: 
+            self.execute(sql)
+        except: 
+            self.logger.errorPair("Unable to create table", self.dvoDoneTable)
+            return False
+
+        self.changeEngineToInnoDB(self.dvoDoneTable)
+            
+        return True
 
     '''

branches/eam_branches/ipp-20151113/ippToPsps/jython/skychunk.py

@@ -39 +39 @@
         # XXX this is a poor place to put this information -- it is completely gpc-specific.  
         # probably should go elsewhere.  This number is scaled in loader.py by cos(dec)
-        self.BORDER = 1.65
+        self.BORDER_GPC1    = 1.65
+        self.BORDER_SKYCELL = 0.40
         self.isLoaded = False
 

branches/eam_branches/ipp-20151113/ippToPsps/test

@@ -8 +8 @@
 stackbatch.dat
 objectbatch.dat
+diffbatch.dat
+diffobjbatch.dat
+forcebatch.dat
+forcedobjectbatch.dat

@@ -8 +8 @@
 stackbatch.dat
 objectbatch.dat
+diffbatch.dat
+diffobjbatch.dat
+forcebatch.dat
+forcedobjectbatch.dat

branches/eam_branches/ipp-20151113/ippToPsps/test/fulltest.sh

@@ -561 +561 @@
 
   rm -rf $OUTDIR/FO
+  rm -rf $OUTDIR/FG
 
   ippjython loader.py $options $forcedobjqueuename once -simtest < /dev/null

branches/eam_branches/ipp-20151113/ippToPsps/test/gpc1schema.sql

@@ -21 +21 @@
   CREATE TABLE diffRun             ( diff_id INT, tess_id CHAR(64), state CHAR(16), diff_mode INT, software_ver CHAR(16) );
   CREATE TABLE diffInputSkyfile    ( diff_id INT, warp1 INT, stack2 INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64) );
-  CREATE TABLE diffSkyfile         ( diff_id INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64), path_base CHAR(255) );
+  CREATE TABLE diffSkyfile         ( diff_id INT, skycell_id CHAR(64), path_base CHAR(255) );
 
   CREATE TABLE addRun              ( add_id INT, stage CHAR(64), stage_extra1 INT, state CHAR(64), stage_id INT, minidvodb_name CHAR(64) );

branches/eam_branches/ipp-20151113/ippToPsps/test/mkgpc1data.dvo

@@ -654 +654 @@
   dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M")
   dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID,  $SKYCELL )
-  dbinsert diffSkyfile (diff_id, diff_skyfile_id, tess_id, skycell_id, path_base) values ($difID, $diffSkyfileID, $TESS_ID, $SKYCELL, "$filename")
+  dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, $SKYCELL, "$filename")
 end
 

branches/eam_branches/ipp-20151113/ippTools/src/pstamptool.c

@@ -817 +817 @@
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",  "state", "==");
     PXOPT_COPY_STR(config->args, where, "-jobType", "jobType", "==");
     PXOPT_COPY_S64(config->args, where, "-fault",  "fault", "==");

branches/eam_branches/ipp-20151113/ippTools/src/pstamptoolConfig.c

@@ -163 +163 @@
     psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0,           "select by fault", 0);
     psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-jobType", 0,         "select by jobType", 0);
+    psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-state", 0,           "select by job state", 0);
     psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,          "limit result set to N items", 0);
     psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0,         "use the simple output format", false);

branches/eam_branches/ipp-20151113/ippTools/src/pxtools.h

@@ -206 +206 @@
 
 #define PXOPT_COPY_PRIMITIVE(from, to, type, suffix, oldname, newname, newcomment) \
-{ \
-    psMetadataItem *item = psMetadataLookup(from, oldname); \
-    if (!item) { \
+    {                                                                   \
+      psMetadataItem *item = psMetadataLookup(from, oldname);           \
+      if (!item) {                                                      \
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", oldname); \
-        return false; \
-    } \
-    psAssert(item->comment, "metadata item comment should be defined"); \
-\
-    if (psStrcasestr(item->comment, "(found)")) { \
+        return false;                                                   \
+      }                                                                 \
+      psAssert(item->comment, "metadata item comment should be defined"); \
+                                                                        \
+      if (psStrcasestr(item->comment, "(found)")) {                     \
         if (!psMetadataAdd##suffix(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, newcomment, item->data.suffix)) { \
-            psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \
-            psFree(to); \
-            return false; \
-        } \
-    } \
-}
+          psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \
+          psFree(to);                                                   \
+          return false;                                                 \
+        }                                                               \
+      }                                                                 \
+    }
 
 #define PXOPT_COPY_V(from, to, type, suffix, oldname, newname, comment) \

branches/eam_branches/ipp-20151113/ippconfig/recipes/fitstypes.mdc

@@ -81 +81 @@
         NOISE           S32     8
 END
+
 # Compressed mask
 COMP_MASK       METADATA

branches/eam_branches/ipp-20151113/ippconfig/tc3/format_ps2.config

@@ -3 +3 @@
 # How to recognise this type
 RULE    METADATA
-#        IMAGESWV        STR     GPC DaqCom v2.05 (Mar 08 2007)
+#       IMAGESWV        STR     GPC DaqCom v2.05 (Mar 08 2007)
         ORIGIN          STR     PS2
         CONTROLR        STR     STARGRASP

branches/eam_branches/ipp-20151113/ippconfig/tc3/format_tc1.config

@@ -99 +99 @@
 END             
 
+# Specify the cell data 
+# NOTE: A duplicate metadata entry will override the first entry
+CELLS   METADATA
+END
+
 # Specify the cell data
 CELLS   METADATA
-        pitch10u        METADATA
+        # if the test camera is generating BIASEC,DATASEC, use this:
+        tc1raw  METADATA
                 CELL.BIASSEC.SOURCE     STR     HEADER
                 CELL.TRIMSEC.SOURCE     STR     HEADER
@@ -107 +113 @@
                 CELL.TRIMSEC            STR     DATASEC
         END
-END
-
-# Specify the cell data
-CELLS   METADATA
-        tc1raw  METADATA
+
+        # if the test camera is NOT generating BIASEC,DATASEC, use this:
+        tc1old  METADATA
                 CELL.BIASSEC.SOURCE     STR     VALUE
                 CELL.TRIMSEC.SOURCE     STR     VALUE
@@ -121 +125 @@
 # How to translate PS concepts into FITS headers
 TRANSLATION     METADATA
-        FPA.FILTERID    STR     FILTERID
-        FPA.FILTER      STR     FILTERID
+#       FPA.FILTERID    STR     FILTERID
+#       FPA.FILTER      STR     FILTERID
         FPA.RA          STR     RA
         FPA.DEC         STR     DEC
@@ -134 +138 @@
         FPA.ROTANGLE    STR     ROT
         FPA.FOCUS       STR     M2Z
-        FPA.TIME        STR     SHUTOUTC
+#       FPA.TIME        STR     SHUTOUTC
         FPA.EXPOSURE    STR     EXPREQ          # Requested exposure time, presumably camera exposure time
         FPA.ALT         STR     ALT
@@ -180 +184 @@
         CELL.EXPOSURE   STR     EXPTIME         # Exposure time measured by shutter
         CELL.DARKTIME   STR     DARKTIME        # Exposure time for camera
-        CELL.TIME       STR     SHUTOUTC        # Observation time
+#       CELL.TIME       STR     SHUTOUTC        # Observation time
         # XXX CELL.GAIN       STR     GAIN              # Cell gain (e/ADU)
         CELL.READNOISE  STR     RDNOISE         # Cell read noise (ADU)
@@ -208 +212 @@
         # XXX these should come from the header, but not for now
         CELL.GAIN       F32     1.0             # Cell gain (e/ADU)
-        CELL.SATURATION F32     40000           # Saturation point (ADU)
+        CELL.SATURATION F32     64000           # Saturation point (ADU)
 
         ### How to get FPA coordinates from chip coordinates

branches/eam_branches/ipp-20151113/ppNoiseMap

@@ -15 +15 @@
 install-sh
 a.out.dSYM
+compile

@@ -15 +15 @@
 install-sh
 a.out.dSYM
+compile

branches/eam_branches/ipp-20151113/ppSim

@@ -15 +15 @@
 test
 a.out.dSYM
+compile

@@ -15 +15 @@
 test
 a.out.dSYM
+compile

branches/eam_branches/ipp-20151113/psLib

@@ -35 +35 @@
 *.da
 a.out.dSYM
+test-driver

@@ -35 +35 @@
 *.da
 a.out.dSYM
+test-driver

branches/eam_branches/ipp-20151113/psModules

@@ -29 +29 @@
 psmodules-*.tar.*
 a.out.dSYM
+test-driver

@@ -29 +29 @@
 psmodules-*.tar.*
 a.out.dSYM
+test-driver

branches/eam_branches/ipp-20151113/pstamp/scripts/pstamp_checkdependent.pl

@@ -480 +480 @@
         # XXX: change tools to include cam_state (camRun.state). If it is not full the warp updates will never run.
         my $cam_fault = $chip->{cam_fault};
-        if (($chip->{camState} eq 'cleaned') or defined $cam_fault and $cam_fault > 0) {
+        if ($chip->{camState} eq 'cleaned') {
+            print STDERR "camRun for $warp_id skycell_id $skycell_id is cleaned so warp cannot be updated.\n";
+            return $PSTAMP_GONE;
+        }
+        if (defined $cam_fault and $cam_fault > 0) {
             print STDERR "camRun for $warp_id skycell_id $skycell_id is faulted: $cam_fault\n";
             return $PSTAMP_GONE;

branches/eam_branches/ipp-20151113/tools/bills/adddeltacolumns

@@ -152 +152 @@
 xPosErr         X_PSF_SIG
 yPosErr         Y_PSF_SIG
-pltScale        PLTSCALE
-posAngle        POSANGLE
+pltScale        PLTSCALE_i
+posAngle        POSANGLE_i
 ra
 dec
@@ -163 +163 @@
 expTime
 airMass
-psfFlux         PSF_INST_FLUX           *zpFactor/expTime/PSF_NDOF
+psfFlux         PSF_INST_FLUX           *zpFactor/expTime
 psfFluxErr      PSF_INST_FLUX_SIG       *zpFactor/expTime
 psfMajorFWHM    PSF_FWHM_MAJ            *abs(PLTSCALE)
@@ -171 +171 @@
 psfQf           PSF_QF
 psfQfPerfect    PSF_QF_PERFECT
-psfChiSq        PSF_CHISQ
+psfChiSq        PSF_CHISQ               /PSF_NDOF
 psfLikelihood                 # actually psfLikelihood(EXT_NSIGMA). We assume the formula is ok.
 momentXX        MOMENTS_XX              *PLTSCALE*PLTSCALE
@@ -189 +189 @@
 kronFluxErr     KRON_FLUX_ERR           *zpFactor/expTime
 kronRad         MOMENTS_R1              *2.5*abs(PLTSCALE)
-sky             SKY                     /PLTSCALE/PLTSCALE*zpFactor/expTime
+sky             SKY_i                   /PLTSCALE/PLTSCALE*zpFactor/expTime
 skyErr          SKY_SIGMA               /PLTSCALE/PLTSCALE*zpFactor/expTime
 infoFlag        FLAGS

branches/eam_branches/ipp-20151113/tools/bills/comparep2detections

@@ -84 +84 @@
     die "failed to find smf for $exp_name in sasdata\n" unless $smf;
 } else {
-    # XXX: look up the smf for the exposure here
-    # die "only test mode works right now\n";
-    my $cmd = "releasetool -listrelexp -release_name $release -exp_id $exp_id | grep path_base | awk '{print \$3}'";
+    my $cmd = "releasetool -listrelexp -release_name $release -exp_id $exp_id | grep cam_path_base | awk '{print \$3}'";
     print "$cmd\n" if $verbose;
     my $path_base = `$cmd`;

branches/eam_branches/ipp-20151113/tools/bills/comparestackdetections

@@ -22 +22 @@
 my $testdata;
 
-my ($checkdeltas, $verbose, $save_temps);
+my ($checkdeltas, $just_join, $verbose, $save_temps);
 
 my $label = 'SAS.20141118.sc8';
@@ -30 +30 @@
     "testdata=s"        =>  \$testdata,
     "join=s"            =>  \$joinType,
+    "just-join"         =>  \$just_join,
     "check"             =>  \$checkdeltas,
     "verbose|v"         =>  \$verbose,
@@ -90 +91 @@
     $command .= " --test" if $testdata;
     $command .= " --verbose" if $verbose;
+    # FITS has a limit of 999 for the number of columns in a table
+    # If we're adding the delta columns that's too many. So skip the
+    # joins (we heed to run joinstackcmf to add some columns that we need)
+    $command .= " --skip-joins" unless $just_join;
 
     runcommand($command);
@@ -112 +117 @@
     $input = $outfile;
 }
+
+if ($just_join) {
+    rename ($outfile, $out) or die "failed to rename $outfile to $out\n";
+    exit 0;
+}
+
 
 my @deltas;
@@ -141 +152 @@
 
 if ($checkdeltas) {
+    my ($fmt, $sep);
+    if ($out =~ /fits$/) {
+        $fmt = "-fits StackObjectAttributes";
+        $sep = "";
+    } elsif ($out =~ /csv$/) {
+        $fmt = "-csv";
+        $sep = ",";
+    } else {
+        die "don't know how to read $out. Use .fits or .csv\n";
+    }
     $save_temps = 1;
     my ($manafile, $mananame) = tempfile( "/var/tmp/manacmds.XXXX", UNLINK => !$save_temps);
@@ -172 +193 @@
 
     print $manafile "datafile $out\n";
-    print $manafile "read -fits StackObjectAttributes";
+    print $manafile "read $fmt";
+    my $first = 1;
     foreach my $col (@deltas) {
+        if ($sep and !$first) {
+            print $manafile $sep;
+        } else {
+            $first = 0;
+        }
         print $manafile " $col";
     }

branches/eam_branches/ipp-20151113/tools/bills/concatsmf

@@ -53 +53 @@
     $path = `neb-locate -p $smf`;
     if (!$path) {
-        die "ailed to find location of $smf\n";
+        die "failed to find location of $smf\n";
     }
     print $path if $verbose;

branches/eam_branches/ipp-20151113/tools/bills/joinstackcmf

@@ -23 +23 @@
 # for now just all or one
 
-my ($stack_id, $skycal_id);
+my ($stack_id, $skycal_id, $skip_joins);
 my $test;
 my ($save_temps, $verbose);
@@ -30 +30 @@
 GetOptions(
     "stack_id|s=s"      =>  \$stack_id,
+    "skip-joins"        =>  \$skip_joins,
     "test"              =>  \$test,
     "ofmt=s"            =>  \$ofmt,
@@ -83 +84 @@
 my $exthead = 'SkyChip';
 
-my $cmd = "$stilts tmatchn nin=5 matcher=exact values1=IPP_IDET values2=IPP_IDET values3=IPP_IDET"
+my $cmd;
+if ($skip_joins) {
+    $cmd = "$stilts tpipe in=$path cmd='"
+    .         "addcol -after IPP_IDET STACK_ID $stack_id;"
+    .         "addcol -after STACK_ID SKYCAL_ID $skycal_id;"
+    .         "addcol -after SKYCAL_ID PROJ_ID $proj;"
+    .         "addcol -after PROJ_ID CELL_ID $cell;"
+    . "'"
+    . " out=$output";
+} else {
+    $cmd = "$stilts tmatchn nin=5 matcher=exact values1=IPP_IDET values2=IPP_IDET values3=IPP_IDET"
     . " values4=IPP_IDET values5=IPP_IDET"
     . " fixcols=all suffix1='' suffix2=_exp suffix3=_dev suffix4=_ser suffix5='_xsrc'"
@@ -107 +118 @@
     . " progress=none"
     ;
+}
 
 $cmd .= " ofmt=$ofmt" if $ofmt;

branches/eam_branches/ipp-20151113/tools/dvodb_calib/HDFV2calib/run.dvolens

@@ -20 +20 @@
   set options = ""
   set options = "$options -update-objects"
-  set options = "$options -region 320 340 -10 10 "
+  set options = "$options -region 255 275 -33 -13 "
   set options = "$options -D CATDIR $catdir"
   set options = "$options -update"

branches/eam_branches/ipp-20151113/tools/dvodb_calib/HDFV2calib/run.relphot

@@ -34 +34 @@
   set options = "$options -max-density 3000"
   set options = "$options -cloud-limit 0.5"
-  set options = "$options -region 320 340 -10 10"
+#  set options = "$options -region 320 340 -10 10"
+  set options = "$options -region 255 275 -33 -13"
   set options = "$options -boundary-tree tess.3pi.fits"
   set options = "$options -D CATDIR $catdir"

branches/eam_branches/ipp-20151113/tools/releasecalib/scripts/extractinfo

@@ -65 +65 @@
     my $zpt = $clam - $mcal;
 
+    if ($name =~ ':') {
+        # semi colons in file name, must be a nebulous path split by ':' characters
+        # The smf name that we want is the last one
+        my @words = split ':', $name;
+        my $n = scalar @words;
+        die "cannot parse file name $name" unless $n;
+        $name = @words[$n-1];
+    }
     my ($exp_name, $exp_id, undef, $cam_id) = split '\.', $name;
 

@@ -20 +20 @@
 nebclient.pc
 stamp-h1
+test-driver

@@ -7 +7 @@
 string
 typetest
+sprintf_floats

@@ -14 +14 @@
 install-sh
 Doxyfile
+compile

@@ -8 +8 @@
 stackbatch.dat
 objectbatch.dat
+diffbatch.dat
+diffobjbatch.dat
+forcebatch.dat
+forcedobjectbatch.dat

@@ -15 +15 @@
 install-sh
 a.out.dSYM
+compile

@@ -15 +15 @@
 test
 a.out.dSYM
+compile

@@ -35 +35 @@
 *.da
 a.out.dSYM
+test-driver

@@ -29 +29 @@
 psmodules-*.tar.*
 a.out.dSYM
+test-driver