Changeset 39015 for trunk/ippToPsps/jython/diffbatch.py

Timestamp:

Oct 30, 2015, 5:01:15 PM (11 years ago)

Author:

watersc1

Message:

Updates to make DF batches work. Test suite changes to implement that. There are still a few issues with header keywords not being respected completely, but I'm not fully sure if that's due to the simulated data or not. Change to fulltest.sh to move dbuser/pass into variables that can be changed easily.

File:

• trunk/ippToPsps/jython/diffbatch.py (modified) (7 diffs)

trunk/ippToPsps/jython/diffbatch.py

@@ -114 +114 @@
 
            self.difftypeID[num] = meta[1];
-           self.ippDiffID[num] = stage_id
+           self.ippDiffID[num] = stageid
            self.posImageID[num] = meta[2];
            self.negImageID[num] = meta[3];
@@ -122 +122 @@
            ## MEH -- should be able to use diff header now for mjd (see below)
            #self.obsTime[num] = meta[7];
-           self.expID[num] = meta[8];
+           # tess_id is handled below, that's #8.
+           self.expID[num] = meta[9];
            self.logger.infoPair("loading diff stage cmf", diffSkyFileID)
            self.fits[num] = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, int(diffSkyFileID))
@@ -160 +161 @@
            self.dropTableVerbose("DiffToImage_"+str(num))
            self.dropTableVerbose("DiffDetection_"+str(num))
+           self.dropTableVerbose("DiffDetEffMeta_"+str(num))
 
            if not self.analysisVer[num]: self.analysisVer[num] = -999
@@ -440 +442 @@
 
 
+    ''' 
+    Populate DiffDetEffMeta
+    '''
+    def populateDiffDetEffMeta(self,num):
+        diffSkyFileID = self.number[num]
+        self.logger.infoPair("Processing table", "DiffDetEffMeta")
+
+        if diffSkyFileID <= 0:
+            self.logger.infoPair("no diff data for number", num)
+            return False
+
+        cmfTable = "SkyChip_deteff_" + str(num)
+        if not cmfTable in self.tablesLoaded:
+            self.logger.infoPair("no deteff data for diff", diffSkyFileID)
+            self.logger.infoPair("(did not find table in tablesLoaded)", cmfTable)
+            for name in self.tablesLoaded:
+                print "-- table: " + name
+            os._exit(3)
+            return False
+
+        self.logger.infoPair("Found cmfTable:", cmfTable)
+
+        ippTableName = "DiffDetEffMeta_" + str(num)
+        sql = "CREATE TABLE " + ippTableName + " LIKE DiffDetEffMeta"
+        try: self.scratchDb.execute(sql)
+        except: pass
+
+        self.logger.infoPair("Created tmp table:", ippTableName)
+
+        header = self.header[num]
+        # CZW: this is broken, and can't get the zeropoint
+        #        zpImage = self.scratchDb.getImageZeroPoint(diffSkyFileID)
+        zpImage = 0.0
+
+        if zpImage < 0.0:
+            zpImage = 0.0
+        if zpImage == 0.0:
+            self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage))
+
+
+        magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage
+        self.logger.infoPair("Found zeropoint and set magref", str(magref))
+        self.logger.infoPair("CZW If that magref is -5.804804, then it's reading from the PHU, not the Deteff extension.","In a test, I mean.")
+        sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(")
+        sqlLine.group("diffImageID", str(diffSkyFileID))
+        sqlLine.group("magref",       magref)
+        sql = sqlLine.make(") VALUES ( ", ")")
+            
+
+        try: self.scratchDb.execute(sql)
+        except: 
+            self.logger.errorPair('failed sql: ', sql)
+            raise
+
+        self.logger.infoPair("CZW:", "About to loop over deteff bins.")
+
+        for i in xrange(1, 14):
+            stringint = str(i)
+            if i < 10:
+                stringint = '0' + str(i)
+                
+            sql = "UPDATE " + ippTableName + " , " + cmfTable + \
+                " SET offset"+stringint + " = OFFSET, counts" \
+                +stringint + " = COUNTS, diffMean" \
+                +stringint + " = DIFF_MEAN, diffStdev" \
+                +stringint + " = DIFF_STDEV, errMean" \
+                +stringint + " = ERR_MEAN WHERE table_index = " + str(i)
+                
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair('filed sql: ', sql)
+                raise
+            # end for
+                # Append this like this?
+        self.tablesToExport.append(ippTableName)
+        # end filters
+
+
+        return True
 
 
@@ -452 +533 @@
         self.logger.debug("Altering PSPS tables")
         self.logger.debug("Creating indexes on PSPS tables")
-        self.scratchDb.makeColumnUnique("DiffDetection", "objID")
+        self.scratchDb.makeColumnUnique("DiffDetection", "diffObjID")
         self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
 
@@ -555 +636 @@
             self.setMinMaxDiffObjID(["DiffDetection_"+str(num)])
 
-
+            self.populateDiffDetEffMeta(num)
+            self.logger.infoPair("populated","DiffDetEffMeta_"+str(num))
+            # CZW: I appended to tablesToExport inside populateDiffDetEffMeta to be like stack.
         #test = self.scratchDb.getRowCount("Object")
         #return False
@@ -623 +706 @@
                         self.haveApNpix[num] = 1
 
-                myTable = table.name + "_" + str(num)
+
+                myTable = table.name.replace(".","_") + "_" + str(num)
                 count = count + 1