Changeset 33098 for branches/meh_branches/ppsub_test/ippToPsps/jython/detectionbatch.py

Timestamp:

Jan 11, 2012, 11:19:21 PM (15 years ago)

Author:

mhuber

Message:

merged latest r33093 trunk changes to meh_branches/ppsub_test

Location:

branches/meh_branches/ppsub_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/detectionbatch.py (modified) (22 diffs)

branches/meh_branches/ppsub_test/ippToPsps/jython/detectionbatch.py

@@ -22 +22 @@
 '''
 DetectionBatch class
+
+This class, a sub-class of Batch, processes single exposures in the form of IPP smf files
+
 '''
 class DetectionBatch(Batch):
@@ -34 +37 @@
                  gpc1Db,
                  ippToPspsDb,
-                 camID):
+                 camID,
+                 batchID):
 
        super(DetectionBatch, self).__init__(
@@ -43 +47 @@
                ippToPspsDb,
                camID,
+               batchID,
                "P2", 
                gpc1Db.getCameraStageSmf(camID))
@@ -50 +55 @@
        # get camera meta data
        meta = self.gpc1Db.getCameraStageMeta(self.id)
+       if not meta:
+           self.everythingOK = False
+           return
+
        self.expID = meta[0];
        self.expName = meta[1];
@@ -101 +110 @@
 
        self.filter = self.header['FILTERID'][0:1]
+       self.filterID = self.scratchDb.getFilterID(self.filter) 
 
        # insert what we know about this stack batch into the stack table
@@ -367 +377 @@
         pspsTableName = "Detection_" + ota
         ippTableName = ota + "_psf"
-        
+       
+        results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
+
         # drop then re-create table
         self.scratchDb.dropTable(pspsTableName)
@@ -385 +397 @@
         sql = "INSERT IGNORE INTO " + pspsTableName + " ( \
                ippDetectID \
+               ,filterID \
+               ,surveyID \
+               ,obsTime \
                ,xPos \
                ,yPos \
@@ -395 +410 @@
                ,psfWidMinor \
                ,psfTheta \
+               ,psfLikelihood \
                ,psfCf \
                ,momentXX \
@@ -406 +422 @@
                ,skyErr \
                ,sgSep \
+               ,activeFlag \
+               ,assocDate \
+               ,historyModNum \
+               ,dataRelease \
                ) \
                SELECT \
                IPP_IDET \
+               , " + str(self.filterID) + "\
+               , " + str(self.surveyID) + " \
+               ," + str(self.obsTime) + " \
                ,X_PSF \
                ,Y_PSF \
@@ -419 +442 @@
                ,PSF_MINOR \
                ,PSF_THETA \
+               ,psfLikelihood(EXT_NSIGMA) \
                ,PSF_QF \
                ,MOMENTS_XX \
@@ -430 +454 @@
                ,SKY_SIGMA \
                ,EXT_NSIGMA \
+               , 0 \
+               , '" + self.dateStr + "' \
+               , 0 \
+               , " + str(self.dataRelease) + "\
                FROM " + ippTableName
         self.scratchDb.execute(sql)
 
-        # set obsTime
-        sql = "UPDATE " + pspsTableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
-        self.scratchDb.execute(sql)
-        self.scratchDb.updateAllRows(pspsTableName, "dataRelease", str(self.dataRelease))
-        self.scratchDb.updateAllRows(pspsTableName, "historyModNum", "0")
-
-        self.scratchDb.updateAllRows(pspsTableName, "surveyID", str(self.surveyID))
-        self.scratchDb.updateFilterID(pspsTableName, self.filter)
-
         # now delete bad flux and bad chip positions
+        sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
+        self.scratchDb.execute(sql)
+
+        # update cosmic ray and extended likelihoods
+        sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
+        self.scratchDb.execute(sql)
+        sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
+        self.scratchDb.execute(sql)
+
+        # remove detections will NULL inst flux or NULL peak ADU
         results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
         results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
@@ -514 +543 @@
         self.scratchDb.makeColumnPrimaryKey("Detection", "ippDetectID")
 
-    '''
-    Applies indexes to the IPP tables
-    '''
-    def indexIppTables(self):
-
-        self.logger.infoPair("Creating indexes on", "IPP tables")
-
-        for x in range(self.startX, self.endX):
-            for y in range(self.startY, self.endY):
-
-                # dodge the corners
-                if x==0 and y==0: continue
-                if x==0 and y==7: continue
-                if x==7 and y==0: continue
-                if x==7 and y==7: continue
-
-                extension = "XY%d%d_psf" % (x, y)
-                self.scratchDb.createIndex(extension, "IPP_IDET")
-
-    '''
-    Updates provided table with DVO IDs from DVO table
-    '''
-    def updateDvoIDs(self, table, sourceID, externID):
-
-        imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
-        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
-        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
-               a.ippObjID = b.ippObjID, \
-               a.detectID = b.detectID, \
-               a.objID = b.objID, \
-               a.infoFlag = b.flags << 32 | a.infoFlag \
-               WHERE a.ippDetectID = b.ippDetectID \
-               AND b.sourceID = " + str(sourceID) + " \
-               AND b.imageID = " + str(imageID)
-
-        self.scratchDb.execute(sql)
-
-    '''
-    Generates psf, cosmic ray and extended source likelihoods
-    '''
-    def populateLikelihoods(self, tableName):
-
-        sql = "SELECT ippDetectID, psfLikelihood, crLikelihood, extendedLikelihood, sgSep FROM " + tableName
-        rs = self.scratchDb.executeUpdatableQuery(sql)
-
-        sqrt2 = Math.sqrt(2)
-
-        while rs.next():
-
-            sgSep = rs.getDouble(5)
-            psfLikelihood = Erf.erfc(Math.abs(sgSep)/sqrt2)
-        
-            if sgSep > 0:
-                crLikelihood = 0
-                extendedLikelihood = 1.0 - psfLikelihood
-            else:
-                crLikelihood = 1.0 - psfLikelihood
-                extendedLikelihood = 0
-
-            # update columns
-            rs.updateDouble(2, psfLikelihood )
-            rs.updateDouble(3, crLikelihood )
-            rs.updateDouble(4, extendedLikelihood )
-
-            # now 'commit' to database
-            rs.updateRow();
-
-
-    '''
-    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
-    '''
-    def populatePspsTables(self):
-
         self.populateFrameMeta()
      
         # dictionary objects to hold sourceIDs and imageIDs for later
-        sourceIDs = {}
-        imageIDs = {}
+        self.sourceIDs = {}
+        self.imageIDs = {}
 
         # loop through all OTAs and populate ImageMeta extensions
@@ -614 +570 @@
 
                 # check we have valid sourceID/imageID pair from the header
-                if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL":
-                    continue
-                if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL":
-                    continue
+                if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue
+                if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue
 
                 # store sourceID/imageID combo in Db so DVO can look up later
@@ -624 +578 @@
 
                 # store these for later
-                sourceIDs[ota] = header['SOURCEID']
-                imageIDs[ota] = header['IMAGEID']
+                self.sourceIDs[ota] = header['SOURCEID']
+                self.imageIDs[ota] = header['IMAGEID']
 
                 # populate ImageMeta
@@ -632 +586 @@
              
         # now run DVO code to get all IDs
-        if not self.useFullTables: self.getIDsFromDVO()
+        if not self.useFullTables: 
+            if not self.getIDsFromDVO(): return False
+
+        return True
+
+
+    '''
+    Applies indexes to the IPP tables
+    '''
+    def indexIppTables(self):
+
+        self.logger.infoPair("Creating indexes on", "IPP tables")
+
+        for x in range(self.startX, self.endX):
+            for y in range(self.startY, self.endY):
+
+                # dodge the corners
+                if x==0 and y==0: continue
+                if x==0 and y==7: continue
+                if x==7 and y==0: continue
+                if x==7 and y==7: continue
+
+                extension = "XY%d%d_psf" % (x, y)
+                self.scratchDb.createIndex(extension, "IPP_IDET")
+
+    '''
+    Updates provided table with DVO IDs from DVO table
+    '''
+    def updateDvoIDs(self, table, sourceID, externID):
+
+        imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
+               a.ippObjID = b.ippObjID, \
+               a.detectID = b.detectID, \
+               a.objID = b.objID, \
+               a.infoFlag = b.flags << 32 | a.infoFlag \
+               WHERE a.ippDetectID = b.ippDetectID \
+               AND b.sourceID = " + str(sourceID) + " \
+               AND b.imageID = " + str(imageID)
+
+        self.scratchDb.execute(sql)
+
+
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
 
         # loop through all OTAs again to update with DVO IDs
@@ -640 +641 @@
         otaCount = 0
         results = {}
-        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
-        self.logger.info("|  OTA  |   Sat Det     | NULL instFlux | NULL peak ADU | NULL obj ID   |  Remainder    |")
-        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+        totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID = totalDetections = 0
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
+        self.logger.info("|  OTA  | Initial total |   Sat Det     | NULL instFlux | NULL peak ADU | NULL obj ID   |  Remainder    |")
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
@@ -653 +655 @@
 
                 ota = "XY%d%d" % (x, y)
-                if ota not in sourceIDs: continue
+                if ota not in self.sourceIDs: continue
 
                 #self.logger.infoTitle("Processing " + ota)
+                if not self.scratchDb.astrometricSolutionOK(ota):
+                    self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
+                    continue
 
                 # populate remainder of tables
@@ -661 +666 @@
 
                 # now add DVO IDs
-                self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota])
+                self.updateDvoIDs("Detection_" + ota, self.sourceIDs[ota], self.imageIDs[ota])
                 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
                 self.updateImageID("Detection_" + ota, x, y) 
-                self.populateLikelihoods("Detection_" + ota)
                 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 
-                self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", 
+                self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 
                         ota, 
+                        results['ORIGINALTOTAL'], 
                         results['SATDET'], 
                         results['NULLINSTFLUX'], 
@@ -674 +679 @@
                         rowCount)
 
+                totalOriginal = totalOriginal + results['ORIGINALTOTAL']
+                totalSatDet = totalSatDet + results['SATDET']
+                totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX']
+                totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU']
+                totalNullObjID = totalNullObjID + results['NULLOBJID']
+                totalDetections = totalDetections + rowCount
+
                 # check we have something in this Detection table TODO add this to table above
                 if rowCount < 1:
@@ -681 +693 @@
                 # update ImageMeta with count of detections for this OTA and photoCodeID
                 sql = "UPDATE ImageMeta_" + ota + " \
-                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
+                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.sourceIDs[ota], self.imageIDs[ota]))
                 self.scratchDb.execute(sql)
 
@@ -696 +708 @@
                 otaCount = otaCount + 1
 
-        # TODO print totals here
-        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+        # print totals
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
+        self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", 
+                totalOriginal, 
+                totalSatDet, 
+                totalNulIInstFlux, 
+                totalNullPeakFlux, 
+                totalNullObjID,
+                totalDetections)
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
 
         # if we only have one table export, i.e. FrameMeta, then get out of here
@@ -739 +759 @@
        if self.testMode: regex = "XY33.psf"
        else : regex = ".*.psf"
-
-       return super(DetectionBatch, self).importIppTables(regex)
+  
+       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA"
+
+       return super(DetectionBatch, self).importIppTables(columns, regex)