Changeset 34615 for trunk/ippToPsps/jython/detectionbatch.py

Timestamp:

Oct 26, 2012, 5:15:52 PM (14 years ago)

Author:

heather

Message:

schema change sa10

File:

• trunk/ippToPsps/jython/detectionbatch.py (modified) (14 diffs)

trunk/ippToPsps/jython/detectionbatch.py

@@ -388 +388 @@
         # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene
         BEFORE = self.scratchDb.getRowCount(ippTableName)
-        sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
-        self.scratchDb.execute(sql)
+        #don't do this anymore
+        #sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
+        #self.scratchDb.execute(sql)
         #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName)
-        results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
+        #results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
+        #we aren't deletng these now
+        results['SATDET'] = 0 
         #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED)
 
@@ -404 +407 @@
                ,xPosErr \
                ,yPosErr \
-               ,instFlux \
-               ,instFluxErr \
+               ,psfFlux \
+               ,psfFluxErr \
                ,peakADU \
                ,psfWidMajor \
@@ -411 +414 @@
                ,psfTheta \
                ,psfLikelihood \
-               ,psfCf \
+               ,psfQf \
                ,momentXX \
                ,momentXY \
                ,momentYY \
-               ,apMag \
+               ,apFlux \
                ,kronFlux \
                ,kronFluxErr \
+               ,psfQfPerfect \
+               ,psfChiSq \
                ,infoFlag \
                ,sky \
@@ -450 +455 @@
                ,KRON_FLUX / " + self.header['EXPTIME'] + " \
                ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \
-               ,FLAGS\
+               ,PSF_QF_PERFECT \
+               ,PSF_CHISQ \
+               ,FLAGS2 << 32 | FLAGS\
                ,SKY \
                ,SKY_SIGMA \
@@ -459 +466 @@
                , " + str(self.config.dataRelease) + "\
                FROM " + ippTableName
+        # SA10 todo
+        # need to correct formulas for psfFlux/psfFluxErr
+        # need to correct apMag apFlux
+        # need to correct unites kronFlux/Err to correct units
+        # need to change psfqfperfect -> psfqfperfect PSF_QF_PERFECT?
+        # need to change psfchisq  PSF_CHI_SQ?
+        # need to change unites of sky/skyErr to adu/s/pixel
+        # need formula for sgsep/psflikelhood
+        # self.logger.info(sql)
         self.scratchDb.execute(sql)
 
         # now delete bad flux and bad chip positions
-        sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
-        self.scratchDb.execute(sql)
-
+        #sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
+        #self.scratchDb.execute(sql)
+
+        # add a instFlux = 0.0 -> 0.000001
+        sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux =  0"
+        self.scratchDb.execute(sql)
+      
         # update cosmic ray and extended likelihoods
-        sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
-        self.scratchDb.execute(sql)
-        sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
-        self.scratchDb.execute(sql)
+        #sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
+        #self.scratchDb.execute(sql)
+        #sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
+        #self.scratchDb.execute(sql)
 
         # remove detections will NULL inst flux or NULL peak ADU
-        results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
-        results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
+        # leave them in at this stage- don't delete them.
+        #results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
+        #results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
+        #we don't delete thos anymore
+        results['NULLINSTFLUX'] = 0;
+        results['NULLPEAKADU'] = 0;
 
     '''
@@ -504 +528 @@
 
     '''
+    Populates the DetectionCalib table for this OTA
+    '''
+    def populateDetectionCalibTable(self, ota):
+
+        tableName = "DetectionCalib_" + ota
+        # drop then re-create table
+        self.scratchDb.dropTable(tableName)
+        sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib"
+        try: self.scratchDb.execute(sql)
+        except: pass
+
+        # insert all detections into table
+        sql = "INSERT INTO " + tableName + " ( \
+               objID \
+               ,detectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               ) \
+               SELECT \
+               objID \
+               ,detectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               FROM Detection_" + ota
+        self.scratchDb.execute(sql)
+        #SA10 TODO
+        # insert ra/dec/calstuff into Table
+        # heather discovered dec is called 'dec_'
+        sql = "UPDATE " + tableName + " SET ra = -999, dec_ =-999, raErr = -999, decErr = -999, zp = -999, zpErr = -999, expTime = -999, airMass = -999"
+        self.scratchDb.execute(sql)
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        
+    '''
     Populates the Detection table for this OTA
     '''
@@ -608 +669 @@
                 extension = "XY%d%d_psf" % (x, y)
                 self.scratchDb.createIndex(extension, "IPP_IDET")
-
+        self.logger.infoPair("created indexes on", "IPP tables")      
     '''
     Updates provided table with DVO IDs from DVO table
@@ -620 +681 @@
                a.detectID = b.detectID, \
                a.objID = b.objID, \
-               a.infoFlag = b.flags << 32 | a.infoFlag \
+               a.infoFlag = b.flags << 46 | a.infoFlag \
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.imageID = " + str(imageID)
@@ -658 +719 @@
                     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
                     continue
-
+               # self.logger.info("populate stuff ");
                 # populate remainder of tables
                 self.populateDetectionTable(ota, results)
-
+               # self.logger.info("successful populate ");
                 # now add DVO IDs
                 self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota])
+               # self.logger.info("updated dvoids")
                 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
-                self.updateImageID("Detection_" + ota, x, y) 
+                #self.logger.info("deleted nulls")
+                self.updateImageID("Detection_" + ota, x, y)
+               # self.logger.info("updateImageId")
                 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 
+              #  self.logger.info("got row count")
                 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 
                         ota, 
@@ -675 +740 @@
                         results['NULLOBJID'],
                         rowCount)
-
                 totalOriginal = totalOriginal + results['ORIGINALTOTAL']
                 totalSatDet = totalSatDet + results['SATDET']
@@ -682 +746 @@
                 totalNullObjID = totalNullObjID + results['NULLOBJID']
                 totalDetections = totalDetections + rowCount
-
+                #self.logger.info("updated totals")
                 # check we have something in this Detection table TODO add this to table above
                 if rowCount < 1:
@@ -692 +756 @@
                        SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota]))
                 self.scratchDb.execute(sql)
-
+                
+                #self.logger.info("updated imagedata")
                 self.populateSkinnyObjectTable(ota)
+                #self.logger.info("updated skinnyobject")
                 self.populateObjectCalColorTable(ota)
-
+                #self.logger.info("updated objectcalcolor")
+                self.populateDetectionCalibTable(ota)
+                #self.logger.info("updated detectioncalibtable")
+                
                 # add these to list of tables to export later
                 self.tablesToExport.append("ImageMeta_" + ota)
@@ -701 +770 @@
                 self.tablesToExport.append("SkinnyObject_" + ota)
                 self.tablesToExport.append("ObjectCalColor_" + ota)
+                self.tablesToExport.append("DetectionCalib_" + ota)
                 tables.append("Detection_" + ota)
            
                 otaCount = otaCount + 1
-
+                #self.logger.info("end of ota") 
         # print totals
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
@@ -757 +827 @@
        else : regex = ".*.psf"
   
-       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA"
+       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
 
        return super(DetectionBatch, self).importIppTables(columns, regex)