Changeset 34772 for branches/czw_branch/20120906/ippToPsps/jython/detectionbatch.py

Timestamp:

Dec 6, 2012, 6:32:19 PM (14 years ago)

Author:

watersc1

Message:

merge from trunk. Preliminary versions of stacktool/warptool updates and regenerate_background.pl script. The warp code is finished and tested, and I still need to get the stack version resolved.

Location:

branches/czw_branch/20120906/ippToPsps

Files:

• . (modified) (1 prop)
• jython/detectionbatch.py (modified) (15 diffs)

branches/czw_branch/20120906/ippToPsps/jython/detectionbatch.py

@@ -388 +388 @@
         # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene
         BEFORE = self.scratchDb.getRowCount(ippTableName)
-        sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
-        self.scratchDb.execute(sql)
+        #don't do this anymore
+        #sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
+        #self.scratchDb.execute(sql)
         #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName)
-        results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
+        #results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
+        #we aren't deletng these now
+        results['SATDET'] = 0 
         #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED)
 
@@ -404 +407 @@
                ,xPosErr \
                ,yPosErr \
-               ,instFlux \
-               ,instFluxErr \
+               ,psfFlux \
+               ,psfFluxErr \
                ,peakADU \
                ,psfWidMajor \
@@ -411 +414 @@
                ,psfTheta \
                ,psfLikelihood \
-               ,psfCf \
+               ,psfQf \
                ,momentXX \
                ,momentXY \
                ,momentYY \
-               ,apMag \
+               ,apFlux \
                ,kronFlux \
                ,kronFluxErr \
+               ,psfQfPerfect \
+               ,psfChiSq \
                ,infoFlag \
                ,sky \
@@ -447 +452 @@
                ,MOMENTS_XY \
                ,MOMENTS_YY \
-               ,AP_MAG \
+               ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \
                ,KRON_FLUX / " + self.header['EXPTIME'] + " \
                ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \
-               ,FLAGS\
-               ,SKY \
-               ,SKY_SIGMA \
+               ,PSF_QF_PERFECT \
+               ,PSF_CHISQ \
+               ,FLAGS2 << 32 | FLAGS \
+               ,SKY  / " + self.header['EXPTIME'] + " \
+               ,SKY_SIGMA  / " + self.header['EXPTIME'] + " \
                ,EXT_NSIGMA \
                , 0 \
@@ -459 +466 @@
                , " + str(self.config.dataRelease) + "\
                FROM " + ippTableName
+        # self.logger.info(sql)
         self.scratchDb.execute(sql)
 
         # now delete bad flux and bad chip positions
-        sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
-        self.scratchDb.execute(sql)
-
+        #sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
+        #self.scratchDb.execute(sql)
+
+        # add a instFlux = 0.0 -> 0.000001
+        sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux =  0"
+        self.scratchDb.execute(sql)
+      
         # update cosmic ray and extended likelihoods
-        sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
-        self.scratchDb.execute(sql)
-        sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
-        self.scratchDb.execute(sql)
+        #sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
+        #self.scratchDb.execute(sql)
+        #sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
+        #self.scratchDb.execute(sql)
 
         # remove detections will NULL inst flux or NULL peak ADU
-        results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
-        results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
+        # leave them in at this stage- don't delete them.
+        #results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
+        #results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
+        #we don't delete thos anymore
+        results['NULLINSTFLUX'] = 0;
+        results['NULLPEAKADU'] = 0;
 
     '''
@@ -504 +520 @@
 
     '''
+    Populates the DetectionCalib table for this OTA
+    '''
+    def populateDetectionCalibTable(self, ota):
+
+        tableName = "DetectionCalib_" + ota
+        # drop then re-create table
+        self.scratchDb.dropTable(tableName)
+        sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib"
+        try: self.scratchDb.execute(sql)
+        except: pass
+
+        # insert all detections into table
+        sql = "INSERT INTO " + tableName + " ( \
+               objID \
+               ,detectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               ) \
+               SELECT \
+               objID \
+               ,detectID \
+               ,ippObjID \
+               ,ippDetectID \
+               ,filterID \
+               ,surveyID \
+               FROM Detection_" + ota
+        self.scratchDb.execute(sql)
+
+        # insert calibration information from dvoDetections into the Table
+        sql = "UPDATE " + tableName + " AS a, "  + self.scratchDb.dvoDetectionTable + " AS b \
+            SET a.ra = b.ra, \
+            a.dec = b.dec_, \
+            a.raErr = b.raErr, \
+            a.decErr = b.decErr, \
+            a.zp = b.zp, \
+            a.zpErr = b.zpErr, \
+            a.expTime = b.expTime, \
+            a.airMass = b.airMass   \
+            WHERE a.objID = b.objID AND a.detectID = b.detectID"
+        self.scratchDb.execute(sql)
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        
+    '''
     Populates the Detection table for this OTA
     '''
@@ -587 +648 @@
             if not self.getIDsFromDVO(): return False
 
+        # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of 
+        # the column in PSPS
+        self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double")
+
+
+
         return True
 
@@ -608 +675 @@
                 extension = "XY%d%d_psf" % (x, y)
                 self.scratchDb.createIndex(extension, "IPP_IDET")
-
+        self.logger.infoPair("created indexes on", "IPP tables")      
     '''
     Updates provided table with DVO IDs from DVO table
@@ -620 +687 @@
                a.detectID = b.detectID, \
                a.objID = b.objID, \
-               a.infoFlag = b.flags << 32 | a.infoFlag \
+               a.infoFlag = b.flags << 45 | a.infoFlag \
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.imageID = " + str(imageID)
@@ -658 +725 @@
                     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
                     continue
-
+               # self.logger.info("populate stuff ");
                 # populate remainder of tables
                 self.populateDetectionTable(ota, results)
-
+               # self.logger.info("successful populate ");
                 # now add DVO IDs
                 self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota])
+               # self.logger.info("updated dvoids")
                 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
-                self.updateImageID("Detection_" + ota, x, y) 
+                #self.logger.info("deleted nulls")
+                self.updateImageID("Detection_" + ota, x, y)
+               # self.logger.info("updateImageId")
                 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 
+              #  self.logger.info("got row count")
                 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 
                         ota, 
@@ -675 +746 @@
                         results['NULLOBJID'],
                         rowCount)
-
                 totalOriginal = totalOriginal + results['ORIGINALTOTAL']
                 totalSatDet = totalSatDet + results['SATDET']
@@ -682 +752 @@
                 totalNullObjID = totalNullObjID + results['NULLOBJID']
                 totalDetections = totalDetections + rowCount
-
+                #self.logger.info("updated totals")
                 # check we have something in this Detection table TODO add this to table above
                 if rowCount < 1:
@@ -692 +762 @@
                        SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota]))
                 self.scratchDb.execute(sql)
-
+                
+                #self.logger.info("updated imagedata")
                 self.populateSkinnyObjectTable(ota)
+                #self.logger.info("updated skinnyobject")
                 self.populateObjectCalColorTable(ota)
-
+                #self.logger.info("updated objectcalcolor")
+                self.populateDetectionCalibTable(ota)
+                #self.logger.info("updated detectioncalibtable")
+                
                 # add these to list of tables to export later
                 self.tablesToExport.append("ImageMeta_" + ota)
@@ -701 +776 @@
                 self.tablesToExport.append("SkinnyObject_" + ota)
                 self.tablesToExport.append("ObjectCalColor_" + ota)
+                self.tablesToExport.append("DetectionCalib_" + ota)
                 tables.append("Detection_" + ota)
            
                 otaCount = otaCount + 1
-
+                #self.logger.info("end of ota") 
         # print totals
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
@@ -757 +833 @@
        else : regex = ".*.psf"
   
-       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA"
+       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
 
        return super(DetectionBatch, self).importIppTables(columns, regex)