Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 33830) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 34615) @@ -388,8 +388,11 @@ # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene BEFORE = self.scratchDb.getRowCount(ippTableName) - sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5" - self.scratchDb.execute(sql) + #don't do this anymore + #sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5" + #self.scratchDb.execute(sql) #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName) - results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName) + #results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName) + #we aren't deletng these now + results['SATDET'] = 0 #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED) @@ -404,6 +407,6 @@ ,xPosErr \ ,yPosErr \ - ,instFlux \ - ,instFluxErr \ + ,psfFlux \ + ,psfFluxErr \ ,peakADU \ ,psfWidMajor \ @@ -411,11 +414,13 @@ ,psfTheta \ ,psfLikelihood \ - ,psfCf \ + ,psfQf \ ,momentXX \ ,momentXY \ ,momentYY \ - ,apMag \ + ,apFlux \ ,kronFlux \ ,kronFluxErr \ + ,psfQfPerfect \ + ,psfChiSq \ ,infoFlag \ ,sky \ @@ -450,5 +455,7 @@ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ - ,FLAGS\ + ,PSF_QF_PERFECT \ + ,PSF_CHISQ \ + ,FLAGS2 << 32 | FLAGS\ ,SKY \ ,SKY_SIGMA \ @@ -459,19 +466,36 @@ , " + str(self.config.dataRelease) + "\ FROM " + ippTableName + # SA10 todo + # need to correct formulas for psfFlux/psfFluxErr + # need to correct apMag apFlux + # need to correct unites kronFlux/Err to correct units + # need to change psfqfperfect -> psfqfperfect PSF_QF_PERFECT? + # need to change psfchisq PSF_CHI_SQ? + # need to change unites of sky/skyErr to adu/s/pixel + # need formula for sgsep/psflikelhood + # self.logger.info(sql) self.scratchDb.execute(sql) # now delete bad flux and bad chip positions - sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux - self.scratchDb.execute(sql) - + #sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux + #self.scratchDb.execute(sql) + + # add a instFlux = 0.0 -> 0.000001 + sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0" + self.scratchDb.execute(sql) + # update cosmic ray and extended likelihoods - sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" - self.scratchDb.execute(sql) - sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" - self.scratchDb.execute(sql) + #sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" + #self.scratchDb.execute(sql) + #sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" + #self.scratchDb.execute(sql) # remove detections will NULL inst flux or NULL peak ADU - results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") - results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") + # leave them in at this stage- don't delete them. + #results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") + #results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") + #we don't delete thos anymore + results['NULLINSTFLUX'] = 0; + results['NULLPEAKADU'] = 0; ''' @@ -504,4 +528,41 @@ ''' + Populates the DetectionCalib table for this OTA + ''' + def populateDetectionCalibTable(self, ota): + + tableName = "DetectionCalib_" + ota + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" + try: self.scratchDb.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,detectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + ) \ + SELECT \ + objID \ + ,detectID \ + ,ippObjID \ + ,ippDetectID \ + ,filterID \ + ,surveyID \ + FROM Detection_" + ota + self.scratchDb.execute(sql) + #SA10 TODO + # insert ra/dec/calstuff into Table + # heather discovered dec is called 'dec_' + sql = "UPDATE " + tableName + " SET ra = -999, dec_ =-999, raErr = -999, decErr = -999, zp = -999, zpErr = -999, expTime = -999, airMass = -999" + self.scratchDb.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + + ''' Populates the Detection table for this OTA ''' @@ -608,5 +669,5 @@ extension = "XY%d%d_psf" % (x, y) self.scratchDb.createIndex(extension, "IPP_IDET") - + self.logger.infoPair("created indexes on", "IPP tables") ''' Updates provided table with DVO IDs from DVO table @@ -620,5 +681,5 @@ a.detectID = b.detectID, \ a.objID = b.objID, \ - a.infoFlag = b.flags << 32 | a.infoFlag \ + a.infoFlag = b.flags << 46 | a.infoFlag \ WHERE a.ippDetectID = b.ippDetectID \ AND b.imageID = " + str(imageID) @@ -658,13 +719,17 @@ self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) continue - + # self.logger.info("populate stuff "); # populate remainder of tables self.populateDetectionTable(ota, results) - + # self.logger.info("successful populate "); # now add DVO IDs self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) + # self.logger.info("updated dvoids") results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") - self.updateImageID("Detection_" + ota, x, y) + #self.logger.info("deleted nulls") + self.updateImageID("Detection_" + ota, x, y) + # self.logger.info("updateImageId") rowCount = self.scratchDb.getRowCount("Detection_" + ota) + # self.logger.info("got row count") self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", ota, @@ -675,5 +740,4 @@ results['NULLOBJID'], rowCount) - totalOriginal = totalOriginal + results['ORIGINALTOTAL'] totalSatDet = totalSatDet + results['SATDET'] @@ -682,5 +746,5 @@ totalNullObjID = totalNullObjID + results['NULLOBJID'] totalDetections = totalDetections + rowCount - + #self.logger.info("updated totals") # check we have something in this Detection table TODO add this to table above if rowCount < 1: @@ -692,8 +756,13 @@ SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) self.scratchDb.execute(sql) - + + #self.logger.info("updated imagedata") self.populateSkinnyObjectTable(ota) + #self.logger.info("updated skinnyobject") self.populateObjectCalColorTable(ota) - + #self.logger.info("updated objectcalcolor") + self.populateDetectionCalibTable(ota) + #self.logger.info("updated detectioncalibtable") + # add these to list of tables to export later self.tablesToExport.append("ImageMeta_" + ota) @@ -701,8 +770,9 @@ self.tablesToExport.append("SkinnyObject_" + ota) self.tablesToExport.append("ObjectCalColor_" + ota) + self.tablesToExport.append("DetectionCalib_" + ota) tables.append("Detection_" + ota) otaCount = otaCount + 1 - + #self.logger.info("end of ota") # print totals self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") @@ -757,5 +827,5 @@ else : regex = ".*.psf" - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" return super(DetectionBatch, self).importIppTables(columns, regex)