Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 38810) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 38837) @@ -3,4 +3,5 @@ import os.path import sys +import math import glob import time @@ -184,10 +185,12 @@ def populateImageMetaTable(self, ota, header): + # the supplied 'header' matches this chip + # self.header is the PHU header for this smf + tableName = "ImageMeta_" + ota - # we drop the table before calling this functoin so it is not left behind on failure + # we drop the table before calling this function so it is not left behind on failure sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" - try: self.scratchDb.execute(sql) - except: pass + self.scratchDb.execute(sql) if (ota[0:2] == "XY"): ccdID = ota[2:4] @@ -200,5 +203,5 @@ psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') - psfFwhm = 0.5*(float(psfFwhmMajor) + float(psfFwhmMinor)) + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) psfmodel_name = self.getKeyValue(header, 'PSFMODEL') @@ -207,45 +210,52 @@ ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = sqrt(ast_cdx^2 + ast_cdy^2) + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + + # Convert detectionThreshold to appropriate magnitudes + detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') + expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') + zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + + # XXX zp correction should come from DVO + detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) # insert image metadata into table sqlLine = sqlUtility("INSERT INTO " + tableName + "(") - ## extract ZPT_OBS and define the zpFactor, apply to sky, skyScat? - - sqlLine.group("frameID", str(self.expID)) - sqlLine.group("ccdID", str(ccdID)) - sqlLine.group("bias", str(self.bias)) - sqlLine.group("biasScat", str(self.biasScat)) - sqlLine.group("sky", self.getKeyFloat(header, "%.8f", 'MSKY_MN')) - sqlLine.group("skyScat", self.getKeyFloat(header, "%.8f", 'MSKY_SIG')) - sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) - sqlLine.group("astroScat", astroscat) - sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')) - sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) - sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) - sqlLine.group("nAxis1", self.getKeyInt(header, 0, 'NAXIS1')) - sqlLine.group("nAxis2", self.getKeyInt(header, 0, 'NAXIS2')) - sqlLine.group("psfModelID" psfmodelID) - sqlLine.group("psfFwhm", str(psfFwhm)) - sqlLine.group("psfWidMajor", psfFwhmMajor) - sqlLine.group("psfWidMinor", psfFwhmMinor) - sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) - sqlLine.group("momentMajor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1')) - sqlLine.group("momentMinor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2')) - sqlLine.group("momentM2C", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C')) - sqlLine.group("momentM2S", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S')) - sqlLine.group("momentM3", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3')) - sqlLine.group("momentM4", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4')) - sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) - sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) - sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) - sqlLine.group("qaFlags", str(self.scratchDb.getDvoImageFlags(header['IMAGEID']))) - sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) - sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) - sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) - #what happens if it can't find it? - # CZW: I think this should properly trap the fringe - if self.getKeyValue(header, 'DETREND.FRINGE') != "NULL": + sqlLine.group("frameID", self.expID) + sqlLine.group("ccdID", ccdID) + sqlLine.group("bias", self.bias) + sqlLine.group("biasScat", self.biasScat) + sqlLine.group("sky", self.getKeyFloat(header, "%.8f", 'MSKY_MN')) + sqlLine.group("skyScat", self.getKeyFloat(header, "%.8f", 'MSKY_SIG')) + sqlLine.group("detectionThreshold", detectionThreshold) + sqlLine.group("astroScat", astroscat) + sqlLine.group("photoScat", self.getKeyFloat(header, "%.8f", 'ZPT_ERR')) + sqlLine.group("photoZero", zp) + sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nAxis1", self.getKeyInt(header, 0, 'NAXIS1')) + sqlLine.group("nAxis2", self.getKeyInt(header, 0, 'NAXIS2')) + sqlLine.group("psfModelID", psfmodelID) + sqlLine.group("psfFWHM", psfFWHM) + sqlLine.group("psfWidMajor", psfFwhmMajor) + sqlLine.group("psfWidMinor", psfFwhmMinor) + sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) + sqlLine.group("momentMajor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1')) + sqlLine.group("momentMinor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2')) + sqlLine.group("momentM2C", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C')) + sqlLine.group("momentM2S", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S')) + sqlLine.group("momentM3", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3')) + sqlLine.group("momentM4", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4')) + sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) + sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) + sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) + sqlLine.group("qaFlags", self.scratchDb.getDvoImageFlags(header['IMAGEID'])) + sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) + sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) + sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) + + # DETREND.FRINGE is not required : check if it is in the header before setting + if 'DETREND.FRINGE' in header: sqlLine.group("detrend4", self.getKeyValue(header, 'DETREND.FRINGE')) @@ -256,5 +266,4 @@ # sqlLine.group("detrend8", " ") - sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')) sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')) @@ -307,12 +316,9 @@ sql = sqlLine.make(") VALUES ( ", ")") - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair('failed sql: ', sql) - raise + self.scratchDb.execute(sql) self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "processingVersion", str(self.skychunk.processingVersion)) + if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -428,4 +434,7 @@ # instrumental fluxes in counts/sec and calibrated sizes (arcsec). # below, we convert instrumental fluxes to Jy + + # NOTE: math operations below take place in SQL, not JYTHON. thus we use (e.g.) + # abs() not math.fabs() sqlLine.group("ippDetectID", "IPP_IDET") @@ -644,4 +653,5 @@ ota = "XY%d%d" % (x, y) + if ota not in self.imageIDs: continue # I need better control over this.. @@ -682,6 +692,10 @@ if x==7 and y==7: continue - extension = "XY%d%d_psf" % (x, y) + ota = "XY%d%d" % (x, y) + if ota not in self.imageIDs: continue + + extension = ota + "_psf" self.scratchDb.createIndex(extension, "IPP_IDET") + # try the test Chip self.scratchDb.createIndex("Chip_psf", "IPP_IDET") @@ -698,33 +712,28 @@ # XXX : EAM 20150925 : is this IGNORE necessary? - sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ - a.objID = b.objID, \ - a.detectID = b.detectID, \ - a.ippObjID = b.ippObjID, \ - a.dvoRegionID = b.catID, \ - a.ra = b.ra, \ - a.dec = b.dec_, \ - a.zp = b.zp, \ - a.telluricExt = b.telluricExt, \ - a.airmass = b.airmass, \ - a.infoFlag3 = b.flags, \ - a.expTime = b.expTime, \ - a.psfFlux = a.psfFlux * b.zpFactor, \ - a.psfFluxErr = a.psfFluxErr * b.zpFactor, \ - a.apFlux = a.apFlux * b.zpFactor, \ - a.apFluxErr = a.apFluxErr * b.zpFactor, \ - a.kronFlux = a.kronFlux * b.zpFactor, \ - a.kronFluxErr = a.kronFluxErr * b.zpFactor \ - a.sky = a.sky * b.zpFactor, \ - a.skyErr = a.skyErr * b.zpFactor \ - WHERE a.ippDetectID = b.ippDetectID \ - AND b.imageID = " + str(imageID) - - ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.psfFluxErr = a.psfFluxErr * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.apFlux = a.apFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.apFluxErr = a.apFluxErr * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.kronFlux = a.kronFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.kronFluxErr = a.kronFluxErr * 3630.78 * POW(10.0, -0.4*b.zp) \ + sqlLine = sqlUtility("UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET") + sqlLine.group("a.objID", "b.objID") + sqlLine.group("a.detectID", "b.detectID") + sqlLine.group("a.ippObjID", "b.ippObjID") + sqlLine.group("a.dvoRegionID", "b.catID") + sqlLine.group("a.ra", "b.ra") + sqlLine.group("a.dec", "b.dec_") + sqlLine.group("a.zp", "b.zp") + sqlLine.group("a.telluricExt", "b.telluricExt") + sqlLine.group("a.airmass", "b.airmass") + sqlLine.group("a.infoFlag3", "b.flags") + sqlLine.group("a.expTime", "b.expTime") + sqlLine.group("a.psfFlux", "a.psfFlux * b.zpFactor") + sqlLine.group("a.psfFluxErr", "a.psfFluxErr * b.zpFactor") + sqlLine.group("a.apFlux", "a.apFlux * b.zpFactor") + sqlLine.group("a.apFluxErr", "a.apFluxErr * b.zpFactor") + sqlLine.group("a.kronFlux", "a.kronFlux * b.zpFactor") + sqlLine.group("a.kronFluxErr", "a.kronFluxErr * b.zpFactor") + sqlLine.group("a.sky", "a.sky * b.zpFactor") + sqlLine.group("a.skyErr", "a.skyErr * b.zpFactor") + + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) + + ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ # instrumental flux vs Janskies: @@ -741,9 +750,6 @@ # NOTE: update dvopsps to populate zpFactor = 3630.78 * ten(-0.4*zp) - print sql - try: self.scratchDb.execute(sql) - except: - self.logger.infoPair("failed sql",sql) - raise + self.scratchDb.execute(sql) + ''' Updates table and generates pspsuniqueids @@ -929,5 +935,5 @@ # update FrameMeta with count OTAs in this file and total number of photometric reference sources - sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) + sql = "UPDATE FrameMeta SET nOTA = %d, nPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) self.scratchDb.execute(sql) #to make it stop @@ -939,16 +945,8 @@ def removeDuplicateObjects(self): - sql = "DROP TABLE SkinnyObject_All" - try: - self.scratchDb.execute(sql) - except: - print "SkinnyObject_All table not yet defined" + self.scratchDb.dropTable("SkinnyObject_All") sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))" - try: - self.scratchDb.execute(sql) - except: - print "failed to create table SkinnyObject_All" - raise + self.scratchDb.execute(sql) for chipname in self.validChips: @@ -956,10 +954,5 @@ sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \ SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname - try: - self.scratchDb.execute(sql) - except: - print "failed to insert entry SkinnyObject_All: ", chipname - raise - + self.scratchDb.execute(sql) Nduplicates = 0