Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 39304) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 39305) @@ -149,4 +149,8 @@ sqlLine.group("analysisVer", self.analysisVer); sqlLine.group("md5sum", str(self.md5sum)) + + # these lines should write "NULL" if the header contains NaN -- NULLs are converted below + # to -999 for the sql database (a terrible solution, but mandatory?) + sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')); sqlLine.group("expStart", self.getKeyFloat(self.header, "%.10f", 'MJD-OBS')); @@ -218,11 +222,30 @@ else: ccdID = 0 + ### any operations below which act on the output of getKeyFloat need to be + ### able to handle NULL results. + # XXX hard-wired platescale : 0.257 pltscale = 0.257 pltscale2 = pltscale * pltscale - psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') - psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') - psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) + # NOTE: getKeyFloat returns the header value if it is a float, + # -999 if the value is missing and "NULL" if the value in the + # header is a NaN. any math operations below need to handle + # these cases correctly + + # NOTE 2: getKeyFloat NOW returns the header value if it is a + # float or "NULL" if the value is missing or if the value in + # the header is a NaN. any math operations below need to + # handle these cases correctly + + psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale + + psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN') + if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale + + psfFWHM = "NULL" + if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"): + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) psfmodel_name = self.getKeyValue(header, 'PSFMODEL') @@ -231,16 +254,39 @@ ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + + astroscat = "NULL" + if (ast_cdx != "NULL") and (ast_cdy != "NULL"): + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) # Convert detectionThreshold to appropriate magnitudes - detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') + magref = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + # zp correction from DVO zpImage = self.scratchDb.getImageZeroPoint(self.imageIDs[ota]) - # XXX zp correction should come from DVO - detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime) - + if (zpImage == "NULL"): + zpImage = zp + + detectionThreshold = "NULL" + if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"): + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + + # other header fields which depend on pltscale: + momentMajor = self.getKeyFloat(header, "%.8f", 'IQ_FW1')) + momentMinor = self.getKeyFloat(header, "%.8f", 'IQ_FW2')) + momentM2C = self.getKeyFloat(header, "%.8f", 'IQ_M2C')) + momentM2S = self.getKeyFloat(header, "%.8f", 'IQ_M2S')) + momentM3 = self.getKeyFloat(header, "%.8f", 'IQ_M3')) + momentM4 = self.getKeyFloat(header, "%.8f", 'IQ_M4')) + + if (momentMajor != "NULL"): momentMajor *= pltscale + if (momentMinor != "NULL"): momentMinor *= pltscale + if (momentM2C != "NULL"): momentM2C *= pltscale2 + if (momentM2S != "NULL"): momentM2S *= pltscale2 + if (momentM3 != "NULL"): momentM3 *= pltscale2 + if (momentM4 != "NULL"): momentM4 *= pltscale2 + # insert image metadata into table sqlLine = sqlUtility("INSERT INTO " + tableName + "(") @@ -265,10 +311,10 @@ sqlLine.group("psfWidMinor", psfFwhmMinor) sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) - sqlLine.group("momentMajor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1')) - sqlLine.group("momentMinor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2')) - sqlLine.group("momentM2C", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C')) - sqlLine.group("momentM2S", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S')) - sqlLine.group("momentM3", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3')) - sqlLine.group("momentM4", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4')) + sqlLine.group("momentMajor", momentMajor) + sqlLine.group("momentMinor", momentMinor) + sqlLine.group("momentM2C", momentM2C) + sqlLine.group("momentM2S", momentM2S) + sqlLine.group("momentM3", momentM3) + sqlLine.group("momentM4", momentM4) sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) @@ -374,6 +420,5 @@ # CZW: 20151110 This needs to get the deteff header as well. nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) - magref = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') - + magref = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') @@ -382,8 +427,10 @@ zpImage = self.scratchDb.getImageZeroPoint(self.imageIDs[ota]) - if zpImage < 0.0: + if (zpImage == "NULL"): zpImage = zp - detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + detectionThreshold = "NULL" + if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"): + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) sqlLine.group("frameID", str(self.expID)) Index: /trunk/ippToPsps/jython/diffbatch.py =================================================================== --- /trunk/ippToPsps/jython/diffbatch.py (revision 39304) +++ /trunk/ippToPsps/jython/diffbatch.py (revision 39305) @@ -56,5 +56,4 @@ "DF", None) - # gpc1Db.getDiffStageCmf(skychunk.dvoLabel,diffSkyFileID)) # get diff meta data @@ -195,10 +194,16 @@ ## -- XXX hard-wired platescale : 0.25 pltscale = 0.25 - self.psfFwhmMajor[num] = pltscale * self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MAJ') - self.psfFwhmMinor[num] = pltscale * self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MIN') - self.psfFWHM[num] = 0.5*(self.psfFwhmMajor[num] + self.psfFwhmMinor[num]) - - self.psfModelName[num] = self.getKeyValue(self.header[num], 'PSFMODEL') - self.psfModelID[num] = self.scratchDb.getFitModelID(self.psfModelName[num]) + self.psfFwhmMajor[num] = self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MAJ') + if (self.psfFwhmMajor[num] != "NULL"): self.psfFwhmMajor[num] *= pltscale + + self.psfFwhmMinor[num] = self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MIN') + if (self.psfFwhmMinor[num] != "NULL"): self.psfFwhmMinor[num] *= pltscale + + self.psfFWHM[num] = "NULL" + if (self.psfFwhmMajor[num] != "NULL") and (self.psfFwhmMinor[num] != "NULL"): + self.psfFWHM[num] = 0.5*(self.psfFwhmMajor[num] + self.psfFwhmMinor[num]) + + self.psfModelName[num] = self.getKeyValue(self.header[num], 'PSFMODEL') + self.psfModelID[num] = self.scratchDb.getFitModelID(self.psfModelName[num]) # drop the existing tables @@ -244,9 +249,10 @@ # Convert detectionThreshold to appropriate magnitudes # we have read these values from the appropriate locations in init() - zpImage = float(self.zpImage[num]) - expTime = float(self.expTime[num]) - magref = float(self.magref[num]) - - detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + zpImage = "NULL" if (self.zpImage[num] == "NULL") else float(self.zpImage[num]) + expTime = "NULL" if (self.expTime[num] == "NULL") else float(self.expTime[num]) + magref = "NULL" if (self.magref[num] == "NULL") else float(self.magref[num]) + + isNull = (zpImage == "NULL") or (expTime == "NULL") or (magref == "NULL") + detectionThreshold = "NULL" if isNull else magref + zpImage + 2.5 * math.log10(expTime) sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta" @@ -532,9 +538,10 @@ # we have read these values from the appropriate locations in init() - zpImage = float(self.zpImage[num]) - expTime = float(self.expTime[num]) - magref = float(self.magref[num]) - - detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + zpImage = "NULL" if (self.zpImage[num] == "NULL") else float(self.zpImage[num]) + expTime = "NULL" if (self.expTime[num] == "NULL") else float(self.expTime[num]) + magref = "NULL" if (self.magref[num] == "NULL") else float(self.magref[num]) + + isNull = (zpImage == "NULL") or (expTime == "NULL") or (magref == "NULL") + detectionThreshold = "NULL" if isNull else magref + zpImage + 2.5 * math.log10(expTime) nInjected = int(self.nInjected[num]) Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39304) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39305) @@ -228,7 +228,23 @@ pltscale2 = pltscale * pltscale - psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') - psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') - psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) + # NOTE: getKeyFloat returns the header value if it is a float, + # -999 if the value is missing and "NULL" if the value in the + # header is a NaN. any math operations below need to handle + # these cases correctly + + # NOTE 2: getKeyFloat NOW returns the header value if it is a + # float or "NULL" if the value is missing or if the value in + # the header is a NaN. any math operations below need to + # handle these cases correctly + + psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale + + psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN') + if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale + + psfFWHM = "NULL" + if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"): + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) psfmodel_name = self.getKeyValue(header, 'PSFMODEL') @@ -252,19 +268,12 @@ sqlLine.group("ippSkycalID", self.ippSkycalID[num]) sqlLine.group("stackMetaID", self.stackMetaID[num]) -# no astrometry calibration for forced warp -# sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) -# sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) -# sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) -# sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')); sqlLine.group("psfModelID", psfmodelID) sqlLine.group("psfFWHM", psfFWHM) sqlLine.group("psfWidMajor", psfFwhmMajor) sqlLine.group("psfWidMinor", psfFwhmMinor) - # sqlLine.group("psfFwhm_mean", self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')) - # sqlLine.group("psfFwhm_max", self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')) + + # sqlLine.group("photoZero", self.getKeyFloat(header, "%.8f", 'FPA.ZP')) sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) - - # sqlLine.group("photoZero", self.getKeyFloat(header, "%.8f", 'FPA.ZP')) sqlLine.group("photoZero", zpImage) Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 39304) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 39305) @@ -274,7 +274,13 @@ pltscale2 = pltscale * pltscale - psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') - psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') - psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) + psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale + + psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN') + if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale + + psfFWHM = "NULL" + if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"): + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) psfmodel_name = self.getKeyValue(header, 'PSFMODEL') @@ -283,18 +289,24 @@ ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + + astroscat = "NULL" + if (ast_cdx != "NULL") and (ast_cdy != "NULL"): + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) # Convert detectionThreshold to appropriate magnitudes - magref = self.magref[filter] - expTime = self.expTime[filter] - zp = self.zpImage[filter] - zpErr = self.zpError[filter] + magref = "NULL" if (self.magref[filter] == "NULL") else float(self.magref[filter]) + expTime = "NULL" if (self.expTime[filter] == "NULL") else float(self.expTime[filter]) + zp = "NULL" if (self.zpImage[filter] == "NULL") else float(self.zpImage[filter]) + zpErr = "NULL" if (self.zpError[filter] == "NULL") else float(self.zpError[filter]) # zp correction should comes from DVO (unless image is not in DVO) zpImage = self.scratchDb.getImageZeroPoint(stackID) - if zpImage < 0.0: + + if (zpImage == "NULL"): zpImage = zp - detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + detectionThreshold = "NULL" + if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"): + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) # insert stack metadata into table @@ -1040,4 +1052,9 @@ exptimeString = str(self.getKeyFloat(header, "%.5f", "EXPTIME")) + ## this is a serious problem: not sure we have to handle it. + if (exptimeString == "NULL"): + print "**** serious error: EXPTIME is NULL (missing or NAN)" + os._exit(4) + # generate the sql to do the necessary ops on the columns sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET ")