Index: trunk/ippToPsps/jython/batch.py =================================================================== --- trunk/ippToPsps/jython/batch.py (revision 39093) +++ trunk/ippToPsps/jython/batch.py (revision 39094) @@ -600,4 +600,5 @@ # replace nulls and empty fields with weird PSPS -999 pseudo-null + # this should be sufficient to handle all NANs (which go in as NULL) _table = stilts.tpipe(_table, cmd='replaceval "" -999 *') self.logger.infoPair('regex', regex) Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 39093) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 39094) @@ -241,5 +241,5 @@ # XXX zp correction should come from DVO - detectionThreshold = detectionThreshold + zpImage - 2.5 * math.log10(expTime) + detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime) # insert image metadata into table @@ -489,5 +489,5 @@ else: sqlLine.group("pltScale", "0.257") - sqlLine.group("posAngle", "-999") + # sqlLine.group("posAngle", "-999") sqlLine.group("raErr", "X_PSF_SIG * 0.257") sqlLine.group("decErr", "Y_PSF_SIG * 0.257") Index: trunk/ippToPsps/jython/diffbatch.py =================================================================== --- trunk/ippToPsps/jython/diffbatch.py (revision 39093) +++ trunk/ippToPsps/jython/diffbatch.py (revision 39094) @@ -5,4 +5,5 @@ import glob import time +import math import stilts import re @@ -154,18 +155,18 @@ self.logger.infoPair("TessName ",tessName) - # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL if self.tessName[num] == 'RINGS.V3': - # skycell is, eg "skycell.1133.081" - # 0123456789012345 + # skycell is, eg "skycell.1133.081" + # 0123456789012345 self.projectionID[num] = self.skycellName[num][8:12] - self.skycellID[num] = self.skycellName[num][13:] - self.skycellID[num] = self.skycellName[num][8:] + self.skycellID[num] = self.skycellName[num][13:] else: - # skycell is, eg "skycell.081" - # 01234567890 + # skycell is, eg "skycell.081" + # 01234567890 self.projectionID[num] = 0 - self.skycellID[num] = self.skycellName[num][8:11] - self.skycellID[num]=self.skycellName[num][8:] - if self.skycellID[num] == "": + self.skycellID[num] = self.skycellName[num][8:] + + if self.projectionID[num] == "": + self.projectionID[num] = -1 + if self.skycellID[num] == "": self.skycellID[num] = -1 @@ -183,5 +184,4 @@ self.psfModelID[num] = self.scratchDb.getFitModelID(self.psfModelName[num]) - # drop the existing tables @@ -191,5 +191,5 @@ self.dropTableVerbose("DiffDetEffMeta_"+str(num)) - if not self.analysisVer[num]: self.analysisVer[num] = -999 + # if not self.analysisVer[num]: self.analysisVer[num] = -999 self.logger.infoPair("getPhotoCalID",diffSkyFileID) @@ -222,4 +222,12 @@ self.logger.infoPair("Proccesing table", "DiffMeta_"+str(num)) + header = self.header[num] + + # Convert detectionThreshold to appropriate magnitudes + detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') + zpImage = self.getKeyFloat(header, "%.8f", 'FPA.ZP') + + detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(float(self.expTime[num])) + sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta" try: self.scratchDb.execute(sql) @@ -234,4 +242,5 @@ sqlLine.group("negImageID", self.negImageID[num]); sqlLine.group("ippDiffID", self.ippDiffID[num]); + sqlLine.group("tessID", str(self.tessID[num])) sqlLine.group("projectionID", str(self.projectionID[num])) sqlLine.group("skycellID", str(self.skycellID[num])) @@ -239,4 +248,5 @@ sqlLine.group("analysisVer", str(self.analysisVer[num])); sqlLine.group("md5sum", str(self.md5sum[num])) + sqlLine.group("detectionThreshold", detectionThreshold) sqlLine.group("expTime", self.expTime[num]); sqlLine.group("psfModelID", self.psfModelID[num]) @@ -244,25 +254,26 @@ sqlLine.group("psfWidMajor", self.psfFwhmMajor[num]) sqlLine.group("psfWidMinor", self.psfFwhmMinor[num]) - sqlLine.group("kernel", self.getKeyValue(self.header[num], 'SUBTRACTION.KERNEL')) - sqlLine.group("mode", self.getKeyValue(self.header[num], 'SUBTRACTION.MODE')) - sqlLine.group("psfTheta", self.getKeyFloat(self.header[num], "%.8f", 'ANGLE')) - sqlLine.group("numStamps", self.getKeyValue(self.header[num], 'SUBTRACTION.STAMPS')) - sqlLine.group("stampDevMean", self.getKeyFloat(self.header[num], "%.8e", 'SUBTRACTION.DEV.MEAN')) - sqlLine.group("stampDevRMS", self.getKeyFloat(self.header[num], "%.8e", 'SUBTRACTION.DEV.RMS')) - sqlLine.group("normalization", self.getKeyFloat(self.header[num], "%.8e", 'SUBTRACTION.NORM')) - sqlLine.group("convolveMax", self.getKeyFloat(self.header[num], "%.8e", 'SUBTRACTION.CONV.MAX')) - sqlLine.group("deconvolveMax", self.getKeyFloat(self.header[num], "%.8e", 'SUBTRACTION.DECONV.MAX')) - sqlLine.group("ctype1", self.getKeyValue(self.header[num], 'CTYPE1')); - sqlLine.group("ctype2", self.getKeyValue(self.header[num], 'CTYPE2')); - sqlLine.group("crval1", self.getKeyFloat(self.header[num], "%.8f", 'CRVAL1')); - sqlLine.group("crval2", self.getKeyFloat(self.header[num], "%.8f", 'CRVAL2')); - sqlLine.group("crpix1", self.getKeyFloat(self.header[num], "%.8f", 'CRPIX1')); - sqlLine.group("crpix2", self.getKeyFloat(self.header[num], "%.8f", 'CRPIX2')); - sqlLine.group("cdelt1", self.getKeyFloat(self.header[num], "%.8e", 'CDELT1')); - sqlLine.group("cdelt2", self.getKeyFloat(self.header[num], "%.8e", 'CDELT2')); - sqlLine.group("pc001001", self.getKeyFloat(self.header[num], "%.8e", 'PC001001')); - sqlLine.group("pc001002", self.getKeyFloat(self.header[num], "%.8e", 'PC001002')); - sqlLine.group("pc002001", self.getKeyFloat(self.header[num], "%.8e", 'PC002001')); - sqlLine.group("pc002002", self.getKeyFloat(self.header[num], "%.8e", 'PC002002')); + + sqlLine.group("kernel", self.getKeyValue(header, 'SUBTRACTION.KERNEL')) + sqlLine.group("mode", self.getKeyValue(header, 'SUBTRACTION.MODE')) + sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) + sqlLine.group("numStamps", self.getKeyValue(header, 'SUBTRACTION.STAMPS')) + sqlLine.group("stampDevMean", self.getKeyFloat(header, "%.8e", 'SUBTRACTION.DEV.MEAN')) + sqlLine.group("stampDevRMS", self.getKeyFloat(header, "%.8e", 'SUBTRACTION.DEV.RMS')) + sqlLine.group("normalization", self.getKeyFloat(header, "%.8e", 'SUBTRACTION.NORM')) + sqlLine.group("convolveMax", self.getKeyFloat(header, "%.8e", 'SUBTRACTION.CONV.MAX')) + sqlLine.group("deconvolveMax", self.getKeyFloat(header, "%.8e", 'SUBTRACTION.DECONV.MAX')) + sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')); sql = sqlLine.make(") VALUES ( ", ")") @@ -323,4 +334,6 @@ sqlLine.group("filterID", str(self.filterID[num])) sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("tessID", str(self.tessID[num])) + sqlLine.group("projectionID", str(self.projectionID[num])) sqlLine.group("skycellID", str(self.skycellID[num])) sqlLine.group("expTime", str(self.expTime[num])) @@ -336,6 +349,6 @@ sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)") else: - sqlLine.group("pltScale", "0.257") - sqlLine.group("posAngle", "-999") + sqlLine.group("pltScale", "0.25") + # sqlLine.group("posAngle", "-999") sqlLine.group("raErr", "X_PSF_SIG * 0.25") sqlLine.group("decErr", "Y_PSF_SIG * 0.25") @@ -433,5 +446,4 @@ self.logger.infoPair("Dropping row column from", "DiffDetection table") self.scratchDb.dropColumn("DiffDetection_"+str(num), "row") - ''' @@ -494,22 +506,16 @@ header = self.header[num] - # CZW: this is broken, and can't get the zeropoint - # zpImage = self.scratchDb.getImageZeroPoint(diffSkyFileID) - zpImage = 0.0 - - if zpImage < 0.0: - zpImage = 0.0 + zpImage = self.getKeyFloat(header,"%.8f","FPA.ZP") + if zpImage == 0.0: self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage)) - magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage self.logger.infoPair("Found zeropoint and set magref", str(magref)) - self.logger.infoPair("CZW If that magref is -5.804804, then it's reading from the PHU, not the Deteff extension.","In a test, I mean.") + sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(") sqlLine.group("diffImageID", str(diffSkyFileID)) sqlLine.group("magref", magref) sql = sqlLine.make(") VALUES ( ", ")") - try: self.scratchDb.execute(sql) Index: trunk/ippToPsps/jython/forcedgalaxybatch.py =================================================================== --- trunk/ippToPsps/jython/forcedgalaxybatch.py (revision 39093) +++ trunk/ippToPsps/jython/forcedgalaxybatch.py (revision 39094) @@ -147,4 +147,7 @@ photcode = self.scratchDb.getPhotcodeByName(photcodeName) + # XXX hard-wired platescale : 0.25 + pltscale = 0.25 + # This is going to need to join elsewhere to get all the fields? # XforcedSummaryID, Ra, Dec, RaErr, DecErr @@ -154,8 +157,8 @@ sqlLine.group("a." + filter[1] + "ippDetectID", "b.DET_ID") sqlLine.group("a." + filter[1] + "stackImageID", "b.IMAGE_ID") - sqlLine.group("a." + filter[1] + "GalMajor", "b.MAJOR_AXIS") - sqlLine.group("a." + filter[1] + "GalMajorErr", "b.MAJOR_AXIS_ERR") - sqlLine.group("a." + filter[1] + "GalMinor", "b.MINOR_AXIS") - sqlLine.group("a." + filter[1] + "GalMinorErr", "b.MINOR_AXIS_ERR") + sqlLine.group("a." + filter[1] + "GalMajor", "b.MAJOR_AXIS * " + str(pltscale)) + sqlLine.group("a." + filter[1] + "GalMajorErr", "b.MAJOR_AXIS_ERR * " + str(pltscale)) + sqlLine.group("a." + filter[1] + "GalMinor", "b.MINOR_AXIS * " + str(pltscale)) + sqlLine.group("a." + filter[1] + "GalMinorErr", "b.MINOR_AXIS_ERR * " + str(pltscale)) sqlLine.group("a." + filter[1] + "GalMag", "b.MAG") sqlLine.group("a." + filter[1] + "GalMagErr", "b.MAG_ERR") Index: trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39093) +++ trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39094) @@ -269,5 +269,4 @@ sqlLine.group("photoZero", zpImage) - # photoColor -- how do I set this? it's also not set in stack meta sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')); sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')); @@ -482,16 +481,16 @@ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("flxR5", "APER_FLUX_3") - sqlLine.group("flxR5Err", "APER_FLUX_ERR_3" ) - sqlLine.group("flxR5Std", "APER_FLUX_STDEV_3") - sqlLine.group("flxR5Fill", "APER_FILL_3") - sqlLine.group("flxR6", "APER_FLUX_4") - sqlLine.group("flxR6Err", "APER_FLUX_ERR_4" ) - sqlLine.group("flxR6Std", "APER_FLUX_STDEV_4") - sqlLine.group("flxR6Fill", "APER_FILL_4") - sqlLine.group("flxR7", "APER_FLUX_5") - sqlLine.group("flxR7Err", "APER_FLUX_ERR_5" ) - sqlLine.group("flxR7Std", "APER_FLUX_STDEV_5") - sqlLine.group("flxR7Fill", "APER_FILL_5") + sqlLine.group("flxR5", "APER_FLUX_3") + sqlLine.group("flxR5Err", "APER_FLUX_ERR_3" ) + sqlLine.group("flxR5Std", "APER_FLUX_STDEV_3") + sqlLine.group("flxR5Fill", "APER_FILL_3") + sqlLine.group("flxR6", "APER_FLUX_4") + sqlLine.group("flxR6Err", "APER_FLUX_ERR_4" ) + sqlLine.group("flxR6Std", "APER_FLUX_STDEV_4") + sqlLine.group("flxR6Fill", "APER_FILL_4") + sqlLine.group("flxR7", "APER_FLUX_5") + sqlLine.group("flxR7Err", "APER_FLUX_ERR_5" ) + sqlLine.group("flxR7Std", "APER_FLUX_STDEV_5") + sqlLine.group("flxR7Fill", "APER_FILL_5") sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -501,16 +500,18 @@ self.logger.infoPair("adding objID, etc from", fMeasTableName) sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, " + fMeasTableName + " as b SET ") - sqlLine.group("a.objID","b.objID") + sqlLine.group("a.objID", "b.objID") sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid") - sqlLine.group("a.detectID","b.detectID") - sqlLine.group("a.ippObjID","b.ippObjID") - sqlLine.group("a.filterID","b.filterID") - sqlLine.group("a.surveyID","b.surveyID") - sqlLine.group("a.forcedWarpID","b.forcedWarpID") - sqlLine.group("a.randomWarpID","b.randomWarpID") - sqlLine.group("a.skycellID","b.skycellID") - sqlLine.group("a.dvoRegionID","b.dvoRegionID") - sqlLine.group("a.obsTime","b.obsTime") - sql = sqlLine.makeEquals("Where a.ippDetectID = b.ippDetectID") + sqlLine.group("a.detectID", "b.detectID") + sqlLine.group("a.ippObjID", "b.ippObjID") + sqlLine.group("a.filterID", "b.filterID") + sqlLine.group("a.surveyID", "b.surveyID") + sqlLine.group("a.forcedWarpID", "b.forcedWarpID") + sqlLine.group("a.randomWarpID", "b.randomWarpID") + sqlLine.group("a.tessID", "b.tessID") + sqlLine.group("a.projectionID", "b.projectionID") + sqlLine.group("a.skycellID", "b.skycellID") + sqlLine.group("a.dvoRegionID", "b.dvoRegionID") + sqlLine.group("a.obsTime", "b.obsTime") + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID") self.scratchDb.execute(sql) @@ -539,23 +540,23 @@ sqlLine.group("lensObjSmearX12", "LENS_X12_SM_OBJ") sqlLine.group("lensObjSmearX22", "LENS_X22_SM_OBJ") - sqlLine.group("lensObjSmearE1", "LENS_E1_SM_OBJ") - sqlLine.group("lensObjSmearE2", "LENS_E2_SM_OBJ") + sqlLine.group("lensObjSmearE1", "LENS_E1_SM_OBJ") + sqlLine.group("lensObjSmearE2", "LENS_E2_SM_OBJ") sqlLine.group("lensObjShearX11", "LENS_X11_SH_OBJ") sqlLine.group("lensObjShearX12", "LENS_X12_SH_OBJ") sqlLine.group("lensObjShearX22", "LENS_X22_SH_OBJ") - sqlLine.group("lensObjShearE1", "LENS_E1_SH_OBJ") - sqlLine.group("lensObjShearE2", "LENS_E2_SH_OBJ") + sqlLine.group("lensObjShearE1", "LENS_E1_SH_OBJ") + sqlLine.group("lensObjShearE2", "LENS_E2_SH_OBJ") sqlLine.group("lensPSFSmearX11", "LENS_X11_SM_PSF") sqlLine.group("lensPSFSmearX12", "LENS_X12_SM_PSF") sqlLine.group("lensPSFSmearX22", "LENS_X22_SM_PSF") - sqlLine.group("lensPSFSmearE1", "LENS_E1_SM_PSF") - sqlLine.group("lensPSFSmearE2", "LENS_E2_SM_PSF") + sqlLine.group("lensPSFSmearE1", "LENS_E1_SM_PSF") + sqlLine.group("lensPSFSmearE2", "LENS_E2_SM_PSF") sqlLine.group("lensPSFShearX11", "LENS_X11_SH_PSF") sqlLine.group("lensPSFShearX12", "LENS_X12_SH_PSF") sqlLine.group("lensPSFShearX22", "LENS_X22_SH_PSF") - sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF") - sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF") - sqlLine.group("psfE1", "LENS_E1_PSF") - sqlLine.group("psfE2", "LENS_E2_PSF") + sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF") + sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF") + sqlLine.group("psfE1", "LENS_E1_PSF") + sqlLine.group("psfE2", "LENS_E2_PSF") sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -565,17 +566,19 @@ self.logger.infoPair("inserting objID,etc from", fMeasTableName) sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, " + fMeasTableName + " as b SET ") - sqlLine.group("a.objID","b.objID") - sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid") - sqlLine.group("a.detectID","b.detectID") - sqlLine.group("a.ippObjID","b.ippObjID") - sqlLine.group("a.filterID","b.filterID") - sqlLine.group("a.surveyID","b.surveyID") - sqlLine.group("a.forcedWarpID","b.forcedWarpID") - sqlLine.group("a.randomWarpID","b.randomWarpID") - sqlLine.group("a.skycellID","b.skycellID") - sqlLine.group("a.dvoRegionID","b.dvoRegionID") - sqlLine.group("a.obsTime","b.obsTime") - - sql = sqlLine.makeEquals("Where a.ippDetectID = b.ippDetectID") + sqlLine.group("a.objID", "b.objID") + sqlLine.group("a.uniquePspsFWid", "b.uniquePspsFWid") + sqlLine.group("a.detectID", "b.detectID") + sqlLine.group("a.ippObjID", "b.ippObjID") + sqlLine.group("a.filterID", "b.filterID") + sqlLine.group("a.surveyID", "b.surveyID") + sqlLine.group("a.forcedWarpID", "b.forcedWarpID") + sqlLine.group("a.randomWarpID", "b.randomWarpID") + sqlLine.group("a.tessID", "b.tessID") + sqlLine.group("a.projectionID", "b.projectionID") + sqlLine.group("a.skycellID", "b.skycellID") + sqlLine.group("a.dvoRegionID", "b.dvoRegionID") + sqlLine.group("a.obsTime", "b.obsTime") + + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID") self.scratchDb.execute(sql) Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 39093) +++ trunk/ippToPsps/jython/stackbatch.py (revision 39094) @@ -275,5 +275,5 @@ zpImage = zp - detectionThreshold = detectionThreshold + zpImage - 2.5 * math.log10(expTime) + detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime) # insert stack metadata into table @@ -345,5 +345,5 @@ self.scratchDb.addRowCountColumn("StackObjectThin", "row") self.logger.infoPair("Adding unique psps ids to", "StackObjectThin") - self.generateStackDetectRowIDsAndPspsUniqueIDs() + self.generatePspsUniqueIDs() self.logger.infoPair("Dropping row column from", "StackObjectThin table") self.scratchDb.dropColumn("StackObjectThin", "row") @@ -553,11 +553,10 @@ # self.scratchDb.createIndex(tablename, "objID") - #objid - #uniquepsps id + # objid + # uniquepsps id # ippobjid # randomstackobjid - # stackdetectrowid # primarydetection - #bestdetection + # bestdetection #g / r / i /z / y @@ -951,4 +950,9 @@ maxRadius = 5 + # XXX : by setting these to -999 here, the values are then + # modified downstream to apply exposure time and zero point + # factors (or platescale terms). this means they will not + # have the correct -999 values in the end. + # properly -999 these to start with. the default should take # care of this, but does not? @@ -1286,5 +1290,5 @@ self.scratchDb.execute(sql) - def generateStackDetectRowIDsAndPspsUniqueIDs(self): + def generatePspsUniqueIDs(self): sql = "UPDATE StackObjectThin set uniquePspsSTid = (("+str(self.batchID)+"*1000000000 ) + row)" self.scratchDb.execute(sql) @@ -1349,5 +1353,5 @@ self.scratchDb.addColumn(table, filter + "haveData", "tinyint") - fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, StackDetectRowID, " + fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, " for i in range(len(self.filters)):