Index: /trunk/ippToPsps/config/tables.IN.vot =================================================================== --- /trunk/ippToPsps/config/tables.IN.vot (revision 39102) +++ /trunk/ippToPsps/config/tables.IN.vot (revision 39103) @@ -540,6 +540,6 @@ - 1RINGS.V3Tessellation for the 3pi survey, ex pole regions - 2CNP.LAPTessellation for the 3pi survey pole regions (LOCAL) + 1CNP.LAPTessellation for the 3pi survey pole regions (LOCAL) + 2RINGS.V3Tessellation for the 3pi survey, ex pole regions 3MD01.V3Tessellation for MD01 field (LOCAL) 4MD02.V3Tessellation for MD02 field (LOCAL) Index: /trunk/ippToPsps/jython/diffbatch.py =================================================================== --- /trunk/ippToPsps/jython/diffbatch.py (revision 39102) +++ /trunk/ippToPsps/jython/diffbatch.py (revision 39103) @@ -77,4 +77,7 @@ self.md5sum = {} self.expTime = {} + self.zpImage = {} + self.magref = {} + self.nInjected = {} self.obsTime = {} self.posFits = {} @@ -150,4 +153,19 @@ self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0) + # grap zpImage from the PHU header + self.zpImage[num] = self.getKeyFloat(self.header[num],"%.8f","FPA.ZP") + + deteffHeader = self.posFits[num].findAndReadHeader("SkyChip.deteff", self.config.test) + if not deteffHeader: + if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) + return False + + # grab the detection threshold reference magnitude + self.magref[num] = self.getKeyFloat(deteffHeader, "%.8f", "DETEFF.MAGREF") + + self.nInjected[num] = self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM') + + # self.logger.infoPair("Found zeropoint and set magref", str(magref)) + tessName = self.tessName[num] self.logger.infoPair("TessName ", tessName) @@ -225,8 +243,10 @@ # Convert detectionThreshold to appropriate magnitudes - detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') - zpImage = self.getKeyFloat(header, "%.8f", 'FPA.ZP') - - detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(float(self.expTime[num])) + # we have read these values from the appropriate locations in init() + zpImage = float(self.zpImage[num]) + expTime = float(self.expTime[num]) + magref = float(self.magref[num]) + + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta" @@ -506,28 +526,16 @@ self.logger.infoPair("Created tmp table:", ippTableName) - header = self.header[num] - zpImage = self.getKeyFloat(header,"%.8f","FPA.ZP") - expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') - - if zpImage == 0.0: - self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage)) - -# CZW 20151110 This is broken due to findAndReadHeader being broken. -# deteffHeader = self.posFits[num].findAndReadHeader("SkyChip.deteff", self.config.test) -# if not deteffHeader: -# if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) -# return False -# This next line is a work around. - - deteffHeader = header - magref = self.getKeyFloat(deteffHeader,"%.8f","DETEFF.MAGREF") + zpImage + 2.5 * math.log10(expTime) - self.logger.infoPair("Found zeropoint and set magref", str(magref)) - - nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) - + # we have read these values from the appropriate locations in init() + zpImage = float(self.zpImage[num]) + expTime = float(self.expTime[num]) + magref = float(self.magref[num]) + + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + + nInjected = int(self.nInjected[num]) sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(") sqlLine.group("diffImageID", str(diffSkyFileID)) - sqlLine.group("magref", magref) + sqlLine.group("magref", detectionThreshold) sqlLine.group("nInjected", nInjected) sql = sqlLine.make(") VALUES ( ", ")") @@ -574,6 +582,4 @@ self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID") - #self.populateDiffMeta() - # dictionary objects to hold imageIDs for later self.imageIDs = {} Index: /trunk/ippToPsps/jython/fits.py =================================================================== --- /trunk/ippToPsps/jython/fits.py (revision 39102) +++ /trunk/ippToPsps/jython/fits.py (revision 39103) @@ -6,4 +6,6 @@ import hashlib from subprocess import call, PIPE, Popen + +import tempfile ''' @@ -121,4 +123,16 @@ ''' + Find and read a header extension. + ''' + def findAndReadHeader(self, name, VERBOSE): + + if False: + status = self.findAndReadHeaderRaw(name, VERBOSE) + return status + + status = self.findAndReadHeaderOhana(name, VERBOSE) + return status + + ''' Finds and reads a header extension. @@ -132,5 +146,5 @@ through STILTS ''' - def findAndReadHeader(self, name, VERBOSE): + def findAndReadHeaderRaw(self, name, VERBOSE): found = False @@ -167,4 +181,30 @@ ''' + Find and read a header extension. + ''' + def findAndReadHeaderOhana(self, name, VERBOSE): + + cmd = "fhead -n %s %s" % (name, self.originalPath) + p = Popen(cmd, shell=True, stdout=PIPE) + output = p.communicate()[0] + + header = {} + + # split the output header bytes into key/value pairs for each 80-byte line + for i in range(0, len(output), 80): + # print "line %d : %s" % (i, output[i:i+80]) + record = output[i:i+80] + + match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#\(\)\,]+)\'*\\/*', record) + + if match: + param = match.group(2) + value = match.group(3).strip() + if value == "NaN": value = "NULL" + header[param] = value + + return header + + ''' Reads FITS header and stores all fields in a dictionary object ''' Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 39102) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 39103) @@ -65,4 +65,9 @@ self.md5sum = {} self.stackEpochs = {} + self.expTime = {} + self.zpImage = {} + self.zpError = {} + self.magref = {} + self.nInjected = {} # we have two sets of imageIDs here: @@ -116,4 +121,16 @@ self.stackEpochs[filter] = self.getKeyFloat(header, "%.10f", 'MJD-OBS') + + self.expTime[filter] = self.getKeyFloat(header, "%.8f", 'EXPTIME') + self.zpImage[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + self.zpError[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_ERR') + + deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test) + if not deteffHeader: + if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) + return False + + self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') + self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) ## MEH -- md5sum test @@ -268,18 +285,8 @@ # Convert detectionThreshold to appropriate magnitudes - # Grab this from the appropriate extension. - -# CZW 20151110 This is broken. -# deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test) -# if not deteffHeader: -# if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) -# return False -# CZW This is the workaround - - deteffHeader = header - detectionThreshold = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') - expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') - zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') - zpErr = self.getKeyFloat(header, "%.8f", 'ZPT_ERR') + magref = self.magref[filter] + expTime = self.expTime[filter] + zp = self.zpImage[filter] + zpErr = self.zpError[filter] # zp correction should comes from DVO (unless image is not in DVO) @@ -288,6 +295,6 @@ zpImage = zp - detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime) - + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + # insert stack metadata into table sqlLine = sqlUtility("INSERT INTO " + tablename + " (") @@ -305,5 +312,5 @@ sqlLine.group("md5sum", str(self.md5sum[filter])) sqlLine.group("detectionThreshold", detectionThreshold) - sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) + sqlLine.group("expTime", expTime) sqlLine.group("psfModelID", psfmodelID) sqlLine.group("psfFWHM", psfFWHM) @@ -481,5 +488,5 @@ header = self.headerSet[filter] - exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") + exptime = self.expTime[filter] expTimeString = str(exptime) @@ -850,6 +857,6 @@ ## XXX is this statement true: is FPA.ZP + 2.5log(exptime) applied to PETRO_MAG? - hdrZP = "%.5f" % float(header['FPA.ZP']) - magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME']))) + hdrZP = str(self.zpImage[filter]) + # magtime = "%.5f" % (2.5*math.log10(self.expTime[filter])) # XXX hard-wired platescale : 0.25 (could get this from header, but stacks have constant plate scale @@ -1135,5 +1142,5 @@ field = "EXP_%04d" % input - expTime = self.getKeyFloat(header, "%6.3f", field) + expTime = self.getKeyFloat(header, "%6.3f", field) # exptime of INPUT image, not stack field = "AIR_%04d" % input @@ -1183,41 +1190,23 @@ except: pass - - header = self.headerSet[filter] - # Convert detectionThreshold to appropriate magnitudes - # Grab this from the appropriate extension. -# CZW 20151110 This is broken. -# deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test) -# if not deteffHeader: -# if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) -# return False -# CZW This is the workaround - - deteffHeader = header - detectionThreshold = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') - expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') - zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') - zpErr = self.getKeyFloat(header, "%.8f", 'ZPT_ERR') - - # zp correction should comes from DVO (unless image is not in DVO) - zpImage = self.scratchDb.getImageZeroPoint(stackID) + magref = self.magref[filter] + expTime = self.expTime[filter] + zp = self.zpImage[filter] + zpErr = self.zpError[filter] + + # zp correction should comes from DVO (unless image is not in DVO) + zpImage = self.scratchDb.getImageZeroPoint(stackImageID) if zpImage < 0.0: zpImage = zp - detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime) - nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) - - zpImage = self.scratchDb.getImageZeroPoint(stackImageID) - if zpImage < 0.0: - zpImage = 0.0 - magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage + detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + nInjected = self.nInjected[filter] sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(") sqlLine.group("stackImageID", str(stackImageID)) - sqlLine.group("magref", magref) + sqlLine.group("magref", detectionThreshold) sqlLine.group("nInjected", nInjected) sql = sqlLine.make(") VALUES ( ", ")") - try: self.scratchDb.execute(sql) Index: /trunk/ippToPsps/jython/testCode.py =================================================================== --- /trunk/ippToPsps/jython/testCode.py (revision 39102) +++ /trunk/ippToPsps/jython/testCode.py (revision 39103) @@ -96,4 +96,6 @@ print "words[1]: " + words[1] + + # self.connectMysql(argv) # self.importIppTables() @@ -118,2 +120,20 @@ testcode.run(sys.argv) except: pass + + # XXX a test of fits header reading: + ## XX myFits = Fits(self.logger, self.config, sys.argv[2]) + ## XX + ## XX for ix in range(7, -1, -1): + ## XX for iy in range(7, -1, -1): + ## XX if (ix == 0) and (iy == 0): continue + ## XX if (ix == 7) and (iy == 0): continue + ## XX if (ix == 0) and (iy == 7): continue + ## XX if (ix == 7) and (iy == 7): continue + ## XX extname = "XY%d%d.hdr" % (ix, iy) + ## XX + ## XX header = myFits.findAndReadHeader(extname, False) + ## XX print "read " + extname + ## XX + ## XX os._exit(3) + + ## the above works in loader.init()