Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 38950) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 38951) @@ -199,4 +199,9 @@ def populateImageMetaTable(self, ota, header): + # does this chip exist in the DVO image table? (if NOT, then skip it) + if not self.scratchDb.haveThisChip(self.imageIDs[ota], self.sourceIDs[ota]): + self.logger.info("| %5s | ------------- Chip not in DVO : rejecting ------------ |" % ota) + return False + # the supplied 'header' matches this chip # self.header is the PHU header for this smf @@ -231,6 +236,8 @@ zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + zpImage = self.scratchDb.getImageZeroPoint(self.imageIDs[ota]) + # XXX zp correction should come from DVO - detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) + detectionThreshold = detectionThreshold + zpImage - 2.5 * math.log10(expTime) # insert image metadata into table @@ -246,5 +253,5 @@ sqlLine.group("astroScat", astroscat) sqlLine.group("photoScat", self.getKeyFloat(header, "%.8f", 'ZPT_ERR')) - sqlLine.group("photoZero", zp) + sqlLine.group("photoZero", zpImage) sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) @@ -629,5 +636,8 @@ # populate ImageMeta - self.populateImageMetaTable(chipname, header) + if not self.populateImageMetaTable(chipname, header): + self.logger.infoPair("skipping chip: ", chipname) + return False + self.updateImageID("ImageMeta_" + chipname, x, y) @@ -810,5 +820,4 @@ ota = x*10+y - self.logger.info("add psps unique p2 ids") self.updatePspsUniqueIDs("Detection_" + chipname, ota) @@ -901,22 +910,30 @@ if x==7 and y==7: continue + # skip chips which were not loaded ota = "XY%d%d" % (x, y) extension = ota + ".psf" if extension not in self.tablesLoaded: continue - if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1 + # skip chips which had a problem (e.g., not in DVO) + if not self.populatePspsTablesChip(ota, x, y, results, tables): + if self.skipBatch: + self.logger.error("fatal problem for exposure, skipping") + return False + continue + + otaCount = otaCount + 1 + self.populateImageDetEffMetaTablePart2(ota) + + extension = "Chip.psf" + if extension in self.tablesLoaded: + if not self.populatePspsTablesChip("Chip", 0, 0, results, tables): + print "--- skipped Chip.psf" if self.skipBatch: self.logger.error("fatal problem for exposure, skipping") return False - - self.populateImageDetEffMetaTablePart2(ota) - - - extension = "Chip.psf" - if extension in self.tablesLoaded: - if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 - if self.skipBatch: - self.logger.error("fatal problem for exposure, skipping") - return False + else: + otaCount = otaCount + 1 + + print "--- tried Chip.psf" # print totals @@ -931,4 +948,6 @@ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") + print "--- tried Chip.psf (2)" + # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort) if len(self.tablesToExport) == 1: Index: /trunk/ippToPsps/jython/dvo.py =================================================================== --- /trunk/ippToPsps/jython/dvo.py (revision 38950) +++ /trunk/ippToPsps/jython/dvo.py (revision 38951) @@ -146,5 +146,5 @@ self.importFits( path, - "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM", + "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS MCAL SECZ PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM", self.scratchDb.dvoImagesTable) self.logger.infoPair("Adding primary key to", self.scratchDb.dvoImagesTable) Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38950) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38951) @@ -133,5 +133,5 @@ self.logger.infoPair("found exp_id", str(self.expID[num])) self.logger.infoPair("found exp_time", str(self.expTime[num])) - self.logger.infoPair("found filter", str(self.filterNam[num])) + self.logger.infoPair("found filter", str(self.filterName[num])) self.logger.infoPair("found skycell_id", self.skycellName[num]) self.logger.infoPair("found tess_id", self.tessName[num]) @@ -216,15 +216,6 @@ self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - - # dump stuff to log self.logger.infoPair("add ID", "%d" % self.id) - -# self.logger.infoPair("Exp ID", self.expID) - - - - - ''' @@ -243,8 +234,10 @@ self.logger.infoPair("found header","ok") - # photoCalID = str(self.scratchDb.getPhotoCalID(header['IMAGEID'])) - - # XXX fix this: - photoCalID = GPC1.filtername.ForcedWarp + photoCalID = str(self.scratchDb.getPhotoCalID(header['IMAGEID'])) + + # PHOTCODE and FILTER are not in the header; reconstruct here + # photoCalID = "GPC1.%s.ForcedWarp" % self.filterName[num][0] + + zpImage = self.scratchDb.getImageZeroPoint(header['IMAGEID']) # XXX hard-wired platescale : 0.25 @@ -285,7 +278,9 @@ # sqlLine.group("psfFwhm_max", self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')) - # XXX: should psfTheta include the angle of the source chip in some way? sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) - sqlLine.group("photoZero", self.getKeyFloat(header, "%.8f", 'FPA.ZP')) + + # sqlLine.group("photoZero", self.getKeyFloat(header, "%.8f", 'FPA.ZP')) + sqlLine.group("photoZero", zpImage) + # photoColor -- how do I set this? it's also not set in stack meta sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')); Index: /trunk/ippToPsps/jython/mysql.py =================================================================== --- /trunk/ippToPsps/jython/mysql.py (revision 38950) +++ /trunk/ippToPsps/jython/mysql.py (revision 38951) @@ -8,5 +8,5 @@ from java.lang import * from java.sql import * -from xml.etree.ElementTree import ElementTree +# from xml.etree.ElementTree import ElementTree ''' @@ -355,6 +355,4 @@ print "--- " + str(e) os._exit(2) - -# stmt.close() return rs Index: /trunk/ippToPsps/jython/scratchdb.py =================================================================== --- /trunk/ippToPsps/jython/scratchdb.py (revision 38950) +++ /trunk/ippToPsps/jython/scratchdb.py (revision 38951) @@ -214,4 +214,29 @@ ''' + Gets photcode (aka photoCalID from dvo table) + ''' + def getImageZeroPoint(self, externID): + + zeroPoint = -999 + + sql = "SELECT MCAL, SECZ, C_LAM, K FROM %s AS a JOIN %s as b WHERE EXTERN_ID = %s AND a.PHOTCODE = b.CODE" % (self.dvoImagesTable, self.dvoPhotcodesTable, (externID)) + rs = self.executeQuery(sql) + if not rs: + print "missing result set for imageID query" + os._exit(2) + + if not rs.first(): + self.logger.infoPair("no zero point; image is missing, imageID: ", imageID) + return zeroPoint + + Mcal = rs.getFloat(1) + airmass = rs.getFloat(2) + Clam = rs.getFloat(3) + Klam = rs.getFloat(4) + + zeroPoint = 0.001*Clam + Klam*(airmass - 1.0) - Mcal + return zeroPoint + + ''' Updates a table with filterID grabbed from Filter init table ''' @@ -449,33 +474,22 @@ " AND SOURCE_ID = " + str(sourceID) - try: - rs = self.executeQuery(sql) - - if not rs: - print "missing result set for imageID query" - raise - - # if no returned rows, return an error - if not rs.next(): - self.logger.infoPair("DVO lacks imageID ", imageID) - return False - - if rs.getInt(1) == int(imageID): - # print "DVO has imageID ", imageID - return True - - else: - self.logger.error("DVO has wrong imageID? ") - self.logger.error("imageID: ", imageID) - self.logger.error("EXTERN_ID: ", rs.getInt(1)) - self.logger.infoPair("sql: ", sql) - raise - - except: - print "DVO imageID query failed ", imageID - print "sql: ", sql + rs = self.executeQuery(sql) + if not rs: + print "missing result set for imageID query" raise - # how can I reach this? + # if no returned rows, return an error + if not rs.next(): + self.logger.infoPair("DVO lacks imageID ", imageID) + return False + + if rs.getInt(1) != int(imageID): + self.logger.error("DVO has wrong imageID? ") + self.logger.error("imageID: ", imageID) + self.logger.error("EXTERN_ID: ", rs.getInt(1)) + self.logger.infoPair("sql: ", sql) + os._exit(2) + + # print "DVO has imageID ", imageID return True @@ -497,4 +511,6 @@ EXTERN_ID INT, \ FLAGS INT, \ + MCAL FLOAT, \ + SECZ FLOAT, \ PHOTCODE SMALLINT, \ XPIX_SYS_ERR FLOAT, \ Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 38950) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 38951) @@ -261,6 +261,8 @@ zpErr = self.getKeyFloat(header, "%.8f", 'ZPT_ERR') + zpImage = self.scratchDb.getImageZeroPoint(stackID) + # XXX zp correction should come from DVO - detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) + detectionThreshold = detectionThreshold + zpImage - 2.5 * math.log10(expTime) # insert stack metadata into table @@ -287,5 +289,5 @@ # sqlLine.group("psfFwhm_max", fwhm_maj_uq) sqlLine.group("astroScat", astroscat) - sqlLine.group("photoZero", zp) + sqlLine.group("photoZero", zpImage) sqlLine.group("photoScat", zpErr) sqlLine.group("nAstroRef", header['NASTRO'])