Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 31503) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 31504) @@ -6,6 +6,9 @@ from java.lang import * from java.sql import * + from batch import Batch from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb import logging.config @@ -19,22 +22,35 @@ Constructor ''' - def __init__(self, logger, camID, inputFile, test=False, useFullTables=False): + def __init__(self, + logger, + gpc1Db, + ippToPspsDb, + scratchDb, + camID, + inputFile, + test=False, + useFullTables=False): + super(DetectionBatch, self).__init__( logger, + gpc1Db, + ippToPspsDb, + scratchDb, + camID, "detection", inputFile, - "MD04", - useFullTables) # TODO + "MD04", # TODO #"3PI") # TODO - - self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'") - - meta = self.gpc1Db.getCameraStageMeta(camID) - - self.expID = meta[0]; - self.expName = meta[1]; - self.distGroup = meta[2]; - - self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup)) + useFullTables) + + if not self.everythingOK: return + + # meta data to the log + self.logger.info("New Detection Batch:") + self.logger.info("Cam ID: %d" % self.id) + self.logger.info("file: %s" % inputFile) + self.logger.info("Exp ID: %d" % self.expID) + self.logger.info("Exp name: %s" % self.expName) + self.logger.info("Distribution group: %s" % self.distGroup) # create an output filename, which is {expID}.FITS @@ -167,5 +183,5 @@ ," + self.header['PCA2X0Y2'] + " \ )" - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID)) @@ -184,5 +200,5 @@ self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" - try: self.scratchDb.stmt.execute(sql) + try: self.scratchDb.execute(sql) except: pass @@ -310,9 +326,9 @@ )" - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) - self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) + if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -327,5 +343,5 @@ self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE Detection" - try: self.scratchDb.stmt.execute(sql) + try: self.scratchDb.execute(sql) except: pass @@ -375,10 +391,9 @@ ,EXT_NSIGMA \ FROM " + ota + "_psf" - - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) # set obsTime sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) self.scratchDb.updateAllRows(tableName, "historyModNum", "0") @@ -387,5 +402,5 @@ self.scratchDb.updateFilterID(tableName, self.filter) - # now delete bad flux + # now delete bad flux and bad chip positions self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux") self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU") @@ -401,5 +416,5 @@ self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject" - try: self.scratchDb.stmt.execute(sql) + try: self.scratchDb.execute(sql) except: pass @@ -415,5 +430,5 @@ ,surveyID \ FROM Detection_" + ota - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) @@ -429,5 +444,5 @@ self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor" - try: self.scratchDb.stmt.execute(sql) + try: self.scratchDb.execute(sql) except: pass @@ -443,5 +458,5 @@ ,filterID \ FROM Detection_" + ota - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) @@ -484,5 +499,5 @@ imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) - self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) + self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \ a.ippObjID = b.ippObjID, \ @@ -494,5 +509,5 @@ AND b.imageID = " + str(imageID) - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) @@ -511,4 +526,5 @@ # loop through all OTAs and populate ImageMeta extensions + self.logger.info("Reading all fits headers and populating ImageMeta tables") for x in range(self.startX, self.endX): for y in range(self.startY, self.endY): @@ -524,4 +540,8 @@ # load corresponding header into memory header = self.findAndReadFITSHeader(ota + ".hdr", file) + if not header: + self.logger.error("No header found for OTA " + ota) + continue + # store sourceID/imageID combo in Db so DVO can look up later @@ -578,6 +598,7 @@ # update ImageMeta with count of detections for this OTA and photoCodeID - sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota])) - self.scratchDb.stmt.execute(sql) + sql = "UPDATE ImageMeta_" + ota + " \ + SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota])) + self.scratchDb.execute(sql) self.populateSkinnyObjectTable(ota) @@ -603,5 +624,5 @@ # update FrameMeta with count OTAs in this file and total number of photometric reference sources sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) return True @@ -613,5 +634,5 @@ sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) - self.scratchDb.stmt.execute(sql) + self.scratchDb.execute(sql) ''' @@ -619,4 +640,9 @@ ''' def alreadyProcessed(self): + + meta = self.gpc1Db.getCameraStageMeta(self.id) + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID) @@ -634,9 +660,31 @@ + ''' + Overriding this method. Filter to only import *.psf extensions + ''' + def importIppTables(self, filter=""): + return super(DetectionBatch, self).importIppTables(".*.psf") + + + ''' + Overriding this method. Use regex to trim off, eg _XY33 extension + ''' + def exportPspsTablesToFits(self, regex="(.*)"): + return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") + + +# TODO put in config +useFullTables=True +testMode=False + logging.config.fileConfig("logging.conf") logger = logging.getLogger("detectionbatch") +logger.setLevel(logging.INFO) logger.info("Starting") gpc1Db = Gpc1Db(logger) +ippToPspsDb = IppToPspsDb(logger) +scratchDb = ScratchDb(logger, useFullTables) + camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam") logger.info("Found %d exposures" % len(camIDs)) @@ -645,25 +693,16 @@ for camID in camIDs: - logger.info("-------------------------------------------------- cam ID: %d" % camID) + #if camID < 43764: continue # TODO file = gpc1Db.getCameraStageSmf(camID) - if not os.path.isfile(file): - logger.error("Cannot read file at '" + file) - continue - - detectionBatch = DetectionBatch(logger, camID, file, False, True) - - if not detectionBatch.alreadyProcessed(): - - detectionBatch.createEmptyPspsTables() - detectionBatch.importIppTables(".*.psf") - if detectionBatch.populatePspsTables(): - detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)") - detectionBatch.writeBatchManifest() - #detectionBatch.reportNullsInAllPspsTables(False) - #detectionBatch.createTarball() - #detectionBatch.publishToDatastore() - - i = i+1 - # if i > 0: sys.exit() - + + detectionBatch = DetectionBatch(logger, + gpc1Db, + ippToPspsDb, + scratchDb, + camID, + file, + testMode, + useFullTables) + detectionBatch.run() +