Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 31345) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 31346) @@ -5,4 +5,5 @@ from java.sql import * from batch import Batch +from gpc1db import Gpc1Db import logging.config @@ -16,18 +17,45 @@ Constructor ''' - def __init__(self, logger, expID, inputFile): + def __init__(self, logger, camID, inputFile, test=False): super(DetectionBatch, self).__init__( logger, "detection", inputFile, - "3PI") # TODO + "MD04") # TODO + #"3PI") # TODO self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'") - self.expID = expID + meta = self.gpc1Db.getCameraStageMeta(camID) + + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; + + self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup)) + # create an output filename, which is {expID}.FITS - self.outputFitsFile = "%08d.FITS" % expID + self.outputFitsFile = "%08d.FITS" % self.expID self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + # if test mode + if test: + self.startX = 3 + self.endX = 4 + self.startY = 3 + self.endY = 4 + else: + self.startX = 0 + self.endX = 8 + self.startY = 0 + self.endY = 8 + + self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) + + # set up some defauts + self.calibModNum = 0 + self.dataRelease = 0 + + self.totalNumPhotoRef = 0 ''' @@ -35,8 +63,16 @@ ''' def populateFrameMeta(self): - self.log("Procesing FrameMeta table") + self.logger.info("Procesing FrameMeta table") sql = "INSERT INTO FrameMeta (\ - frameID \ + frameID \ + ,frameName \ + ,cameraID \ + ,cameraConfigID \ + ,telescopeID \ + ,analysisVer \ + ,p1Recip \ + ,p2Recip \ + ,p3Recip \ ,photoScat \ ,expStart \ @@ -74,4 +110,12 @@ ) VALUES ( \ " + str(self.expID) + " \ + ,'" + self.expName + "' \ + ,1 \ + ,1 \ + ,1 \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ ," + self.header['ZPT_ERR'] + " \ ," + self.header['MJD-OBS'] + " \ @@ -108,8 +152,10 @@ ," + self.header['PCA2X0Y2'] + " \ )" - self.localStmt.execute(sql) - - self.updateSurveyID("FrameMeta") - self.updateFilterID("FrameMeta") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("FrameMeta", self.filter) + self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease)) ''' @@ -119,10 +165,10 @@ tableName = "ImageMeta_" + ota - self.log(" Dealing with table " + tableName) + self.logger.info(" Dealing with table " + tableName) # drop then re-create table - self.dropTable(tableName) + self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" - try: self.localStmt.execute(sql) + try: self.scratchDb.stmt.execute(sql) except: pass @@ -130,4 +176,5 @@ sql = "INSERT INTO " + tableName + " ( \ frameID \ + ,ccdID \ ,sky \ ,skyScat \ @@ -135,11 +182,14 @@ ,completMag \ ,astroScat \ + ,photoScat \ ,numAstroRef \ ,numPhotoRef \ ,nx \ ,ny \ + ,psfFwhm \ ,psfWidMajor \ ,psfWidMinor \ ,psfTheta \ + ,momentFwhm \ ,momentWidMajor \ ,momentWidMinor \ @@ -147,7 +197,14 @@ ,dapResid \ ,detectorID \ + ,qaFlags \ ,detrend1 \ ,detrend2 \ ,detrend3 \ + ,detrend4 \ + ,detrend5 \ + ,detrend6 \ + ,detrend7 \ + ,detrend8 \ + ,photoZero \ ,ctype1 \ ,ctype2 \ @@ -179,54 +236,69 @@ ) VALUES ( \ " + str(self.expID) + " \ - ," + header['MSKY_MN'] + " \ - ," + header['MSKY_SIG'] + " \ - ," + header['FSATUR'] + " \ - ," + header['FLIMIT'] + " \ - ," + header['CERROR'] + " \ - ," + header['NASTRO'] + " \ - ," + header['NASTRO'] + " \ - ," + header['CNAXIS1'] + " \ - ," + header['CNAXIS2'] + " \ - ," + header['FWHM_MAJ'] + " \ - ," + header['FWHM_MIN'] + " \ - ," + header['ANGLE'] + " \ - ," + header['IQ_FW1'] + " \ - ," + header['IQ_FW2'] + " \ - ," + header['APMIFIT'] + " \ - ," + header['DAPMIFIT'] + " \ - ,'" + header['DETECTOR'] + "' \ - ,'" + header['DETREND.MASK'] + "' \ - ,'" + header['DETREND.DARK'] + "' \ - ,'" + header['DETREND.FLAT'] + "' \ - ,'" + header['CTYPE1'] + "' \ - ,'" + header['CTYPE2'] + "' \ - ," + header['CRVAL1'] + " \ - ," + header['CRVAL2'] + " \ - ," + header['CRPIX1'] + " \ - ," + header['CRPIX2'] + " \ - ," + header['CDELT1'] + " \ - ," + header['CDELT2'] + " \ - ," + header['PC001001'] + " \ - ," + header['PC001002'] + " \ - ," + header['PC002001'] + " \ - ," + header['PC002002'] + " \ - ," + header['NPLYTERM'] + " \ - ," + header['PCA1X3Y0'] + " \ - ," + header['PCA1X2Y1'] + " \ - ," + header['PCA1X1Y2'] + " \ - ," + header['PCA1X0Y3'] + " \ - ," + header['PCA1X2Y0'] + " \ - ," + header['PCA1X1Y1'] + " \ - ," + header['PCA1X0Y2'] + " \ - ," + header['PCA2X3Y0'] + " \ - ," + header['PCA2X2Y1'] + " \ - ," + header['PCA2X1Y2'] + " \ - ," + header['PCA2X0Y3'] + " \ - ," + header['PCA2X2Y0'] + " \ - ," + header['PCA2X1Y1'] + " \ - ," + header['PCA2X0Y2'] + " \ + ," + ota[2:4] + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_MN') + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_SIG') + " \ + ," + self.safeDictionaryAccess(header, 'FSATUR') + " \ + ," + self.safeDictionaryAccess(header, 'FLIMIT') + " \ + ," + self.safeDictionaryAccess(header, 'CERROR') + " \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS1') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS2') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'FWHM_MAJ')) + float(self.safeDictionaryAccess(header, 'FWHM_MIN')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MAJ') + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ + ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'IQ_FW1')) + float(self.safeDictionaryAccess(header, 'IQ_FW2')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW1') + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW2') + " \ + ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ + ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ + ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ + ," + str(self.scratchDb.getDvoImageFlags(header['SOURCEID'], header['IMAGEID'])) + " \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.FLAT') + "' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE1') + "' \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE2') + "' \ + ," + self.safeDictionaryAccess(header, 'CRVAL1') + " \ + ," + self.safeDictionaryAccess(header, 'CRVAL2') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX1') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX2') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT1') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT2') + " \ + ," + self.safeDictionaryAccess(header, 'PC001001') + " \ + ," + self.safeDictionaryAccess(header, 'PC001002') + " \ + ," + self.safeDictionaryAccess(header, 'PC002001') + " \ + ," + self.safeDictionaryAccess(header, 'PC002002') + " \ + ," + self.safeDictionaryAccess(header, 'NPLYTERM') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y2') + " \ )" - self.localStmt.execute(sql) - self.updateFilterID(tableName) + + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateFilterID(tableName, self.filter) + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) ''' @@ -236,10 +308,10 @@ tableName = "Detection_" + ota - self.log(" Dealing with table " + tableName) + self.logger.info(" Dealing with table " + tableName) # drop then re-create table - self.dropTable(tableName) + self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE Detection" - try: self.localStmt.execute(sql) + try: self.scratchDb.stmt.execute(sql) except: pass @@ -251,4 +323,7 @@ ,xPosErr \ ,yPosErr \ + ,instFlux \ + ,instFluxErr \ + ,peakADU \ ,psfWidMajor \ ,psfWidMinor \ @@ -258,4 +333,6 @@ ,momentXY \ ,momentYY \ + ,apMag \ + ,infoFlag \ ,sky \ ,skyErr \ @@ -268,4 +345,7 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ ,PSF_MAJOR \ ,PSF_MINOR \ @@ -275,4 +355,6 @@ ,MOMENTS_XY \ ,MOMENTS_YY \ + ,AP_MAG \ + ,FLAGS\ ,SKY \ ,SKY_SIGMA \ @@ -280,11 +362,18 @@ FROM " + ota + "_psf" - self.localStmt.execute(sql) - - self.updateSurveyID(tableName) - self.updateFilterID(tableName) - - # now add DVO IDs - self.updateDvoIDs(tableName) + self.scratchDb.stmt.execute(sql) + + # set obsTime + sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "historyModNum", "0") + + self.scratchDb.updateAllRows(tableName, "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID(tableName, self.filter) + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU") ''' @@ -294,10 +383,10 @@ tableName = "SkinnyObject_" + ota - self.log(" Dealing with table " + tableName) + self.logger.info(" Dealing with table " + tableName) # drop then re-create table - self.dropTable(tableName) + self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject" - try: self.localStmt.execute(sql) + try: self.scratchDb.stmt.execute(sql) except: pass @@ -313,5 +402,7 @@ ,surveyID \ FROM Detection_" + ota - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) ''' @@ -321,10 +412,10 @@ tableName = "ObjectCalColor_" + ota - self.log(" Dealing with table " + tableName) + self.logger.info(" Dealing with table " + tableName) # drop then re-create table - self.dropTable(tableName) + self.scratchDb.dropTable(tableName) sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor" - try: self.localStmt.execute(sql) + try: self.scratchDb.stmt.execute(sql) except: pass @@ -340,13 +431,16 @@ ,filterID \ FROM Detection_" + ota - self.localStmt.execute(sql) - - - ''' - Applies indexes to the PSPS tables + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + + + ''' + Applies indexes to the PSPS tables TODO need this? ''' def indexPspsTables(self): - self.log("Creating indexes on PSPS tables") + self.logger.info("Creating indexes on PSPS tables") # self.createIndex("StackDetection", "ippDetectID") # self.createIndex("StackApFlx", "ippDetectID") @@ -358,8 +452,8 @@ def indexIppTables(self): - self.log("Creating indexes on IPP tables") - - for x in range(0,8): - for y in range(0,8): + self.logger.info("Creating indexes on IPP tables") + + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): # dodge the corners @@ -370,11 +464,10 @@ extension = "XY%d%d_psf" % (x, y) - self.createIndex(extension, "IPP_IDET") - - ''' - Overriden from Batch class + self.scratchDb.createIndex(extension, "IPP_IDET") + + ''' Updates provided table with DVO IDs from DVO table ''' - def updateDvoIDs(self, table): + def updateDvoIDs(self, table, sourceID, imageID): self.logger.info("Updating table '" + table + "' with DVO IDs...") @@ -382,7 +475,11 @@ a.ippObjID = b.ippObjID, \ a.detectID = b.detectID, \ - a.objID = b.objID \ - WHERE a.ippDetectID = b.ippDetectID" - self.localStmt.execute(sql) + a.objID = b.objID, \ + a.infoFlag = b.flags << 32 | a.infoFlag \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + str(sourceID) + " \ + AND b.imageID = " + str(imageID) + + self.scratchDb.stmt.execute(sql) self.logger.info("...done") @@ -392,5 +489,4 @@ ''' def populatePspsTables(self): - # get filterID using init table @@ -398,7 +494,14 @@ self.populateFrameMeta() - - for x in range(3, 4): - for y in range(3, 4): + + # dictionary objects to hold sourceIDs and imageIDs for later + sourceIDs = {} + imageIDs = {} + + file = open(self.inputFitsPath, 'r') + + # loop through all OTAs and populate ImageMeta extensions + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): # dodge the corners @@ -411,14 +514,82 @@ # load corresponding header into memory - header = self.findAndReadFITSHeader(ota + ".hdr") - self.getIDsFromDVO(header['SOURCEID'], header['IMAGEID']) - + header = self.findAndReadFITSHeader(ota + ".hdr", file) + + # store sourceID/imageID combo in Db so DVO can look up later + self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID']) + + # store these for later + sourceIDs[ota] = header['SOURCEID'] + imageIDs[ota] = header['IMAGEID'] + + # populate ImageMeta self.populateImageMetaTable(ota, header) + self.updateImageID("ImageMeta_" + ota, x, y) + + # now run DVO code to get all IDs + self.getIDsFromDVO() + + # loop through all OTAs again to update with DVO IDs + self.tablesToExport = [] + self.tablesToExport.append("FrameMeta") + tables = [] + otaCount = 0 + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + ota = "XY%d%d" % (x, y) + + # populate remainder of tables self.populateDetectionTable(ota) + + # now add DVO IDs + self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota]) + + # check we have something in this Detection table + if self.scratchDb.getRowCount("Detection_" + ota) < 1: + self.logger.info("Empty tables for ota " + ota + ". Skipping.") + continue; + + self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") + self.updateImageID("Detection_" + ota, x, y) + + # update ImageMeta with count of detections for this OTA and photoCodeID + sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(header['SOURCEID'], header['IMAGEID'])) + self.scratchDb.stmt.execute(sql) + self.populateSkinnyObjectTable(ota) self.populateObjectCalColorTable(ota) - - #self.setMinMaxObjID(tableName) # TODO set based on all tables + # add these to list of tables to export later + self.tablesToExport.append("ImageMeta_" + ota) + self.tablesToExport.append("Detection_" + ota) + self.tablesToExport.append("SkinnyObject_" + ota) + self.tablesToExport.append("ObjectCalColor_" + ota) + tables.append("Detection_" + ota) + + otaCount = otaCount + 1 + + self.setMinMaxObjID(tables) + + # update FrameMeta with count OTAs in this file and total number of photometric reference sources + sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) + self.scratchDb.stmt.execute(sql) + + return True + + ''' + Updates imageID {EXP_ID}{OTA} in the provided table + ''' + def updateImageID(self, tableName, x, y): + + sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) + self.scratchDb.stmt.execute(sql) + @@ -430,6 +601,6 @@ # loops round all imported tables, but we want to check one OTA's worth for table in self.pspsTables: - if table.name == "FrameMeta": self.reportNulls(table.name, showPartials) - else: self.reportNulls(table.name + "_XY33", showPartials) + if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials) + else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials) @@ -438,13 +609,24 @@ logger.info("Starting") -expID = 230033; -inputFile = "detdemo.fits" -detectionBatch = DetectionBatch(logger, expID, inputFile) -detectionBatch.createEmptyPspsTables() -detectionBatch.importIppTables("XY33\.psf") -detectionBatch.populatePspsTables() -detectionBatch.exportPspsTablesToFits() -detectionBatch.writeBatchManifest() -detectionBatch.reportNullsInAllPspsTables(False) - - +gpc1Db = Gpc1Db(logger) +camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam") +#camIDs = gpc1Db.getIDsInThisDVODbForThisStage("ThreePi.V1", "cam") + +logger.info("Found %d exposures" % len(camIDs)) + +i = 0 +for camID in camIDs: + + file = gpc1Db.getCameraStageSmf(camID) + #file = "detdemo.fits" + detectionBatch = DetectionBatch(logger, camID, file, True) + detectionBatch.createEmptyPspsTables() + detectionBatch.importIppTables("XY33\.psf") + if detectionBatch.populatePspsTables(): + detectionBatch.exportPspsTablesToFits() + detectionBatch.writeBatchManifest() + detectionBatch.reportNullsInAllPspsTables(False) + + i = i+1 + if i > 0: break +