Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 31401) +++ trunk/ippToPsps/jython/stackbatch.py (revision 31402) @@ -1,3 +1,6 @@ #!/usr/bin/env jython + +import os.path +import sys import stilts @@ -17,14 +20,32 @@ Constructor ''' - def __init__(self, logger, skyID, inputFile): + def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False): super(StackBatch, self).__init__( logger, "stack", inputFile, - "MD04") # TODO + "MD04", + useFullTables) # TODO self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'") self.skyID = skyID + + # get filterID using init table + self.filter = self.header['FPA.FILTER'] + self.filter = self.filter[0:1] + + self.stackType = stackType + meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER']) + if len(meta) < 1: return + self.stackID = meta[0]; + self.skycell = meta[1]; + + # determine skycell from header value + #self.skycell = "skycell.34" #= self.header['SKYCELL'] + self.skycell = self.skycell[8:] + + self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter)) + # delete PSPS tables @@ -43,15 +64,7 @@ #self.scratchDb.dropTable("SkyChip_xrad") + self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType)) # obs time makes no sense except for nightly stacks - self.logger.info("Stack type: " + self.header['STK_TYPE']) - if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999" - - # determine skycell from header value - self.skycell = self.header['SKYCELL'] - self.skycell = self.skycell[8:] - - # get filterID using init table - self.filter = self.header['FPA.FILTER'] - self.filter = self.filter[0:1] + #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999" # create an output filename, which is {filterID}{skycellID}.FITS @@ -64,8 +77,9 @@ # insert what we know about this stack batch into the stack table - self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.header['STK_ID'], self.filter, self.header['STK_TYPE']) + self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType) # insert sourceID/imageID combo so DVO can look it up - self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID']) + if not self.useFullTables: + self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID']) ''' @@ -74,5 +88,5 @@ def updateStackMetaID(self, table): - sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] + sql = "UPDATE " + table + " SET stackMetaID=" + str(self.stackID) self.scratchDb.stmt.execute(sql) @@ -82,5 +96,5 @@ def updateStackTypeID(self, table): - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" + sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'" self.scratchDb.stmt.execute(sql) @@ -195,6 +209,6 @@ "+modelLong+"Covar78=b.EXT_COVAR_06_07, \ "+modelLong+"Covar88=b.EXT_COVAR_07_07, \ - "+modelLong+"serNu=b.EXT_PAR_07, \ - "+modelLong+"serNuErr=SQRT(EXT_COVAR_07_07) \ + "+"serNu=b.EXT_PAR_07, \ + "+"serNuErr=SQRT(EXT_COVAR_07_07) \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" @@ -214,5 +228,4 @@ ,photoZero \ ,expTime \ - ,nP2Images \ ,psfModelID \ ,ctype1 \ @@ -229,11 +242,10 @@ ,pc002002 \ ) VALUES ( \ - " + self.header['STK_ID'] + " \ + " + str(self.stackID) + " \ ," + self.skycell + " \ ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ ," + self.header['EXPTIME'] + " \ - ," + self.header['NINPUTS'] + " \ - ,'" + self.header['PSFMODEL'] + "' \ + ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -439,11 +451,15 @@ self.logger.info("Procesing StackToImage table") - imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.header['STK_ID']) + imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID) for imageID in imageIDs: sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ VALUES (\ - " + self.header['STK_ID'] + ", " + imageID + ")" + " + str(self.stackID) + ", " + imageID + ")" self.scratchDb.stmt.execute(sql) + + # now update StackMeta with correct number of inputs + sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)" + self.scratchDb.stmt.execute(sql) ''' @@ -489,7 +505,8 @@ Applies indexes to the PSPS tables ''' - def indexPspsTables(self): - - self.logger.info("Creating indexes on PSPS tables") + def alterPspsTables(self): + + self.logger.info("Altering PSPS tables") + self.scratchDb.makeColumnUnique("StackDetection", "objID") self.scratchDb.createIndex("StackDetection", "ippDetectID") self.scratchDb.createIndex("StackApFlx", "ippDetectID") @@ -512,6 +529,8 @@ def updateDvoIDs(self, table): + imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID']) + self.logger.info("Updating table '" + table + "' with DVO IDs...") - sql = "UPDATE " + table + " AS a, dvoDetection AS b SET \ + sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \ a.ippObjID = b.ippObjID, \ a.stackDetectID = b.detectID, \ @@ -519,7 +538,6 @@ WHERE a.ippDetectID = b.ippDetectID \ AND b.sourceID = " + self.header['SOURCEID'] + "\ - AND b.imageID = " + self.header['IMAGEID'] - self.scratchDb.stmt.execute(sql) - self.logger.info("...done") + AND b.imageID = " + str(imageID) + self.scratchDb.stmt.execute(sql) @@ -529,5 +547,7 @@ def populatePspsTables(self): - if not self.getIDsFromDVO(): return False + if not self.useFullTables: + if not self.getIDsFromDVO(): + return False self.populateStackMeta() @@ -548,7 +568,5 @@ def alreadyProcessed(self): - return self.ippToPspsDb.alreadyProcessed("stack", "sky_id", self.skyID) - - + return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID) logging.config.fileConfig("logging.conf") @@ -556,12 +574,16 @@ logger.info("Starting") gpc1Db = Gpc1Db(logger) +stackType = "NIGHTLY_STACK" skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky") +#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky") +#stackType = "DEEP_STACK" #skyIDs = [689] #skyIDs = [299] -skyIDs = [306] +#skyIDs = [302] +#skyIDs = [8508] i = 0 for skyID in skyIDs: - logger.info(" sky ID = %d" % skyID) + logger.info("-------------------------------------------------- sky ID: %d" % skyID) cmfFiles = gpc1Db.getStackStageCmfs(skyID) @@ -569,8 +591,11 @@ for file in cmfFiles: - logger.info("-------------- New Batch ----------") - stackBatch = StackBatch(logger, skyID, file) - - if stackBatch.alreadyProcessed(): + if not os.path.isfile(file): + logger.error("Cannot read file at '" + file) + continue + + stackBatch = StackBatch(logger, skyID, file, stackType, True) + + if not stackBatch.alreadyProcessed(): stackBatch.createEmptyPspsTables() @@ -578,5 +603,5 @@ if stackBatch.populatePspsTables(): - stackBatch.reportNullsInAllPspsTables(False) + #stackBatch.reportNullsInAllPspsTables(False) stackBatch.exportPspsTablesToFits() stackBatch.writeBatchManifest() @@ -584,6 +609,6 @@ #stackBatch.publishToDatastore() - i = i + 1 - if i > 0: break # TODO just doing one filter for now + i = i + 1 + #if i > 0: sys.exit() logger.info("Finished")