Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 31846) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 31847) @@ -30,7 +30,5 @@ gpc1Db, ippToPspsDb, - skyID, - inputFile, - stackType): + stackID): super(StackBatch, self).__init__( @@ -39,14 +37,26 @@ gpc1Db, ippToPspsDb, - skyID, + stackID, "stack", - inputFile, - "MD04") # TODO + gpc1Db.getStackStageCmf(stackID), + "3PI") # TODO if not self.everythingOK: return - self.expTime = gpc1Db.getStackExpTime(self.stackID) - - self.logger.info("got exp time of %d" % self.expTime) + self.stackType = "DEEP_STACK" # TODO + + # get stack meta data + meta = self.gpc1Db.getStackStageMeta(self.id) + self.filter = meta[0]; + self.filter = self.filter[0:1] + self.skycell = meta[1]; + self.skycell = self.skycell[8:] + # mangling e.g. 0683.043 into 0683043 for now until we have a schema change + self.skycell = self.skycell.replace(".", "") + self.analysisVer = meta[2]; + + self.expTime = gpc1Db.getStackExpTime(self.id) + + self.logger.debug("Got exp time of %d" % self.expTime) # delete PSPS tables @@ -59,6 +69,12 @@ self.scratchDb.dropTable("ObjectCalColor") + # delete IPP tables + self.scratchDb.dropTable("SkyChip_psf") + self.scratchDb.dropTable("SkyChip_xrad") + self.scratchDb.dropTable("SkyChip_xfit") + self.scratchDb.dropTable("SkyChip_xsrc") + # create an output filename, which is {filterID}{skycellID}.FITS - self.outputFitsFile = "%08d.FITS" % self.stackID + self.outputFitsFile = "%08d.FITS" % self.id self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) @@ -67,5 +83,5 @@ # insert what we know about this stack batch into the stack table - self.ippToPspsDb.insertStackMeta(self.batchID, self.id, self.stackID, self.filter, self.stackType) + self.ippToPspsDb.insertStackMeta(self.batchID, self.filter, self.stackType) # insert sourceID/imageID combo so DVO can look it up @@ -80,9 +96,8 @@ super(StackBatch, self).printMe() - self.logger.info("Sky ID: %d" % self.id) - self.logger.info("Stack ID: %d" % self.stackID) - self.logger.info("Stack type: %s" % self.stackType) - self.logger.info("Skycell: %s" % self.skycell) - self.logger.info("Filter: %s" % self.filter) + self.logger.infoPair("Stack ID", "%d" % self.id) + self.logger.infoPair("Stack type", "%s" % self.stackType) + self.logger.infoPair("Skycell", "%s" % self.skycell) + self.logger.infoPair("Filter", "%s" % self.filter) @@ -92,5 +107,5 @@ def updateStackMetaID(self, table): - sql = "UPDATE " + table + " SET stackMetaID=" + str(self.stackID) + sql = "UPDATE " + table + " SET stackMetaID=" + str(self.id) self.scratchDb.execute(sql) @@ -225,5 +240,5 @@ def populateStackMeta(self): - self.logger.info("Procesing StackMeta table") + self.logger.infoPair("Procesing table", "StackMeta") sql = "INSERT INTO StackMeta (\ @@ -247,5 +262,5 @@ ,pc002002 \ ) VALUES ( \ - " + str(self.stackID) + " \ + " + str(self.id) + " \ ," + self.skycell + " \ ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ @@ -278,5 +293,5 @@ def populateStackDetection(self): - self.logger.info("Procesing StackDetection table") + self.logger.infoPair("Procesing table", "StackDetection") # insert all the detections @@ -390,5 +405,5 @@ def populateStackApFlx(self): - self.logger.info("Procesing StackApFlx table") + self.logger.infoPair("Procesing", "StackApFlx") sql = "INSERT INTO StackApFlx \ @@ -403,12 +418,12 @@ # TODO temporarily loading 1st convolved fluxes into unconvolved fields - self.logger.info("Adding un-convolved fluxes") + self.logger.infoPair("Adding fluxes", "un-convolved") self.updateApFlxs("", "< 7.0") - self.logger.info("Adding 1st convolved fluxes") + self.logger.infoPair("Adding fluxes", "1st convolved") self.updateApFlxs("c1", "< 7.0") - self.logger.info("Adding 2nd convolved fluxes") + self.logger.infoPair("Adding fluxes", "2nd convolved") self.updateApFlxs("c2", "> 7.0") - self.logger.info("Adding petrosians for extended sources") + self.logger.infoPair("Adding", "petrosians for extended sources") sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \ petRadius=b.PETRO_RADIUS \ @@ -440,5 +455,5 @@ def populateStackModelFit(self): - self.logger.info("Procesing StackModelFit table") + self.logger.infoPair("Procesing table", "StackModelFit") # insert all the detections @@ -450,9 +465,9 @@ # populate model parameters - self.logger.info("Adding deVaucouleurs fit") + self.logger.infoPair("Adding model fit", "deVaucouleurs") self.updateModelFit("deV", "PS_MODEL_DEV") - self.logger.info("Adding exponential fit") + self.logger.infoPair("Adding model fit", "exponential") self.updateModelFit("exp", "PS_MODEL_EXP") - self.logger.info("Adding sersic fit") + self.logger.infoPair("Adding model fit", "sersic") self.updateModelFit("ser", "PS_MODEL_SERSIC") @@ -477,5 +492,5 @@ rs.first() nMissing = rs.getInt(1) - self.logger.info("%5d detections in %s table that are not in StackDetection. Deleting" % (nMissing, table)) + self.logger.infoPair("Detections deleted from %s that are not in StackDetection" + table, "%5d" % nMissing) if nMissing < 1: return @@ -490,12 +505,12 @@ def populateStackToImage(self): - self.logger.info("Procesing StackToImage table") - - imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID) + self.logger.infoPair("Procesing table", "StackToImage") + + imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.id) for imageID in imageIDs: sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ VALUES (\ - " + str(self.stackID) + ", " + imageID + ")" + " + str(self.id) + ", " + imageID + ")" self.scratchDb.execute(sql) @@ -509,5 +524,5 @@ def populateSkinnyObject(self): - self.logger.info("Procesing SkinnyObject table") + self.logger.infoPair("Procesing table", "SkinnyObject") sql = "INSERT INTO SkinnyObject (\ @@ -529,5 +544,5 @@ def populateObjectCalColor(self): - self.logger.info("Procesing ObjectCalColor table") + self.logger.infoPair("Procesing table", "ObjectCalColor table") sql = "INSERT INTO ObjectCalColor (\ @@ -550,5 +565,5 @@ def alterPspsTables(self): - self.logger.info("Altering PSPS tables") + self.logger.debug("Altering PSPS tables") #self.scratchDb.makeColumnUnique("StackDetection", "objID") self.scratchDb.createIndex("StackDetection", "ippDetectID") @@ -561,5 +576,5 @@ def indexIppTables(self): - self.logger.info("Creating indexes on IPP tables") + self.logger.infoPair("Creating indexes on", "IPP tables") self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") @@ -575,5 +590,5 @@ self.logger.debug("Updating table '" + table + "' with DVO IDs...") - sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \ + sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \ a.ippObjID = b.ippObjID, \ a.stackDetectID = b.detectID, \ @@ -613,67 +628,2 @@ return True - ''' - Checks whether this batch has already been processed and published - ''' - def alreadyProcessed(self): - - # sadly, we have to read the FITS primary header first - if not self.readPrimaryHeader(): return False - - # get filterID using init table - self.filter = self.header['FPA.FILTER'] - self.filter = self.filter[0:1] - - self.stackType = stackType - meta = self.gpc1Db.getStackStageMeta(self.id, self.header['FPA.FILTER']) - if len(meta) < 1: return False - self.stackID = meta[0]; - self.skycell = meta[1]; - self.skycell = self.skycell[8:] - self.analysisVer = meta[2]; - - #return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID) - return False # TODOI - - -logging.config.fileConfig("logging.conf") -logger = logging.getLogger("stackbatch") -logger.setLevel(logging.INFO) -logger.info("Starting") - -gpc1Db = Gpc1Db(logger) -ippToPspsDb = IppToPspsDb(logger) - -configDoc = ElementTree(file="config.xml") - -#stackType = "NIGHTLY_STACK" -#skyIDs = gpc1Db.getIDsInThisDVODbForThisStageFudge() -#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky") - -stackType = "DEEP_STACK" -skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky") - -#skyIDs = [942] -#skyIDs = [299] -#skyIDs = [302] -#skyIDs = [8508] -#i = 0 -for skyID in skyIDs: - - #if skyID < 1340: continue # nightly - #if skyID < 238: continue # deep - - cmfFiles = gpc1Db.getStackStageCmfs(skyID) - - for file in cmfFiles: - - stackBatch = StackBatch(logger, - configDoc, - gpc1Db, - ippToPspsDb, - skyID, - file, - stackType) - - stackBatch.run() -