Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 31346) +++ trunk/ippToPsps/jython/stackbatch.py (revision 31347) @@ -27,15 +27,17 @@ # delete PSPS tables - self.dropTable("StackMeta") - self.dropTable("StackDetection") - self.dropTable("StackModelFit") - self.dropTable("StackApFlx") - self.dropTable("StackToImage") + self.scratchDb.dropTable("StackMeta") + self.scratchDb.dropTable("StackDetection") + self.scratchDb.dropTable("StackModelFit") + self.scratchDb.dropTable("StackApFlx") + self.scratchDb.dropTable("StackToImage") + self.scratchDb.dropTable("SkinnyObject") + self.scratchDb.dropTable("ObjectCalColor") # delete IPP tables - self.dropTable("SkyChip_psf") - self.dropTable("SkyChip_xsrc") - self.dropTable("SkyChip_xfit") - self.dropTable("SkyChip_xrad") + self.scratchDb.dropTable("SkyChip_psf") + self.scratchDb.dropTable("SkyChip_xsrc") + self.scratchDb.dropTable("SkyChip_xfit") + self.scratchDb.dropTable("SkyChip_xrad") # obs time makes no sense except for nightly stacks @@ -59,12 +61,9 @@ self.historyModNum = "0" - - ''' - Updates a table with data release number - ''' - def updateDataRelease(self, table): - - sql = "UPDATE " + table + " SET dataRelease=" + self.dataRelease - self.localStmt.execute(sql) + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertStackMeta(self.batchID, skyID, self.header['STK_ID'], self.filter, self.header['STK_TYPE']) + + # insert sourceID/imageID combo so DVO can look it up + self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID']) ''' @@ -74,5 +73,5 @@ sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) ''' @@ -82,5 +81,5 @@ sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) @@ -133,5 +132,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) ''' @@ -181,5 +180,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) # sersic fit has an extra parameter @@ -198,5 +197,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) @@ -206,6 +205,4 @@ def populateStackMeta(self): self.logger.info("Procesing StackMeta table") - - photoCalID = self.getPhotoCalID() sql = "INSERT INTO StackMeta (\ @@ -214,4 +211,5 @@ ,photoCalID \ ,photoZero \ + ,expTime \ ,nP2Images \ ,psfModelID \ @@ -231,6 +229,7 @@ " + self.header['STK_ID'] + " \ ," + self.skycell + " \ - ," + str(photoCalID) + " \ + ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ + ," + self.header['EXPTIME'] + " \ ," + self.header['NINPUTS'] + " \ ,'" + self.header['PSFMODEL'] + "' \ @@ -248,10 +247,10 @@ ," + self.header['PC002002'] + " \ )" - self.localStmt.execute(sql) - - self.updateSurveyID("StackMeta") - self.updateFilterID("StackMeta") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackMeta", self.filter) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease)) self.updateStackTypeID("StackMeta") - self.updateDataRelease("StackMeta") ''' @@ -308,6 +307,6 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,PSF_INST_FLUX \ - ,PSF_INST_FLUX_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ ,SKY \ @@ -339,12 +338,19 @@ FROM SkyChip_psf" - self.localStmt.execute(sql) - - self.updateSurveyID("StackDetection") - self.updateFilterID("StackDetection") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackDetection", self.filter) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") + self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") self.updateStackMetaID("StackDetection") - self.updateDataRelease("StackDetection") self.updateStackTypeID("StackDetection") self.updateDvoIDs("StackDetection") + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") + ''' @@ -361,5 +367,5 @@ try: - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) except: return @@ -383,10 +389,12 @@ ,petR90Err=b.PETRO_RADIUS_90_ERR \ WHERE a.ippDetectID=b.IPP_IDET" - self.localStmt.execute(sql) - - self.updateSurveyID("StackApFlx") - self.updateFilterID("StackApFlx") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackApFlx", self.filter) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") + self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") self.updateStackMetaID("StackApFlx") - self.updateDataRelease("StackApFlx") self.updateStackTypeID("StackApFlx") self.updateDvoIDs("StackApFlx") @@ -401,5 +409,5 @@ sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" try: - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) except: return @@ -414,8 +422,10 @@ self.updateModelFit("ser", "PS_MODEL_SERSIC") - self.updateSurveyID("StackModelFit") - self.updateFilterID("StackModelFit") + self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackModelFit", self.filter) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") + self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") self.updateStackMetaID("StackModelFit") - self.updateDataRelease("StackModelFit") self.updateStackTypeID("StackModelFit") self.updateDvoIDs("StackModelFit") @@ -433,5 +443,44 @@ VALUES (\ " + self.header['STK_ID'] + ", " + imageID + ")" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) + + ''' + Populates the SkinnyObject table + ''' + def populateSkinnyObject(self): + self.logger.info("Procesing SkinnyObject table") + + sql = "INSERT INTO SkinnyObject (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.dataRelease)) + + ''' + Populates the ObjectCalColor table + ''' + def populateObjectCalColor(self): + self.logger.info("Procesing ObjectCalColor table") + + sql = "INSERT INTO ObjectCalColor (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateFilterID("ObjectCalColor", self.filter) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.dataRelease)) + ''' @@ -441,7 +490,7 @@ self.logger.info("Creating indexes on PSPS tables") - self.createIndex("StackDetection", "ippDetectID") - self.createIndex("StackApFlx", "ippDetectID") - self.createIndex("StackModelFit", "ippDetectID") + self.scratchDb.createIndex("StackDetection", "ippDetectID") + self.scratchDb.createIndex("StackApFlx", "ippDetectID") + self.scratchDb.createIndex("StackModelFit", "ippDetectID") ''' @@ -451,40 +500,8 @@ self.logger.info("Creating indexes on IPP tables") - self.createIndex("SkyChip_psf", "IPP_IDET") - self.createIndex("SkyChip_xfit", "IPP_IDET") - self.createIndex("SkyChip_xrad", "IPP_IDET") - self.createIndex("SkyChip_xsrc", "IPP_IDET") - - ''' - Temporary hack to fudge some object IDs - ''' - def fudgeIDs(self): - - self.logger.info("Creating bogus IDs on all detections") - - sql = "INSERT INTO dvoDetection (ippDetectID) SELECT IPP_IDET FROM SkyChip_psf"; - self.localStmt.execute(sql) - - sql = "SELECT ippDetectID FROM dvoDetection" - rs = self.localStmt.executeQuery(sql) - - ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - i = 1 - o = 72010000000000001 - - for id in ids: - sql = "UPDATE dvoDetection SET ippObjID = %d, objID = %d WHERE ippDetectID = %s" % (i, o, id) - self.localStmt.execute(sql) - i = i + 1 - o = o + 1 - - self.updateDvoIDs("StackDetection") - self.updateDvoIDs("StackApFlx") - self.updateDvoIDs("StackModelFit") - - self.setMinMaxObjID("StackDetection") + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") ''' @@ -498,6 +515,8 @@ a.stackDetectID = b.detectID, \ a.objID = b.objID \ - WHERE a.ippDetectID = b.ippDetectID" - self.localStmt.execute(sql) + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + self.header['SOURCEID'] + "\ + AND b.imageID = " + self.header['IMAGEID'] + self.scratchDb.stmt.execute(sql) self.logger.info("...done") @@ -508,6 +527,5 @@ def populatePspsTables(self): - if not self.getIDsFromDVO(self.header['SOURCEID'], self.header['IMAGEID']): - return False + if not self.getIDsFromDVO(): return False self.populateStackMeta() @@ -516,7 +534,8 @@ self.populateStackApFlx() self.populateStackToImage() - - #self.fudgeIDs() - self.setMinMaxObjID("StackDetection") + self.populateSkinnyObject() + self.populateObjectCalColor() + + self.setMinMaxObjID(["StackDetection"]) return True @@ -525,9 +544,9 @@ logger = logging.getLogger("stackbatch") logger.info("Starting") -sky_id = 689 # TODO -#sky_id = 299 # TODO gpc1Db = Gpc1Db(logger) -skyIDs = gpc1Db.getSkyIDsInThisDVODb("HAFTest.Staticsky") -skyIDs = [689] +skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky") +#skyIDs = [689] +#skyIDs = [299] +skyIDs = [306] i = 0 for skyID in skyIDs: @@ -545,10 +564,10 @@ stackBatch.importIppTables("") if stackBatch.populatePspsTables(): - + stackBatch.reportNullsInAllPspsTables(False) stackBatch.exportPspsTablesToFits() stackBatch.writeBatchManifest() - #stackBatch.createTarball() - #stackBatch.publishToDatastore() + stackBatch.createTarball() + stackBatch.publishToDatastore() i = i + 1