Index: trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38889) +++ trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38890) @@ -57,10 +57,11 @@ "FW", None) - # gpc1Db.getForcedWarpStageSmf(forcedWarpID) + + # gpc1Db.getForcedWarpStageSmf(forcedWarpID) # there is a funny problem with order here: we need to open the file first to read the header to get # to skycell ID - #forcedWarpID = self.forcedWarpID - - #make a set of lists (needs to be outside of the loop + # forcedWarpID = self.forcedWarpID + + # make a set of lists (needs to be outside of the loop stageID = self.id @@ -89,4 +90,5 @@ self.forcedWarpIds = gpc1Db.getListOfForcedWarpIdsForForcedWarpBatch(skychunk.dvoLabel,stageID) + for forcedWarpID in self.forcedWarpIds: self.number[num] = forcedWarpID @@ -96,4 +98,10 @@ self.fits[num] = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, int(forcedWarpID)) + ## print "--- PHU Header ---" + ## myHeader = self.fits[num].getPrimaryHeader() + ## for key in myHeader: + ## print "--- header: " + key + " = " + myHeader[key] + ## print "--- END Header ---" + self.header[num] = self.fits[num].getPrimaryHeader() self.warpSkyFileID[num] = self.getKeyValue(self.header[num],"IMAGEID") @@ -101,5 +109,7 @@ self.logger.infoPair("found filterName",self.filterName[num]) self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID[num]) - # get camera meta data + + # get camera meta data from gpc1 database + # NOTE: this should all be in the header meta ={} meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID[num]) @@ -111,6 +121,6 @@ self.expID[num] = meta[0] self.expTime[num] = meta[1] -# This isn't populated in the database (it's NULL). The hack is below. -# self.analysisVer[num] = meta[2] + # This isn't populated in the database (it's NULL). The hack is below. + # self.analysisVer[num] = meta[2] self.skycellName[num] = meta[3] self.tessName[num] = meta[4] @@ -121,7 +131,6 @@ self.logger.infoPair("found tess_id",self.tessName[num]) - - #tessName = self.getKeyValue(self.header, 'TESS_ID') - #self.tessID = self.scratchDb.getTessID(tessName) + # tessName = self.getKeyValue(self.header, 'TESS_ID') + # self.tessID = self.scratchDb.getTessID(tessName) # As the database doesn't have the software_ver entry, pull the photometry version as the analysis version. @@ -131,20 +140,20 @@ if self.analysisVer[num] is None: self.analysisVer[num] = "-999" self.logger.infoPair("found analysisVer",self.analysisVer[num]) - + if self.tessName[num] == 'RINGS.V3': - - # skycell is, eg "skycell.1133.081" - # 0123456789012345 + # skycell is, eg "skycell.1133.081" + # 0123456789012345 self.projectionID[num] = self.skycellName[num][8:12] - #self.skycellID = self.skycellName[13:] - self.skycellID[num] = self.skycellName[num][8:] + self.skycellID[num] = self.skycellName[num][13:] else: - # skycell is, eg "skycell.081" - # 01234567890 + # skycell is, eg "skycell.081" + # 01234567890 self.projectionID[num] = 0 - self.skycellID[num] = self.skycellName[num][8:11] - self.skycellID[num]=self.skycellName[num][8:] - if self.skycellID[num] == "": - self.skycellID[num] = -1 + self.skycellID[num] = self.skycellName[num][8:] + + if self.projectionID[num] == "": + self.projectionID[num] = -1 + if self.skycellID[num] == "": + self.skycellID[num] = -1 self.logger.infoPair("projection cell found ",self.projectionID[num]) @@ -164,4 +173,5 @@ # create the tables + # XXX why are these deleted? sql = "CREATE TABLE ForcedWarpMeasurement_" + str(forcedWarpID) + " like ForcedWarpMeasurement" print sql @@ -183,6 +193,5 @@ self.logger.infoPair("calculating ObsTime","ok") - # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 - + # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0) @@ -346,20 +355,15 @@ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") - sqlLine.group("ippDetectID", "IPP_IDET") + sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("forcedWarpID", str(forcedWarpID)) - print " got here too, num", num - sqlLine.group("randomWarpID", "RAND("+str(self.batchID)+")") - print "here" - sqlLine.group("filterID", str(self.filterID[num])) - print "here1" - sqlLine.group("surveyID", str(self.surveyID)) - print "here2" - sqlLine.group("skycellID", str(self.skycellID[num])) - print "here3" - sqlLine.group("expTime", str(self.expTime[num])) - sqlLine.group("forcedSummaryID", str(self.id)) - print "here4" - sqlLine.group("obsTime", str(self.obsTime[num])) + sqlLine.group("forcedWarpID", str(forcedWarpID)) + sqlLine.group("randomWarpID", "RAND("+str(self.batchID)+")") + sqlLine.group("filterID", str(self.filterID[num])) + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("skycellID", str(self.skycellID[num])) + sqlLine.group("expTime", str(self.expTime[num])) + sqlLine.group("forcedSummaryID", str(self.id)) + + sqlLine.group("obsTime", str(self.obsTime[num])) sqlLine.group("FpsfFlux", "PSF_INST_FLUX / " + expTimeString) sqlLine.group("FpsfFluxErr", "PSF_INST_FLUX_SIG / " + expTimeString) @@ -394,16 +398,9 @@ sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) - print "got here too" - sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("failed sql: ", sql) - raise - + self.scratchDb.execute(sql) # insert the per-object data (IDs, random stack ID, processingVersion, etc) - #self.selectDvoObjIDs() + # self.selectDvoObjIDs() self.logger.infoPair("inserting dvo info: find imageID","from externID") imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID[num]) @@ -430,8 +427,6 @@ sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("failed sql:",sql) - raise + self.scratchDb.execute(sql) + self.logger.infoPair("need to cull"," nulls objid") rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID) @@ -456,4 +451,5 @@ self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "objID") self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "ippDetectID") + ''' Populates the ForcedWarpExtended table @@ -651,4 +647,6 @@ def importIppTables(self, columns="*", tableRE=""): + self.tablesLoaded = [] + count = 0 for num in self.number: @@ -666,17 +664,19 @@ self.logger.errorPair("STILTS could not import from", fileName) return False + for table in tables: match = re.match(tableRE, table.name) if not match: continue + self.logger.infoPair("Reading IPP table", table.name+"_"+str(forcedWarpID)) table = stilts.tpipe(table, cmd='addcol table_index $0') table = stilts.tpipe(table, cmd='explodeall') - # drop any previous tables before import + # drop any previous tables before import self.scratchDb.dropTable(table.name+"_"+str(forcedWarpID)) - # IPP FITS files are littered with infinities, so remove these + # IPP FITS files are littered with infinities, so remove these self.logger.info("Removing Infinity values from all columns") table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') @@ -689,8 +689,12 @@ except: self.logger.exception("Problem writing table '" + table.name+"_"+str(forcedWarpID) + "' to the database") - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables(forcedWarpID) + + self.tablesLoaded.append(table.name) + + # after all tables are read for this forcedWarpID, then index the set + self.indexIppTables(forcedWarpID) + return True @@ -748,10 +752,26 @@ def indexIppTables(self,forcedWarpID): - self.logger.infoPair("Creating indexes on"+forcedWarpID, "IPP tables") - self.scratchDb.createIndex("SkyChip_psf_"+forcedWarpID, "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET") + self.logger.infoPair("Creating indexes on " + forcedWarpID, "IPP tables") + + # it probably should not be an error not to have one of these: + cmfTable = "SkyChip_psf_" + forcedWarpID + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = "SkyChip_xfit_" + forcedWarpID + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = "SkyChip_xrad_" + forcedWarpID + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = "SkyChip_xsrc_" + forcedWarpID + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = "SkyChip_xgal_" + forcedWarpID + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + ## self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET") + ## self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET") + ## self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET") + ## self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET") return True