Index: trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38083) +++ trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38358) @@ -43,5 +43,5 @@ ippToPspsDb, scratchDb, - forcedWarpID, + id, batchID): @@ -53,5 +53,5 @@ ippToPspsDb, scratchDb, - forcedWarpID, + id, batchID, "FW", @@ -60,31 +60,64 @@ # there is a funny problem with order here: we need to open the file first to read the header to get # to skycell ID - #forcedWarpID = self.forcedWarpID - self.logger.infoPair("getting the fits file for add_id",forcedWarpID) - self.fits = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, forcedWarpID) - self.forcedWarpID = forcedWarpID - self.header = self.fits.getPrimaryHeader() - self.warpSkyFileID = self.getKeyValue(self.header,"IMAGEID") - self.filterName = self.getKeyValue(self.header,"FPA.FILTERID") - self.logger.infoPair("found filterName",self.filterName) - self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID) + #forcedWarpID = self.forcedWarpID + + #make a set of lists (needs to be outside of the loop + + stageID = self.id + self.fits={} + self.header={} + self.warpSkyFileID={} + self.filterName = {} + self.historyModNum = "0" + + + self.calibModNum = 0 + self.totalNumPhotoRef = 0 + self.number={} + self.expID = {} + self.expTime = {} + self.analysisVer = {} + self.tessName = {} + self.obsTime = {} + self.filterID={} + self.skycellName={} + self.skycellID = {} + self.projectionID = {} + + #I give a number to the forcedwarp stuff, I am not sure if this is the best way but it might work + + num = 0 + + self.forcedWarpIds = gpc1Db.getListOfForcedWarpIdsForForcedWarpBatch(skychunk.dvoLabel,stageID) + for forcedWarpID in self.forcedWarpIds: + self.number[num] = forcedWarpID + self.logger.infoPair("index",num) + self.logger.infoPair("forcedWarpID", forcedWarpID) + self.logger.infoPair("getting the fits file for add_id",forcedWarpID) + self.fits[num] = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, int(forcedWarpID)) + + self.header[num] = self.fits[num].getPrimaryHeader() + self.warpSkyFileID[num] = self.getKeyValue(self.header[num],"IMAGEID") + self.filterName[num] = self.getKeyValue(self.header[num],"FPA.FILTERID") + self.logger.infoPair("found filterName",self.filterName[num]) + self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID[num]) # get camera meta data - - meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID) - - if not meta: - self.logger.errorPair("Could not get", "forced warp metadata") - raise - - self.expID = meta[0] - self.expTime = meta[1] - self.analysisVer = meta[2] - self.skycellName = meta[3] - self.tessName = meta[4] + meta ={} + meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID[num]) + + if not meta: + self.logger.errorPair("Could not get", "forced warp metadata") + raise + + self.expID[num] = meta[0] + self.expTime[num] = meta[1] + self.analysisVer[num] = meta[2] + self.skycellName[num] = meta[3] + self.tessName[num] = meta[4] - self.logger.infoPair("found exp_id",str(self.expID)) - self.logger.infoPair("found exp_time",str(self.expTime)) - self.logger.infoPair("found skycell_id",self.skycellName) - self.logger.infoPair("found tess_id",self.tessName) + self.logger.infoPair("found exp_id",str(self.expID[num])) + self.logger.infoPair("found exp_time",str(self.expTime[num])) + self.logger.infoPair("found skycell_id",self.skycellName[num]) + self.logger.infoPair("found tess_id",self.tessName[num]) @@ -92,65 +125,82 @@ #self.tessID = self.scratchDb.getTessID(tessName) - if not self.analysisVer: self.analysisVer = -999 - if self.analysisVer is None: self.analysisVer = "-999" - self.logger.infoPair("found analysisVer",self.analysisVer) - - if self.tessName == 'RINGS.V3': + if not self.analysisVer[num]: self.analysisVer[num] = -999 + if self.analysisVer[num] is None: self.analysisVer[num] = "-999" + self.logger.infoPair("found analysisVer",self.analysisVer[num]) + + if self.tessName[num] == 'RINGS.V3': # skycell is, eg "skycell.1133.081" # 0123456789012345 - self.projectionID = self.skycellName[8:12] + self.projectionID[num] = self.skycellName[num][8:12] #self.skycellID = self.skycellName[13:] - self.skycellID = self.skycellName[8:] - else: + self.skycellID[num] = self.skycellName[num][8:] + else: # skycell is, eg "skycell.081" # 01234567890 - self.projectionID = 0 - self.skycellID = self.skycellName[8:11] - self.skycellID=self.skycellName[8:] - if self.skycellID == "": - self.skycellID = -1 - - self.logger.infoPair("projection cell found ",self.projectionID) - self.logger.infoPair("skycell found", self.skycellID) - - - + self.projectionID[num] = 0 + self.skycellID[num] = self.skycellName[num][8:11] + self.skycellID[num]=self.skycellName[num][8:] + if self.skycellID[num] == "": + self.skycellID[num] = -1 + + self.logger.infoPair("projection cell found ",self.projectionID[num]) + self.logger.infoPair("skycell found", self.skycellID[num]) + self.dropTableVerbose("ForcedWarpMeasurement"+"_"+str(forcedWarpID)) + self.dropTableVerbose("ForcedWarpExtended"+"_"+str(forcedWarpID)) + self.dropTableVerbose("ForcedWarpMeta"+"_"+str(forcedWarpID)) + self.dropTableVerbose("ForcedWarpToImage"+"_"+str(forcedWarpID)) + self.dropTableVerbose("ForcedWarpLensing"+"_"+str(forcedWarpID)) + if not self.config.retry: + self.scratchDb.dropTable("ForcedWarpMeasurement"+"_"+str(forcedWarpID)) + self.scratchDb.dropTable("ForcedWarpExtended"+"_"+str(forcedWarpID)) + self.scratchDb.dropTable("ForcedWarpMeta"+"_"+str(forcedWarpID)) + self.scratchDb.dropTable("ForcedWarpToImage"+"_"+str(forcedWarpID)) + self.scratchDb.dropTable("ForcedWarpLensing"+"_"+str(forcedWarpID)) + # get a few primary header values. if in test mode, then use defaults + # create the tables + + sql = "CREATE TABLE ForcedWarpMeasurement_" + str(forcedWarpID) + " like ForcedWarpMeasurement" + print sql + #try: self.scratchDb.execute(sql) + #except: raise + sql = "CREATE TABLE ForcedWarpExtended_" + str(forcedWarpID) + " like ForcedWarpExtended" + #try: self.scratchDb.execute(sql) + #except: raise + sql = "CREATE TABLE ForcedWarpMeta_" + str(forcedWarpID) + " like ForcedWarpMeta" + #try: self.scratchDb.execute(sql) + #except: raise + sql = "CREATE TABLE ForcedWarpToImage_" + str(forcedWarpID) + " like ForcedWarpToImage" + #try: self.scratchDb.execute(sql) + #except: raise + sql = "CREATE TABLE ForcedWarpLensing_" + str(forcedWarpID) + " like ForcedWarpLensing" + #try: self.scratchDb.execute(sql) + #except: raise + + + self.logger.infoPair("calculating ObsTime","ok") + # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 + + + self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0) + + self.logger.infoPair("obstime",self.obsTime[num]) + self.filterID[num] = self.scratchDb.getFilterID(self.filterName[num][0]) + self.logger.infoPair("filterID is ",self.filterID[num]) + self.logger.infoPair("inserting basics into ","ForcedWarpMeta"+"_"+forcedWarpID) + + self.ippToPspsDb.insertForcedWarpMeta( int(self.batchID), int(self.expID[num]), self.filterName[num][0]) + + num=num+1 # create an output filename, which is {expID}.FITS - self.outputFitsFile = "%08d.FITS" % forcedWarpID + self.outputFitsFile = "%08d.FITS" % self.id self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - self.dropTableVerbose("ForcedWarpMeasurement") - self.dropTableVerbose("ForcedWarpExtended") - self.dropTableVerbose("ForcedWarpMeta") - self.dropTableVerbose("ForcedWarpToImage") - self.dropTableVerbose("ForcedWarpLensing") - if not self.config.retry: - self.scratchDb.dropTable("ForcedWarpMeasurement") - self.scratchDb.dropTable("ForcedWarpExtended") - self.scratchDb.dropTable("ForcedWarpMeta") - self.scratchDb.dropTable("ForcedWarpToImage") - self.scratchDb.dropTable("ForcedWarpLensing") - self.historyModNum = "0" - - # get a few primary header values. if in test mode, then use defaults - self.logger.infoPair("calculating ObsTime","ok") - # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 - self.obsTime = float(self.header['MJD-OBS']) + (float(self.expTime) / 172800.0) - self.logger.infoPair("obstime",self.obsTime) - # set up some defauts - self.calibModNum = 0 - self.totalNumPhotoRef = 0 - - self.filterID = self.scratchDb.getFilterID(self.filterName[0]) - self.logger.infoPair("filterID is ",self.filterID) - - # insert what we know about this stack batch into the stack table - self.logger.infoPair("inserting basics into ","ForcedWarpMeta") - - self.ippToPspsDb.insertForcedWarpMeta(int(self.batchID), int(self.expID), self.filterName[0]) + + # dump stuff to log self.logger.infoPair("add ID", "%d" % self.id) - self.logger.infoPair("Exp ID", self.expID) + +# self.logger.infoPair("Exp ID", self.expID) @@ -162,20 +212,36 @@ Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateForcedWarpMeta(self): - self.logger.infoPair("Processing table", "ForcedWarpMeta") - - header = self.header + def populateForcedWarpMeta(self,num): + forcedWarpID = self.number[num] + self.logger.infoPair("Processing table", "ForcedWarpMeta_"+forcedWarpID) + + sql = "CREATE TABLE ForcedWarpMeta_"+forcedWarpID+" LIKE ForcedWarpMeta" + try: self.scratchDb.execute(sql) + except: pass + + + header = self.header[num] + self.logger.infoPair("found header","ok") # photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID)) - sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (") + sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta_"+forcedWarpID+" (") #batchID (below) #surveyID (below) #filterdID (below) - sqlLine.group("forcedWarpID", str(self.forcedWarpID)) - sqlLine.group("skyCellID", str(self.skycellID)) -# sqlLine.group("photoCalID", photoCalID) + +# self.logger.infoPair("building sql ","ok") + + sqlLine.group("forcedWarpID", str(forcedWarpID)) +# self.logger.infoPair("building sql ","1") + sqlLine.group("skyCellID", str(self.skycellID[num])) +# sqlLine.group("photoCalID", photoCalID) +# self.logger.infoPair("building sql ","2") sqlLine.group("magSat", self.getKeyFloat(header, "%.8f", 'FSATUR')) - sqlLine.group("analVer", str(self.analysisVer)) - sqlLine.group("expTime",self.expTime) +# self.logger.infoPair("building sql ","3") + sqlLine.group("analVer", str(self.analysisVer[num])) + # self.logger.infoPair("building sql ","3") + + sqlLine.group("expTime",self.expTime[num]) + # self.logger.infoPair("building sql ","3") sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) # sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) @@ -189,16 +255,16 @@ # sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) #photoColor -- how do I set this? it's also not set in stack meta - sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); - sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); - sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); - sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); - sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); - sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); - sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); - sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); - sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); - sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); - sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); - sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); + sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')); @@ -210,10 +276,10 @@ raise - self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID)) - self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "batchID", str(self.batchID)) + self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "surveyID", str(self.surveyID)) # self.scratchDb.updateFilterID("ForcedWarpMeta", self.filterID) - self.scratchDb.updateFilterID("ForcedWarpMeta",self.filterName[0]) - self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease)) + self.scratchDb.updateFilterID("ForcedWarpMeta_"+forcedWarpID,self.filterName[num][0]) + self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "dataRelease", str(self.skychunk.dataRelease)) #self.tablesToExport.append("ForcedWarpMeta") @@ -221,9 +287,12 @@ Populates the ForcedWarpToImage table ''' - def populateForcedWarpToImage(self): - + def populateForcedWarpToImage(self,num): + forcedwarpID = self.number[num] self.logger.infoPair("Procesing table", "ForcedWarpToImage") - - forcedwarpID = self.forcedWarpID + sql = "CREATE TABLE ForcedWarpToImage_"+forcedwarpID+" LIKE ForcedWarpToImage" + try: self.scratchDb.execute(sql) + except: pass + + #forcedwarpID = self.forcedWarpID if forcedwarpID > 0: @@ -232,5 +301,5 @@ for imageID in imageIDs: - sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \ + sql = "INSERT INTO ForcedWarpToImage_"+str(forcedwarpID)+" (forcedwarpID, imageID) \ VALUES (\ " + str(forcedwarpID) + ", " + imageID + ")" @@ -250,26 +319,37 @@ Populates the ForcedWarpMeasurement table ''' - def populateForcedWarpMeasurement(self): - - pspsTableName = "ForcedWarpMeasurement" - ippTableName = "SkyChip_psf" - - - self.logger.infoPair("Procesing table", "ForcedWarpMeasurement") + def populateForcedWarpMeasurement(self,num): + forcedWarpID = self.number[num] + pspsTableName = "ForcedWarpMeasurement_"+forcedWarpID + ippTableName = "SkyChip_psf_"+forcedWarpID + + sql = "CREATE TABLE ForcedWarpMeasurement_"+forcedWarpID+" LIKE ForcedWarpMeasurement" + try: self.scratchDb.execute(sql) + except: pass + + self.logger.infoPair("Procesing table", "ForcedWarpMeasurement_"+forcedWarpID) BEFORE = self.scratchDb.getRowCount(ippTableName) - extTimeString = str(self.expTime) + extTimeString = str(self.expTime[num]) #missing are zp, telluricExt, airmass (not easy to get ?) - + sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("forcedWarpID", str(self.forcedWarpID)); - sqlLine.group("randomWarpID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") - sqlLine.group("filterID", str(self.filterID)) + + sqlLine.group("forcedWarpID", str(forcedWarpID)) + print " got here too, num", num + sqlLine.group("randomWarpID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") + print "here" + sqlLine.group("filterID", str(self.filterID[num])) + print "here1" sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("skycellID", str(self.skycellID)) - sqlLine.group("expTime", str(self.expTime)) - sqlLine.group("obsTime", str(self.obsTime)) + print "here2" + sqlLine.group("skycellID", str(self.skycellID[num])) + print "here3" + sqlLine.group("expTime", str(self.expTime[num])) + sqlLine.group("forcedSummaryID", str(self.id)) + print "here4" + sqlLine.group("obsTime", str(self.obsTime[num])) sqlLine.group("psfFlux", "PSF_INST_FLUX / "+extTimeString) sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / "+extTimeString) @@ -303,4 +383,6 @@ sqlLine.group("dataRelease",str(self.skychunk.dataRelease)) + print "got here too" + sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -316,7 +398,7 @@ #self.selectDvoObjIDs() self.logger.infoPair("inserting dvo info: find imageID","from externID") - imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID) - self.logger.infoPair("updating","forcedWarpMeasurement") - sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") + imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID[num]) + self.logger.infoPair("updating","forcedWarpMeasurement_"+forcedWarpID) + sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement_"+forcedWarpID+" as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") sqlLine.group("a.objID","b.objID") sqlLine.group("a.detectID","b.detectID") @@ -332,9 +414,9 @@ raise self.logger.infoPair("need to cull"," nulls objid") - rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement") - results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement", "objID") - rowCountAfter = self.scratchDb.getRowCount("ForcedWarpMeasurement") + rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID) + results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement_"+forcedWarpID, "objID") + rowCountAfter = self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID) self.logger.infoPair("row count of ForcedWarpMeasurement",rowCountBefore) - self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID", rowCountAfter) + self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID ", rowCountAfter) if rowCountAfter == 0: self.skipBatch = True @@ -343,21 +425,24 @@ # self.generateRandomIDs() - self.logger.infoPair("Adding 'row' columns to", "ForcedWarpMeasurement") - self.scratchDb.addRowCountColumn("ForcedWarpMeasurement", "row") - self.logger.infoPair("Adding unique ids to", "ForcedWarpMeasurement") - self.generateUniquePspsIDs() - self.logger.infoPair("Dropping row column from", "ForcedWarpMeasurement table") - self.scratchDb.dropColumn("ForcedWarpMeasurement", "row") + self.logger.infoPair("Adding 'row' columns to", "ForcedWarpMeasurement_"+forcedWarpID) + self.scratchDb.addRowCountColumn("ForcedWarpMeasurement_"+forcedWarpID, "row") + self.logger.infoPair("Adding unique ids to", "ForcedWarpMeasurement_"+forcedWarpID) + self.generateUniquePspsIDs(num) + self.logger.infoPair("Dropping row column from", "ForcedWarpMeasurement_"+forcedWarpID+" table") + self.scratchDb.dropColumn("ForcedWarpMeasurement_"+forcedWarpID, "row") # add indexes ForcedWarpMeasurement - self.scratchDb.createIndex("ForcedWarpMeasurement", "objID") - self.scratchDb.createIndex("ForcedWarpMeasurement", "ippDetectID") + self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "objID") + self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "ippDetectID") ''' Populates the ForcedWarpExtended table ''' - def populateForcedWarpExtended(self): - - pspsTableName = "ForcedWarpExtended" - ippTableName = "SkyChip_xrad" + def populateForcedWarpExtended(self,num): + forcedWarpID = self.number[num] + pspsTableName = "ForcedWarpExtended_"+forcedWarpID + ippTableName = "SkyChip_xrad_"+forcedWarpID + sql = "CREATE TABLE ForcedWarpExtended_"+forcedWarpID+" LIKE ForcedWarpExtended" + try: self.scratchDb.execute(sql) + except: pass #self.tablesToExport.append("ForcedWarpExtended") @@ -366,12 +451,16 @@ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("flxR5", "APER_FLUX_5") - sqlLine.group("flxR5Err", "APER_FLUX_ERR_5" ) - sqlLine.group("flxR5Std", "APER_FLUX_STDEV_5") - sqlLine.group("flxR5Fill", "APER_FILL_5") - sqlLine.group("flxR6", "APER_FLUX_6") - sqlLine.group("flxR6Err", "APER_FLUX_ERR_6" ) - sqlLine.group("flxR6Std", "APER_FLUX_STDEV_6") - sqlLine.group("flxR6Fill", "APER_FILL_6") + sqlLine.group("flxR5", "APER_FLUX_3") + sqlLine.group("flxR5Err", "APER_FLUX_ERR_3" ) + sqlLine.group("flxR5Std", "APER_FLUX_STDEV_3") + sqlLine.group("flxR5Fill", "APER_FILL_3") + sqlLine.group("flxR6", "APER_FLUX_4") + sqlLine.group("flxR6Err", "APER_FLUX_ERR_4" ) + sqlLine.group("flxR6Std", "APER_FLUX_STDEV_4") + sqlLine.group("flxR6Fill", "APER_FILL_4") + sqlLine.group("flxR7", "APER_FLUX_5") + sqlLine.group("flxR7Err", "APER_FLUX_ERR_5" ) + sqlLine.group("flxR7Std", "APER_FLUX_STDEV_5") + sqlLine.group("flxR7Fill", "APER_FILL_5") sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -383,6 +472,6 @@ #insert the things from FOrcedWarpMeasurement - self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement") - sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ") + self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement_"+forcedWarpID) + sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement_"+forcedWarpID+" as b SET ") sqlLine.group("a.objID","b.objID") sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid") @@ -403,11 +492,11 @@ # add in the psps unique ids from stuff - self.logger.infoPair("adding information from Skychip_xrad into","ForcedWarpExtended") + self.logger.infoPair("adding information from Skychip_xrad_"+forcedWarpID+" into","ForcedWarpExtended_"+forcedWarpID) # cull bad d self.logger.infoPair("need to cull"," nulls objid") - results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpExtended", "objID") + results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpExtended_"+forcedWarpID, "objID") - self.scratchDb.createIndex("ForcedWarpExtended", "objID") + self.scratchDb.createIndex("ForcedWarpExtended_"+forcedWarpID, "objID") @@ -416,8 +505,11 @@ Populates the ForcedWarpExtended table ''' - def populateForcedWarpLensing(self): - - pspsTableName = "ForcedWarpLensing" - ippTableName = "SkyChip_psf" + def populateForcedWarpLensing(self, num): + forcedWarpID = self.number[num] + pspsTableName = "ForcedWarpLensing_"+forcedWarpID + ippTableName = "SkyChip_psf_"+forcedWarpID + sql = "CREATE TABLE ForcedWarpLensing_"+forcedWarpID+" LIKE ForcedWarpLensing" + try: self.scratchDb.execute(sql) + except: pass #self.tablesToExport.append("ForcedWarpExtended") @@ -446,4 +538,6 @@ sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF") sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF") + sqlLine.group("psfE1", "LENS_E1_PSF") + sqlLine.group("psfE2", "LENS_E2_PSF") sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -454,6 +548,6 @@ #insert the things from FOrcedWarpMeasurement - self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement") - sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ") + self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement_"+forcedWarpID) + sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement_"+forcedWarpID+" as b SET ") sqlLine.group("a.objID","b.objID") sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid") @@ -474,11 +568,11 @@ # add in the psps unique ids from stuff - self.logger.infoPair("adding information from Skychip_psf into","ForcedWarpLensing") + self.logger.infoPair("adding information from Skychip_psf_"+forcedWarpID+" into","ForcedWarpLensing_"+forcedWarpID) # cull bad d self.logger.infoPair("need to cull"," nulls objid") - results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpLensing", "objID") - - self.scratchDb.createIndex("ForcedWarpLensing", "objID") + results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpLensing_"+forcedWarpID, "objID") + + self.scratchDb.createIndex("ForcedWarpLensing_"+forcedWarpID, "objID") @@ -494,30 +588,37 @@ #this should be done - self.logger.infoPair("Populating","ForcedWarpMeta") - self.populateForcedWarpMeta() - self.tablesToExport.append("ForcedWarpMeta") - - self.logger.infoPair("Populating","ForcedWarpMeasurement") - self.populateForcedWarpMeasurement() - self.tablesToExport.append("ForcedWarpMeasurement") - - self.logger.infoPair("Populating","ForcedWarpExtended") - self.populateForcedWarpExtended() - self.tablesToExport.append("ForcedWarpExtended") - - self.logger.infoPair("Populating","ForcedWarpLensing") - self.populateForcedWarpLensing() - self.tablesToExport.append("ForcedWarpLensing") - - - self.logger.infoPair("Populating","ForcedWarpToImage") - self.populateForcedWarpToImage() - self.tablesToExport.append("ForcedWarpToImage") - - - self.logger.infoPair("setting min/max objid from","ForcedWarpMeasurement") - self.setMinMaxObjID(["ForcedWarpMeasurement"]) - - self.logger.infoPair("finishing","populatePspsTables"); + #number = 0 + #for forcedWarpID in self.forcedWarpIds: + for num in self.number: + print "hawaht?" + forcedWarpID = self.number[num] + self.logger.infoPair("Populating","ForcedWarpMeta_"+forcedWarpID) + self.populateForcedWarpMeta(num) + self.tablesToExport.append("ForcedWarpMeta_"+forcedWarpID) + + self.logger.infoPair("Populating","ForcedWarpMeasurement_"+forcedWarpID) + self.populateForcedWarpMeasurement(num) + self.tablesToExport.append("ForcedWarpMeasurement_"+forcedWarpID) + + self.logger.infoPair("Populating","ForcedWarpExtended_"+forcedWarpID) + self.populateForcedWarpExtended(num) + self.tablesToExport.append("ForcedWarpExtended_"+forcedWarpID) + + self.logger.infoPair("Populating","ForcedWarpLensing_"+forcedWarpID) + self.populateForcedWarpLensing(num) + self.tablesToExport.append("ForcedWarpLensing_"+forcedWarpID) + + + self.logger.infoPair("Populating","ForcedWarpToImage_"+forcedWarpID) + self.populateForcedWarpToImage(num) + self.tablesToExport.append("ForcedWarpToImage_"+forcedWarpID) + + + self.logger.infoPair("setting min/max objid from","ForcedWarpMeasurement_"+forcedWarpID) + self.setMinMaxObjID(["ForcedWarpMeasurement_"+forcedWarpID]) + + self.logger.infoPair("finishing","populatePspsTables") + + return True @@ -528,47 +629,52 @@ ''' def importIppTables(self, columns="*", tableRE=""): + count = 0 - if self.config.retry: return True - self.logger.infoPair("Importing FW tables with table match expression: ", tableRE) - fileName = self.fits.getPath() - - self.logger.infoPair("using filename:",fileName) + for num in self.number: + forcedWarpID = self.number[num] + if self.config.retry: return True + self.logger.infoPair("loading for add id", forcedWarpID) + self.logger.infoPair("Importing FW tables with table match expression: ", tableRE) + fileName = self.fits[num].getPath() - try: - tables = stilts.treads(fileName) - except: - self.logger.errorPair("STILTS could not import from", fileName) - return False - for table in tables: + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + for table in tables: - match = re.match(tableRE, table.name) - if not match: continue - self.logger.infoPair("Reading IPP table", table.name) - table = stilts.tpipe(table, cmd='addcol table_index $0') - table = stilts.tpipe(table, cmd='explodeall') + match = re.match(tableRE, table.name) + if not match: continue + self.logger.infoPair("Reading IPP table", table.name+"_"+str(forcedWarpID)) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') # drop any previous tables before import - self.scratchDb.dropTable(table.name) + self.scratchDb.dropTable(table.name+"_"+str(forcedWarpID)) # IPP FITS files are littered with infinities, so remove these - self.logger.info("Removing Infinity values from all columns") - table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') - try: - table.write(self.scratchDb.url + '#' + table.name) - count = count + 1 - except: - self.logger.exception("Problem writing table '" + table.name + "' to the database") - - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables() + try: + table.write(self.scratchDb.url + '#' + table.name+"_"+str(forcedWarpID)) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name+"_"+str(forcedWarpID) + "' to the database") + + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables(forcedWarpID) return True - def generateRandomIDs(self): - sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)" + def generateRandomIDs(self,forcedWarpID): + sql = "UPDATE ForcedWarpMeasurement_"+forcedWarpID+" set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)" try: self.scratchDb.execute(sql) @@ -582,6 +688,7 @@ ''' - def generateUniquePspsIDs(self): - sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)" + def generateUniquePspsIDs(self,num): + + sql = "UPDATE ForcedWarpMeasurement_"+str(self.number[num])+" set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + ("+str(num)+"*1000000 ) + row)" try: self.scratchDb.execute(sql) except: @@ -594,5 +701,6 @@ def exportPspsTablesToFits(self, regex="(.*)"): - return super(ForcedWarpBatch, self).exportPspsTablesToFits("(ForcedWarp.*)") +# return super(ForcedWarpBatch, self).exportPspsTablesToFits("(ForcedWarp.*)") + return super(ForcedWarpBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") @@ -617,12 +725,12 @@ Applies indexes to the IPP tables ''' - def indexIppTables(self): - - self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xgal", "IPP_IDET") + def indexIppTables(self,forcedWarpID): + + self.logger.infoPair("Creating indexes on"+forcedWarpID, "IPP tables") + self.scratchDb.createIndex("SkyChip_psf_"+forcedWarpID, "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET") return True @@ -633,7 +741,7 @@ def updatePspsUniqueIDs(self,table): - sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)" - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair('failed sql',sql) - raise + sql = "UPDATE ForcedWarpMeasurement_"+forcedWarpID+" set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + (str(number)*1000000 ) + row)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise