Index: trunk/ippToPsps/jython/diffbatch.py =================================================================== --- trunk/ippToPsps/jython/diffbatch.py (revision 37935) +++ trunk/ippToPsps/jython/diffbatch.py (revision 38358) @@ -41,5 +41,5 @@ ippToPspsDb, scratchDb, - diffSkyFileID, + stageid, batchID): @@ -51,5 +51,5 @@ ippToPspsDb, scratchDb, - diffSkyFileID, + stageid, batchID, "DF", @@ -59,77 +59,136 @@ # get diff meta data # if error about catID you are usings an old ipptopsps_scratch db - self.logger.infoPair("getting diff" ," metadata") - meta = self.gpc1Db.getDiffStageMeta(diffSkyFileID) - self.logger.infoPair("ok, I guess that worked","now doing this") - self.diffSkyFileID = diffSkyFileID - if not meta: - self.logger.errorPair("Could not get", "diff metadata") - raise - self.logger.infoPair("so meta", "much amaze") - self.filterName = meta[0]; - self.difftypeID = meta[1]; - self.posImageID = meta[2]; - self.negImageID = meta[3]; - self.skycellName = meta[4]; - self.analVer = meta[5]; - self.expTime = meta[6]; - self.obsTime = meta[7]; - self.fits = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, diffSkyFileID) - self.header = self.fits.getPrimaryHeader() - tessName = self.getKeyValue(self.header, 'TESS_ID') - self.tessID = self.scratchDb.getTessID(tessName) - - + + # initialize list variables here + + + self.header ={} + self.diffSkyFileIDs = {} + self.filterName = {} + self.filterID = {} + self.photoCalID = {} + self.difftypeID = {} + self.posImageID = {} + self.negImageID = {} + self.skycellName = {} + self.analVer = {} + self.expTime = {} + self.obsTime = {} + self.fits = {} + self.tessID = {} + self.tessName = {} + self.skycellID = {} + self.projectionID ={} + + # number is special - it is simply a list of diff skyfile IDs, used for + # looping over things + self.stageid = stageid + self.number = {} + num = 0 + + # grab list of diffs from stage id + + self.diffSkyFileIDs = gpc1Db.getListOfDiffSkyFileIdsForDiffBatch(skychunk.dvoLabel, stageid) + + # loop over diffs + + for diffSkyFileID in self.diffSkyFileIDs: + self.number[num]= diffSkyFileID + + + + self.logger.infoPair("getting diff" ," metadata") + self.logger.infoPair("for diff skyfile id ", str(diffSkyFileID)) + meta = {} + meta = self.gpc1Db.getDiffStageMeta(int(diffSkyFileID)) + self.logger.infoPair("ok, I guess that worked","now doing this") + + if not meta: + self.logger.errorPair("Could not get", "diff metadata") + raise + self.logger.infoPair("so meta", "much amaze") + self.filterName[num] = meta[0]; + + self.difftypeID[num] = meta[1]; + self.posImageID[num] = meta[2]; + self.negImageID[num] = meta[3]; + self.skycellName[num] = meta[4]; + self.analVer[num] = meta[5]; + self.expTime[num] = meta[6]; + self.obsTime[num] = meta[7]; + self.logger.infoPair("loading diff stage cmf", diffSkyFileID) + self.fits[num] = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, int(diffSkyFileID)) + self.logger.infoPair("loading diff stage header", diffSkyFileID) + self.header[num] = self.fits[num].getPrimaryHeader() + # tessName = None + # tessName = self.getKeyValue(self.header[num], 'TESS_ID') + self.tessName[num]= meta[8]; + tessName = self.tessName[num] + self.logger.infoPair("TessName ",tessName) + self.tessID[num] = self.scratchDb.getTessID(tessName) + + self.logger.infoPair("TessName ",tessName) # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL - if tessName == 'RINGS.V3': + if self.tessName[num] == 'RINGS.V3': # skycell is, eg "skycell.1133.081" # 0123456789012345 - self.projectionID = self.skycellName[8:12] - self.skycellID = self.skycellName[13:] - self.skycellID = self.skycellName[8:] - else: + self.projectionID[num] = self.skycellName[num][8:12] + self.skycellID[num] = self.skycellName[num][13:] + self.skycellID[num] = self.skycellName[num][8:] + else: # skycell is, eg "skycell.081" # 01234567890 - self.projectionID = 0 - self.skycellID = self.skycellName[8:11] - self.skycellID=self.skycellName[8:] - if self.skycellID == "": - self.skycellID = -1 + self.projectionID[num] = 0 + self.skycellID[num] = self.skycellName[num][8:11] + self.skycellID[num]=self.skycellName[num][8:] + if self.skycellID[num] == "": + self.skycellID[num] = -1 # drop the existing tables - self.dropTableVerbose("DiffMeta") - self.dropTableVerbose("DiffToImage") - self.dropTableVerbose("DiffDetection") - + self.dropTableVerbose("DiffMeta_"+str(num)) + self.dropTableVerbose("DiffToImage_"+str(num)) + self.dropTableVerbose("DiffDetection_"+str(num)) + + if not self.analVer[num]: self.analVer[num] = -999 + self.logger.infoPair("getPhotoCalID",diffSkyFileID) + self.photoCalID[num] = str(self.scratchDb.getPhotoCalID(int(diffSkyFileID))) + self.logger.infoPair("get filterID", diffSkyFileID) + self.filterID[num] = self.scratchDb.getFilterID(self.filterName[num][0]) + self.logger.infoPair("insertdiffmeta", diffSkyFileID) + self.logger.infoPair("insertdiffmeta", self.filterName[num][0]) + self.logger.infoPair("insertdiffmeta", self.difftypeID[num]) + self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[num][0], self.difftypeID[num]) + + # make some generic tables + sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta" + try: self.scratchDb.execute(sql) + except: raise + sql = "CREATE TABLE DiffToImage_"+str(num)+" like DiffToImage" + try: self.scratchDb.execute(sql) + except: raise + sql = "CREATE TABLE DiffDetection_"+str(num)+" like DiffDetection" + try: self.scratchDb.execute(sql) + except: raise + + + + num = num + 1 + + # create an output filename, which is {expID}.FITS + + + self.outputFitsFile = "%08d.FITS" % self.stageid + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - - - self.logger.infoPair("I can haz",diffSkyFileID) - - - - if not self.analVer: self.analVer = -999 - - # create an output filename, which is {expID}.FITS - self.outputFitsFile = "%08d.FITS" % self.diffSkyFileID - self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - print "got here" - self.photoCalID = str(self.scratchDb.getPhotoCalID(self.diffSkyFileID)) - print "got photocalid" + + self.calibModNum = 0; self.dataRelease = skychunk.dataRelease; - self.filterID = self.scratchDb.getFilterID(self.filterName[0]) - - print self.filterName - print self.filterName[0] - print self.filterID - # insert what we know about this stack batch into the stack table - self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[0], self.difftypeID) # dump stuff to log - self.logger.infoPair("diff_skyfile_ID", "%d" % self.diffSkyFileID) + self.logger.infoPair("stage_id", "%d" % self.stageid) @@ -137,30 +196,31 @@ Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateDiffMeta(self): - self.logger.infoPair("Proccesing table", "DiffMeta") - - sqlLine = sqlUtility("INSERT INTO DiffMeta (") - - sqlLine.group("diffMetaID", str(self.diffSkyFileID)); - sqlLine.group("difftypeID", self.difftypeID); - sqlLine.group("posImageID", self.posImageID); - sqlLine.group("negImageID", self.negImageID); - sqlLine.group("skycellID", self.skycellID); - sqlLine.group("photoCalID", self.photoCalID); - sqlLine.group("magSat", self.getKeyFloat(self.header, "%.8f", 'FSATUR')); - sqlLine.group("analVer", str(self.analVer)); - sqlLine.group("expTime", self.expTime); - sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); - sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); - sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); - sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); - sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); - sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); - sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); - sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); - sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); - sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); - sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); - sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); + def populateDiffMeta(self,num): + diffSkyFileID = self.number[num] + self.logger.infoPair("Proccesing table", "DiffMeta_"+str(num)) + + sqlLine = sqlUtility("INSERT INTO DiffMeta_"+str(num)+" (") + + sqlLine.group("diffMetaID", str(diffSkyFileID)); + sqlLine.group("difftypeID", self.difftypeID[num]); + sqlLine.group("posImageID", self.posImageID[num]); + sqlLine.group("negImageID", self.negImageID[num]); + sqlLine.group("skycellID", self.skycellID[num]); + sqlLine.group("photoCalID", self.photoCalID[num]); + sqlLine.group("magSat", self.getKeyFloat(self.header[num], "%.8f", 'FSATUR')); + sqlLine.group("analVer", str(self.analVer[num])); + sqlLine.group("expTime", self.expTime[num]); + sqlLine.group("ctype1", self.getKeyValue(self.header[num], 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(self.header[num], 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(self.header[num], "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(self.header[num], "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(self.header[num], "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(self.header[num], "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(self.header[num], "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(self.header[num], "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(self.header[num], "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(self.header[num], "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(self.header[num], "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(self.header[num], "%.8e", 'PC002002')); sql = sqlLine.make(") VALUES ( ", ")") @@ -170,20 +230,20 @@ self.logger.errorPair('failed sql: ', sql) raise - self.logger.infoPair("DiffMeta","final stuff") - self.scratchDb.updateAllRows("DiffMeta", "batchID", str(self.batchID)) - self.scratchDb.updateAllRows("DiffMeta", "surveyID", str(self.surveyID)) - self.scratchDb.updateFilterID("DiffMeta", self.filterName[0]) - self.scratchDb.updateAllRows("DiffMeta", "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows("DiffMeta", "dataRelease", str(self.skychunk.dataRelease)) - self.logger.infoPair("DiffMeta","done") + self.logger.infoPair("DiffMeta_"+str(num),"final stuff") + self.scratchDb.updateAllRows("DiffMeta_"+str(num), "batchID", str(self.batchID)) + self.scratchDb.updateAllRows("DiffMeta_"+str(num), "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("DiffMeta_"+str(num), self.filterName[num][0]) + self.scratchDb.updateAllRows("DiffMeta_"+str(num), "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("DiffMeta_"+str(num), "dataRelease", str(self.skychunk.dataRelease)) + self.logger.infoPair("DiffMeta_"+str(num),"done") ''' Populates the Diff table for this OTA ''' - def populateDiffDetectionTable(self): - - pspsTableName = "DiffDetection" - ippTableName = "SkyChip_psf" - + def populateDiffDetectionTable(self,num): + + pspsTableName = "DiffDetection_"+str(num) + ippTableName = "SkyChip_psf_"+str(num) + diffSkyFileID = self.number[num] #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) @@ -200,5 +260,5 @@ #results['SATDET'] = 0 - extTimeString = str(self.expTime) + extTimeString = str(self.expTime[num]) # insert all detections into table : do NOT include the IGNORE unless we have to @@ -210,12 +270,12 @@ sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("diffDetMetaID", str(self.diffSkyFileID)); + sqlLine.group("diffDetMetaID", str(diffSkyFileID)); sqlLine.group("randomDiffID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") - sqlLine.group("filterID", str(self.filterID)) + sqlLine.group("filterID", str(self.filterID[num])) sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("skycellID", str(self.skycellID)) + sqlLine.group("skycellID", str(self.skycellID[num])) ## sqlLine.group("obsTime", str(self.obsTime)) - sqlLine.group("expTime", str(self.expTime)) - sqlLine.group("obsTime", str(self.obsTime)) + sqlLine.group("expTime", str(self.expTime[num])) + sqlLine.group("obsTime", str(self.obsTime[num])) sqlLine.group("xPos", "X_PSF") sqlLine.group("yPos", "Y_PSF") @@ -302,7 +362,7 @@ # insert stuff from dvo self.logger.infoPair("inserting dvo info:find imageID"," from externID") - imageID = self.scratchDb.getImageIDFromExternID(self.diffSkyFileID) - self.logger.infoPair("updating","diffdetection") - sqlLine = sqlUtility("UPDATE DiffDetection as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") + imageID = self.scratchDb.getImageIDFromExternID(diffSkyFileID) + self.logger.infoPair("updating","diffdetection_"+str(num)) + sqlLine = sqlUtility("UPDATE DiffDetection_"+str(num)+" as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") sqlLine.group("a.diffObjID", "b.objId") sqlLine.group("a.diffDetID", "b.detectID") @@ -326,12 +386,12 @@ raise self.logger.infoPair("need to cull"," nulls objid") - results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection", "diffObjID") - self.logger.infoPair("Adding 'row' columns to", "DiffDetection tables") + results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection_"+str(num), "diffObjID") + self.logger.infoPair("Adding 'row' columns to", "DiffDetection_"+str(num)) # diff has a primary key, so use this instead - self.scratchDb.addRowCountColumnNoKey("DiffDetection", "row") + self.scratchDb.addRowCountColumnNoKey("DiffDetection_"+str(num), "row") self.logger.infoPair("adding","uniquePSPSDFid") - self.updatePspsUniqueIDs("DiffDetection") + self.updatePspsUniqueIDs("DiffDetection_"+str(num),num) self.logger.infoPair("Dropping row column from", "DiffDetection table") - self.scratchDb.dropColumn("DiffDetection", "row") + self.scratchDb.dropColumn("DiffDetection_"+str(num), "row") @@ -339,15 +399,15 @@ Populates the DiffToImage table ''' - def populateDiffToImage(self): - + def populateDiffToImage(self,num): + diffSkyFileID = self.number[num] self.logger.infoPair("Procesing table", "DiffToImage") - imageIDs = self.gpc1Db.getImageIDsForThisDiffID(self.diffSkyFileID) + imageIDs = self.gpc1Db.getImageIDsForThisDiffID(int(diffSkyFileID)) for imageID in imageIDs: - sql = "INSERT INTO DiffToImage (diffMetaID, imageID) \ + sql = "INSERT INTO DiffToImage_"+str(num)+" (diffMetaID, imageID) \ VALUES (\ - " + str(self.diffSkyFileID) + ", " + imageID + ")" + " + str(diffSkyFileID) + ", " + imageID + ")" self.scratchDb.execute(sql) @@ -368,5 +428,5 @@ ''' def alterPspsTables(self): - + self.logger.debug("Altering PSPS tables") self.logger.debug("Creating indexes on PSPS tables") @@ -382,5 +442,5 @@ # loop through all OTAs and populate ImageMeta extensions # the column in PSPS - self.scratchDb.execute("ALTER TABLE DiffDetection CHANGE dec_ `dec` double") + self.scratchDb.execute("ALTER TABLE DiffDetection"+" CHANGE dec_ `dec` double") return True @@ -390,5 +450,5 @@ Applies indexes to the IPP tables ''' - def indexIppTables(self): + def indexIppTables(self,num): self.logger.infoPair("Creating indexes on", "IPP tables") @@ -396,5 +456,5 @@ #self.scratchDb.createIndex(extension, "IPP_IDET") # try the test Chip - self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_psf_"+str(num), "IPP_IDET") self.logger.infoPair("created indexes on", "IPP tables") @@ -431,9 +491,9 @@ ''' - def updatePspsUniqueIDs(self,table): + def updatePspsUniqueIDs(self,table,num): #sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \ #(select @r:=0) r, "+table+" t) as foo using (diffobjID) set \ #uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" - sql = "UPDATE DiffDetection set uniquePspsDFid = (("+str(self.batchID)+"*1000000000 ) + row)" + sql = "UPDATE DiffDetection_"+str(num)+" set uniquePspsDFid = (("+str(self.batchID)+"*1000000000 ) + ("+str(num)+"*1000000)+ row)" try: self.scratchDb.execute(sql) except: @@ -452,19 +512,17 @@ # loop through all OTAs again to update with DVO IDs self.skipBatch = False - self.tablesToExport = [] - self.tablesToExport.append("DiffMeta") - self.populateDiffMeta() - self.logger.infoPair("populated","DiffMeta") - self.tablesToExport.append("DiffToImage") - self.populateDiffToImage() - self.logger.infoPair("populated","DiffToImage") - self.tablesToExport.append("DiffDetection") - self.populateDiffDetectionTable() - self.logger.infoPair("populated","DiffDetection") - - - - - self.setMinMaxDiffObjID(["DiffDetection"]) + self.tablesToExport = [] + for num in self.number: + + self.tablesToExport.append("DiffMeta_"+str(num)) + self.populateDiffMeta(num) + self.logger.infoPair("populated","DiffMeta_"+str(num)) + self.tablesToExport.append("DiffToImage_"+str(num)) + self.populateDiffToImage(num) + self.logger.infoPair("populated","DiffToImage_"+str(num)) + self.tablesToExport.append("DiffDetection_"+str(num)) + self.populateDiffDetectionTable(num) + self.logger.infoPair("populated","DiffDetection_"+str(num)) + self.setMinMaxDiffObjID(["DiffDetection_"+str(num)]) @@ -487,42 +545,44 @@ count = 0 - fileName = self.fits.getPath() + for num in self.number: + diffSkyFileID = self.number[num] + fileName = self.fits[num].getPath() - self.logger.infoPair("using filename:",fileName) - - try: - tables = stilts.treads(fileName) - except: - self.logger.errorPair("STILTS could not import from", fileName) - return False + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False # count = 0 - for table in tables: + for table in tables: - match = re.match(tableRE, table.name) - if not match: continue - - self.logger.infoPair("Reading IPP table", table.name) - table = stilts.tpipe(table, cmd='addcol table_index $0') - table = stilts.tpipe(table, cmd='explodeall') + match = re.match(tableRE, table.name) + if not match: continue + + self.logger.infoPair("Reading IPP table", table.name+"_"+str(num)) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') # drop any previous tables before import - self.scratchDb.dropTable(table.name) + self.scratchDb.dropTable(table.name+"_"+str(num)) # IPP FITS files are littered with infinities, so remove these - self.logger.info("Removing Infinity values from all columns") - table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') - try: - table.write(self.scratchDb.url + '#' + table.name) - count = count + 1 - except: - self.logger.exception("Problem writing table '" + table.name + "' to the database") + try: + table.write(self.scratchDb.url + '#' + table.name+"_"+str(num)) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name+"_"+str(num) + "' to the database") - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables() + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables(num) return True