Index: trunk/ippToPsps/jython/diffbatch.py =================================================================== --- trunk/ippToPsps/jython/diffbatch.py (revision 37749) +++ trunk/ippToPsps/jython/diffbatch.py (revision 37750) @@ -6,5 +6,5 @@ import time import stilts - +import re from java.lang import * from java.sql import * @@ -178,16 +178,16 @@ Populates the Diff table for this OTA ''' - def populateDiffTable(self, ota, results): + def populateDiffDetectionTable(self): pspsTableName = "DiffDetection" - ippTableName = "diff_psf" + ippTableName = "SkyChip_psf" - results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) + #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) # drop then re-create table - self.scratchDb.dropTable(pspsTableName) - sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection" - try: self.scratchDb.execute(sql) - except: pass + #self.scratchDb.dropTable(pspsTableName) + #sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection" + #try: self.scratchDb.execute(sql) + #except: pass # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5) @@ -195,5 +195,5 @@ BEFORE = self.scratchDb.getRowCount(ippTableName) - results['SATDET'] = 0 + #results['SATDET'] = 0 extTimeString = str(self.expTime) @@ -206,9 +206,11 @@ # XXX apFluxF or apFluxNpix + apFluxRadius? - sqlLine.group("ippDetectID", "IPP_IDET") + sqlLine.group("ippDetectID", "IPP_IDET") + sqlLine.group("diffDetMetaID", str(self.diffSkyFileID)); sqlLine.group("randomDiffID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") sqlLine.group("filterID", str(self.filterID)) - sqlLine.group("surveyID", str(self.surveyID)) - ##sqlLine.group("obsTime", str(self.obsTime)) I have expTime + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("skycellID", str(self.skycellID)) +## sqlLine.group("obsTime", str(self.obsTime)) sqlLine.group("expTime", str(self.expTime)) sqlLine.group("xPos", "X_PSF") @@ -228,8 +230,8 @@ sqlLine.group("psfFlux", "PSF_INST_FLUX / " + extTimeString) sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / " + extTimeString) - if (self.id >= 982483): - sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ") - sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN") - sqlLine.group("psfCore", "PSF_CORE") +# if (self.id >= 982483): #we don't have PSF_FWHM_MAJ or MIN or PSF_CORE +# sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ") +# sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN") +# sqlLine.group("psfCore", "PSF_CORE") sqlLine.group("psfTheta", "PSF_THETA") sqlLine.group("psfQf", "PSF_QF") @@ -249,5 +251,5 @@ sqlLine.group("apFlux", "AP_FLUX / " + extTimeString) sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString) - sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))") + # sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))") # sqlLine.group("apRadius", "AP_MAG_RADIUS") sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString) @@ -291,8 +293,22 @@ # we don't delete these anymore - results['NULLINSTFLUX'] = 0; + #results['NULLINSTFLUX'] = 0; # insert stuff from dvo + self.logger.infoPair("inserting dvo info:find imageID"," from externID") + imageID = self.scratchDb.getImageIDFromExternID(self.diffSkyFileID) + self.logger.infoPair("updating","diffdetection") + sqlLine = sqlUtility("UPDATE DiffDetection as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") + sqlLine.group("a.diffObjID", "b.objId") + + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql:",sql) + raise + self.logger.infoPair("need to cull"," nulls objid") + self.logger.infoPair("need to add","uniquePSPSDFid") @@ -304,5 +320,5 @@ self.logger.infoPair("Procesing table", "DiffToImage") - imageIDs = self.gpc1Db.getImageIDsForThisDiffID(diffkmetaID) + imageIDs = self.gpc1Db.getImageIDsForThisDiffID(self.diffSkyFileID) for imageID in imageIDs: @@ -310,5 +326,5 @@ sql = "INSERT INTO DiffToImage (diffMetaID, imageID) \ VALUES (\ - " + str(diffmetaID) + ", " + imageID + ")" + " + str(self.diffSkyFileID) + ", " + imageID + ")" self.scratchDb.execute(sql) @@ -335,5 +351,5 @@ self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID") - self.populateDiffMeta() + #self.populateDiffMeta() # dictionary objects to hold imageIDs for later @@ -355,7 +371,7 @@ self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex(extension, "IPP_IDET") + #self.scratchDb.createIndex(extension, "IPP_IDET") # try the test Chip - self.scratchDb.createIndex("Chip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") self.logger.infoPair("created indexes on", "IPP tables") @@ -391,7 +407,7 @@ def updatePspsUniqueIDs(self,table): - sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \ + sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \ (select @r:=0) r, "+table+" t) as foo using (objID) set \ - uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" + uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" try: self.scratchDb.execute(sql) except: @@ -402,85 +418,4 @@ Does the processing, i.e. pulling stuff from IPP tables into PSPS tables ''' - def populatePspsTablesChip(self, chipname, x, y, results, tables): - # XXX EAM NOTE: drop tables Detection_* here so - # they do not polute the db? - # XXX or put an explicit drop at the end of the loop? - - # self.logger.infoTitle("Processing " + chipname) - # this is a bit crude: if the chip is not present, this test will fail and the chip - # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing") - - # XXX keep this in or not? - # if not self.scratchDb.astrometricSolutionOK(chipname): - # self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) - # return False - - # does this chip exist in the DVO image table (if NOT, then skip it) - if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]): - self.logger.info("| %5s | ------------- Chip not in DVO : rejecting ------------ |" % chipname) - return False - - #self.logger.info("populate stuff "); - # populate remainder of tables - self.populateDetectionTable(chipname, results) - #self.logger.info("successful populate "); - # now add DVO IDs - self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) - #self.logger.info("updated dvoids") - results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") - #self.logger.info("deleted nulls") - self.logger.info("add psps unique p2 ids") - self.updatePspsUniqueIDs("Detection_" + chipname, self.imageIDs[chipname]) - self.updateImageID("Detection_" + chipname, x, y) - #self.logger.info("updateImageId") - rowCount = self.scratchDb.getRowCount("Detection_" + chipname) - #self.logger.info("got row count") - self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", - chipname, - results['ORIGINALTOTAL'], - results['SATDET'], - results['NULLINSTFLUX'], - results['NULLOBJID'], - rowCount) - self.totalOriginal = self.totalOriginal + results['ORIGINALTOTAL'] - self.totalSatDet = self.totalSatDet + results['SATDET'] - self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX'] - self.totalNullObjID = self.totalNullObjID + results['NULLOBJID'] - self.totalDetections = self.totalDetections + rowCount - - fractionKept = float(rowCount) / float(results['ORIGINALTOTAL']) - # print "kept ", str(fractionKept), " detections from ", chipname - - if fractionKept < 0.8: - # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname - self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname) - self.skipBatch = True - return False; - - #self.logger.info("updated totals") - # check we have something in this Detection table TODO add this to table above - if rowCount < 1: - self.logger.infoPair("Empty table for chip, fail on exposure", chipname) - self.skipBatch = True - return False; - - # update ImageMeta with count of detections for this CHIPNAME and photoCodeID - sql = "UPDATE ImageMeta_" + chipname + " \ - SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname])) - self.scratchDb.execute(sql) - - # add these to list of tables to export later - self.tablesToExport.append("ImageMeta_" + chipname) - self.logger.info("export ImageMeta") - - self.tablesToExport.append("Detection_" + chipname) - self.logger.info("export Detection") - - tables.append("Detection_" + chipname) - self.logger.info("updated detection") - - self.validChips.append(chipname) - - return True ''' @@ -494,9 +429,11 @@ self.tablesToExport.append("DiffMeta") self.populateDiffMeta() + self.logger.infoPair("populated","DiffMeta") self.tablesToExport.append("DiffToImage") self.populateDiffToImage() + self.logger.infoPair("populated","DiffToImage") self.tablesToExport.append("DiffDetection") - self.populateDiffDetection() - + self.populateDiffDetectionTable() + self.logger.infoPair("populated","DiffDetection") @@ -575,25 +512,61 @@ else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials) - - ''' - This function reads the cmf/smf file and loads it into the database as its own table - Overriding this method. Filter to only import *.psf extensions - ''' - def importIppTables(self, filter=""): - - regex = ".*.psf" - if False and self.config.test and self.config.camera == "gpc1": - regex = "XY01.psf" + ''' + Imports IPP tables from FITS file + Accepts a regular expression to match the tables so not all tables need to be imported + (This regex feature is not currently used...) + ''' + def importIppTables(self, columns="*", tableRE=""): + + if self.config.retry: return True + + self.logger.infoPair("Importing DF tables with table match expression: ", tableRE) + + count = 0 + fileName = self.fits.getPath() - print "my ID: " + str(self.id) - - if (self.id < 982483): - # XXX EAM NOTE : this is fragile : requires PS1_V4 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_MAJOR PSF_MINOR PSF_THETA PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2" - else: - # XXX EAM NOTE : this is fragile : requires PS1_V5 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2" - - return super(DetectionBatch, self).importIppTables(columns, regex) + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + # count = 0 + for table in tables: + + match = re.match(tableRE, table.name) + if not match: continue + + self.logger.infoPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + try: + table.write(self.scratchDb.url + '#' + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True + + + + + ''' @@ -601,3 +574,12 @@ ''' def exportPspsTablesToFits(self, regex="(.*)"): - return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") + return super(DiffBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") + + + ''' + Drop a table and report hte drop + ''' + def dropTableVerbose(self, table): + self.logger.infoPair("dropping table:",table) + self.scratchDb.dropTable(table) +