Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 38836) +++ /trunk/ippToPsps/jython/batch.py (revision 38837) @@ -234,13 +234,13 @@ ''' - Returns the string keyword value from this header or else "NULL" + Returns the float keyword value from this header or else "NULL" ''' def getKeyFloat(self, header, format, key): if key in header: - value = format % float(header[key]) + value = float(format % float(header[key])) else: self.logger.errorPair("Missing header field", key) - value = format % -999.9 + value = float(format % -999.9) return value @@ -414,13 +414,12 @@ ''' def createEmptyPspsTables(self): - print "THIS SUCKS" - self.pspsTables = stilts.treads(self.pspsVoTableFilePath) - for table in self.pspsTables: - self.logger.debug("Creating PSPS table: " + table.name) - self.logger.infoPair("creating psps table ",table.name) - table.write(self.scratchDb.url + '#' + table.name) - self.tablesToExport.append(table.name) - - return True + self.pspsTables = stilts.treads(self.pspsVoTableFilePath) + for table in self.pspsTables: + self.logger.debug("Creating PSPS table: " + table.name) + self.logger.infoPair("creating psps table ",table.name) + table.write(self.scratchDb.url + '#' + table.name) + self.tablesToExport.append(table.name) + + return True ''' Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 38836) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 38837) @@ -3,4 +3,5 @@ import os.path import sys +import math import glob import time @@ -184,10 +185,12 @@ def populateImageMetaTable(self, ota, header): + # the supplied 'header' matches this chip + # self.header is the PHU header for this smf + tableName = "ImageMeta_" + ota - # we drop the table before calling this functoin so it is not left behind on failure + # we drop the table before calling this function so it is not left behind on failure sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" - try: self.scratchDb.execute(sql) - except: pass + self.scratchDb.execute(sql) if (ota[0:2] == "XY"): ccdID = ota[2:4] @@ -200,5 +203,5 @@ psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') - psfFwhm = 0.5*(float(psfFwhmMajor) + float(psfFwhmMinor)) + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) psfmodel_name = self.getKeyValue(header, 'PSFMODEL') @@ -207,45 +210,52 @@ ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = sqrt(ast_cdx^2 + ast_cdy^2) + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + + # Convert detectionThreshold to appropriate magnitudes + detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') + expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') + zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + + # XXX zp correction should come from DVO + detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) # insert image metadata into table sqlLine = sqlUtility("INSERT INTO " + tableName + "(") - ## extract ZPT_OBS and define the zpFactor, apply to sky, skyScat? - - sqlLine.group("frameID", str(self.expID)) - sqlLine.group("ccdID", str(ccdID)) - sqlLine.group("bias", str(self.bias)) - sqlLine.group("biasScat", str(self.biasScat)) - sqlLine.group("sky", self.getKeyFloat(header, "%.8f", 'MSKY_MN')) - sqlLine.group("skyScat", self.getKeyFloat(header, "%.8f", 'MSKY_SIG')) - sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) - sqlLine.group("astroScat", astroscat) - sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')) - sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) - sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) - sqlLine.group("nAxis1", self.getKeyInt(header, 0, 'NAXIS1')) - sqlLine.group("nAxis2", self.getKeyInt(header, 0, 'NAXIS2')) - sqlLine.group("psfModelID" psfmodelID) - sqlLine.group("psfFwhm", str(psfFwhm)) - sqlLine.group("psfWidMajor", psfFwhmMajor) - sqlLine.group("psfWidMinor", psfFwhmMinor) - sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) - sqlLine.group("momentMajor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1')) - sqlLine.group("momentMinor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2')) - sqlLine.group("momentM2C", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C')) - sqlLine.group("momentM2S", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S')) - sqlLine.group("momentM3", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3')) - sqlLine.group("momentM4", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4')) - sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) - sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) - sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) - sqlLine.group("qaFlags", str(self.scratchDb.getDvoImageFlags(header['IMAGEID']))) - sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) - sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) - sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) - #what happens if it can't find it? - # CZW: I think this should properly trap the fringe - if self.getKeyValue(header, 'DETREND.FRINGE') != "NULL": + sqlLine.group("frameID", self.expID) + sqlLine.group("ccdID", ccdID) + sqlLine.group("bias", self.bias) + sqlLine.group("biasScat", self.biasScat) + sqlLine.group("sky", self.getKeyFloat(header, "%.8f", 'MSKY_MN')) + sqlLine.group("skyScat", self.getKeyFloat(header, "%.8f", 'MSKY_SIG')) + sqlLine.group("detectionThreshold", detectionThreshold) + sqlLine.group("astroScat", astroscat) + sqlLine.group("photoScat", self.getKeyFloat(header, "%.8f", 'ZPT_ERR')) + sqlLine.group("photoZero", zp) + sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nAxis1", self.getKeyInt(header, 0, 'NAXIS1')) + sqlLine.group("nAxis2", self.getKeyInt(header, 0, 'NAXIS2')) + sqlLine.group("psfModelID", psfmodelID) + sqlLine.group("psfFWHM", psfFWHM) + sqlLine.group("psfWidMajor", psfFwhmMajor) + sqlLine.group("psfWidMinor", psfFwhmMinor) + sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) + sqlLine.group("momentMajor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1')) + sqlLine.group("momentMinor", pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2')) + sqlLine.group("momentM2C", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C')) + sqlLine.group("momentM2S", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S')) + sqlLine.group("momentM3", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3')) + sqlLine.group("momentM4", pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4')) + sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) + sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) + sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) + sqlLine.group("qaFlags", self.scratchDb.getDvoImageFlags(header['IMAGEID'])) + sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) + sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) + sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) + + # DETREND.FRINGE is not required : check if it is in the header before setting + if 'DETREND.FRINGE' in header: sqlLine.group("detrend4", self.getKeyValue(header, 'DETREND.FRINGE')) @@ -256,5 +266,4 @@ # sqlLine.group("detrend8", " ") - sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')) sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')) @@ -307,12 +316,9 @@ sql = sqlLine.make(") VALUES ( ", ")") - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair('failed sql: ', sql) - raise + self.scratchDb.execute(sql) self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "processingVersion", str(self.skychunk.processingVersion)) + if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -428,4 +434,7 @@ # instrumental fluxes in counts/sec and calibrated sizes (arcsec). # below, we convert instrumental fluxes to Jy + + # NOTE: math operations below take place in SQL, not JYTHON. thus we use (e.g.) + # abs() not math.fabs() sqlLine.group("ippDetectID", "IPP_IDET") @@ -644,4 +653,5 @@ ota = "XY%d%d" % (x, y) + if ota not in self.imageIDs: continue # I need better control over this.. @@ -682,6 +692,10 @@ if x==7 and y==7: continue - extension = "XY%d%d_psf" % (x, y) + ota = "XY%d%d" % (x, y) + if ota not in self.imageIDs: continue + + extension = ota + "_psf" self.scratchDb.createIndex(extension, "IPP_IDET") + # try the test Chip self.scratchDb.createIndex("Chip_psf", "IPP_IDET") @@ -698,33 +712,28 @@ # XXX : EAM 20150925 : is this IGNORE necessary? - sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ - a.objID = b.objID, \ - a.detectID = b.detectID, \ - a.ippObjID = b.ippObjID, \ - a.dvoRegionID = b.catID, \ - a.ra = b.ra, \ - a.dec = b.dec_, \ - a.zp = b.zp, \ - a.telluricExt = b.telluricExt, \ - a.airmass = b.airmass, \ - a.infoFlag3 = b.flags, \ - a.expTime = b.expTime, \ - a.psfFlux = a.psfFlux * b.zpFactor, \ - a.psfFluxErr = a.psfFluxErr * b.zpFactor, \ - a.apFlux = a.apFlux * b.zpFactor, \ - a.apFluxErr = a.apFluxErr * b.zpFactor, \ - a.kronFlux = a.kronFlux * b.zpFactor, \ - a.kronFluxErr = a.kronFluxErr * b.zpFactor \ - a.sky = a.sky * b.zpFactor, \ - a.skyErr = a.skyErr * b.zpFactor \ - WHERE a.ippDetectID = b.ippDetectID \ - AND b.imageID = " + str(imageID) - - ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.psfFluxErr = a.psfFluxErr * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.apFlux = a.apFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.apFluxErr = a.apFluxErr * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.kronFlux = a.kronFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ - ## a.kronFluxErr = a.kronFluxErr * 3630.78 * POW(10.0, -0.4*b.zp) \ + sqlLine = sqlUtility("UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET") + sqlLine.group("a.objID", "b.objID") + sqlLine.group("a.detectID", "b.detectID") + sqlLine.group("a.ippObjID", "b.ippObjID") + sqlLine.group("a.dvoRegionID", "b.catID") + sqlLine.group("a.ra", "b.ra") + sqlLine.group("a.dec", "b.dec_") + sqlLine.group("a.zp", "b.zp") + sqlLine.group("a.telluricExt", "b.telluricExt") + sqlLine.group("a.airmass", "b.airmass") + sqlLine.group("a.infoFlag3", "b.flags") + sqlLine.group("a.expTime", "b.expTime") + sqlLine.group("a.psfFlux", "a.psfFlux * b.zpFactor") + sqlLine.group("a.psfFluxErr", "a.psfFluxErr * b.zpFactor") + sqlLine.group("a.apFlux", "a.apFlux * b.zpFactor") + sqlLine.group("a.apFluxErr", "a.apFluxErr * b.zpFactor") + sqlLine.group("a.kronFlux", "a.kronFlux * b.zpFactor") + sqlLine.group("a.kronFluxErr", "a.kronFluxErr * b.zpFactor") + sqlLine.group("a.sky", "a.sky * b.zpFactor") + sqlLine.group("a.skyErr", "a.skyErr * b.zpFactor") + + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) + + ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ # instrumental flux vs Janskies: @@ -741,9 +750,6 @@ # NOTE: update dvopsps to populate zpFactor = 3630.78 * ten(-0.4*zp) - print sql - try: self.scratchDb.execute(sql) - except: - self.logger.infoPair("failed sql",sql) - raise + self.scratchDb.execute(sql) + ''' Updates table and generates pspsuniqueids @@ -929,5 +935,5 @@ # update FrameMeta with count OTAs in this file and total number of photometric reference sources - sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) + sql = "UPDATE FrameMeta SET nOTA = %d, nPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) self.scratchDb.execute(sql) #to make it stop @@ -939,16 +945,8 @@ def removeDuplicateObjects(self): - sql = "DROP TABLE SkinnyObject_All" - try: - self.scratchDb.execute(sql) - except: - print "SkinnyObject_All table not yet defined" + self.scratchDb.dropTable("SkinnyObject_All") sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))" - try: - self.scratchDb.execute(sql) - except: - print "failed to create table SkinnyObject_All" - raise + self.scratchDb.execute(sql) for chipname in self.validChips: @@ -956,10 +954,5 @@ sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \ SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname - try: - self.scratchDb.execute(sql) - except: - print "failed to insert entry SkinnyObject_All: ", chipname - raise - + self.scratchDb.execute(sql) Nduplicates = 0 Index: /trunk/ippToPsps/jython/dvo.py =================================================================== --- /trunk/ippToPsps/jython/dvo.py (revision 38836) +++ /trunk/ippToPsps/jython/dvo.py (revision 38837) @@ -167,14 +167,12 @@ return False - self.logger.infoSeparator() - self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") - - # drop dvoSkyTable (if it exists) - sql = "drop TABLE dvoSkyTable" - try: self.scratchDb.execute(sql) - except: pass + # self.logger.infoSeparator() + # self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") + # print "--- go" + + self.scratchDb.dropTable("dvoSkyTable") # create dvoSkyTable - sql = "CREATE TABLE dvoSkyTable (R_MIN REAL, R_MAX REAL, D_MIN REAL, D_MAX REAL, INDEX_ INT, NAME CHAR(18))" + sql = "CREATE TABLE dvoSkyTable (R_MIN REAL, R_MAX REAL, D_MIN REAL, D_MAX REAL, REGION_ID INT, NAME CHAR(18))" self.scratchDb.execute(sql) @@ -196,5 +194,5 @@ self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) - self.scratchDb.createIndex(self.scratchDb.dvoSkyTable, "INDEX") + self.scratchDb.createIndex(self.scratchDb.dvoSkyTable, "REGION_ID") self.scratchDb.setImportedThisDvoTable(path) @@ -580,14 +578,13 @@ self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost) - if True: - print "*******************************************************************************" + if False: self.logger.warning("about to delete detection table and re-ingest; is this correct?") - #response = raw_input("(y/n) ") - #if response != "y": - # raise - #self.logger.warning("Are you ABSOLUTELY sure you want to do this?") - #response = raw_input("(y/n) ") - #if response != "y": - # raise + response = raw_input("(y/n) ") + if response != "y": + raise + self.logger.warning("Are you ABSOLUTELY sure you want to do this?") + response = raw_input("(y/n) ") + if response != "y": + raise # blow away existing dvoDetection table & re-crate Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38836) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38837) @@ -404,4 +404,5 @@ imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID[num]) self.logger.infoPair("updating","forcedWarpMeasurement_"+forcedWarpID) + sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement_"+forcedWarpID+" as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") sqlLine.group("a.objID", "b.objID") @@ -414,12 +415,12 @@ sqlLine.group("a.FinfoFlag3", "b.flags") - sqlLine.group("a.FpsfFlux", "a.FpsfFlux * b.zpFactor") - sqlLine.group("a.FpsfFluxErr", "a.FpsfFluxErr * b.zpFactor") - sqlLine.group("a.FapFlux", "a.FapFlux * b.zpFactor") - sqlLine.group("a.FapFluxErr", "a.FapFluxErr * b.zpFactor") - sqlLine.group("a.FkronFlux", "a.FkronFlux * b.zpFactor") - sqlLine.group("a.FkronFluxErr","a.FkronFluxErr * b.zpFactor") - sqlLine.group("a.Fsky", "a.Fsky * b.zpFactor") - sqlLine.group("a.FskyErr", "a.FskyErr * b.zpFactor") + sqlLine.group("a.FpsfFlux", "a.FpsfFlux * b.zpFactor") + sqlLine.group("a.FpsfFluxErr", "a.FpsfFluxErr * b.zpFactor") + sqlLine.group("a.FapFlux", "a.FapFlux * b.zpFactor") + sqlLine.group("a.FapFluxErr", "a.FapFluxErr * b.zpFactor") + sqlLine.group("a.FkronFlux", "a.FkronFlux * b.zpFactor") + sqlLine.group("a.FkronFluxErr", "a.FkronFluxErr * b.zpFactor") + sqlLine.group("a.Fsky", "a.Fsky * b.zpFactor") + sqlLine.group("a.FskyErr", "a.FskyErr * b.zpFactor") sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) Index: /trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /trunk/ippToPsps/jython/ipptopspsdb.py (revision 38836) +++ /trunk/ippToPsps/jython/ipptopspsdb.py (revision 38837) @@ -914,12 +914,27 @@ ''' + Returns boolean value for this column in clients table + ''' + def existsClient(self, host, pid): + + sql = "SELECT id FROM clients WHERE host = '" + host + "' AND pid = " + str(pid) + " LIMIT 1" + + try: + rs = self.executeQuery(sql) + if rs.first() is False: return False + except: + self.logger.exception("Unable to check if " + host + "-" + str(pid) + " is in clients table") + + return True + + ''' Update clients, or inserts it for the first time ''' def updateClient(self, type, host, pid): - try: + if self.existsClient(host, pid): + self.execute("UPDATE clients SET timestamp = now() WHERE host = '" + host + "' AND pid = " + str(pid)) + else: self.insertClient(type, host, pid) - except: - self.execute("UPDATE clients SET timestamp = now() WHERE host = '" + host + "' AND pid = " + str(pid)) ''' @@ -1337,15 +1352,11 @@ for id in ids: - sql = "INSERT IGNORE INTO pending \ + # why is there an IGNORE here? we have deleted old versions already above + sql = "INSERT INTO pending \ (box_id, batch_type, stage_id) \ VALUES \ (" + str(box_id) + ", '" + batchType + "', " + str(id) + ")" - print sql - try: self.execute(sql) - except: - print "failed to insert into pending" - print "sql: " + sql - raise + self.execute(sql) ''' @@ -1656,12 +1667,8 @@ def storeAllItems(self, rows): - try: - self.execute("DROP TABLE all_pending") - except: - pass + self.dropTable("all_pending") self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)") for row in rows: - try: sql = "INSERT INTO all_pending (stage_id, ra_bore, dec_bore) \ @@ -1678,9 +1685,4 @@ ''' def storeAllItemsInDvodb(self, rows, dvo_db, batchType): - - #try: - # self.execute("DROP TABLE all_pending") - #except: - # pass #self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)") @@ -1699,6 +1701,4 @@ #self.logger.infoPair("Items written to Db", "%d" % count) - - ''' Gets all items in the all_pending temporary table within the bounds of this box Index: /trunk/ippToPsps/jython/mysql.py =================================================================== --- /trunk/ippToPsps/jython/mysql.py (revision 38836) +++ /trunk/ippToPsps/jython/mysql.py (revision 38837) @@ -230,5 +230,5 @@ def dropTable(self, table): - sql = "DROP TABLE " + table + sql = "DROP TABLE IF EXISTS " + table try: self.execute(sql) except: return @@ -334,5 +334,11 @@ stmt = self.con.createStatement() - stmt.execute(sql) + + try: stmt.execute(sql) + except Exception, e: + print "--- Error calling mysql: " + print "--- sql: " + sql + print "--- " + str(e) + os._exit(2) stmt.close() Index: /trunk/ippToPsps/jython/scratchdb.py =================================================================== --- /trunk/ippToPsps/jython/scratchdb.py (revision 38836) +++ /trunk/ippToPsps/jython/scratchdb.py (revision 38837) @@ -475,9 +475,6 @@ # drop and create Images table self.logger.infoPair("Creating DVO table", dvoImagesTable) - sql = "DROP TABLE " + dvoImagesTable - - try: self.execute(sql) - except: pass - + self.dropTable(dvoImagesTable) + sql = "CREATE TABLE " + dvoImagesTable + " ( \ SOURCE_ID SMALLINT, \ @@ -495,9 +492,5 @@ # now detection table self.logger.infoPair("Dropping DVO table", dvoDetectionTable) - sql = "DROP TABLE " + dvoDetectionTable + " IF EXISTS" - try: self.execute(sql) - except: - self.logger.errorPair("problem dropping DVO table", dvoDetectionTable) - pass + self.dropTable(dvoDetectionTable) self.logger.infoPair("Creating DVO table", "dvoDetection") @@ -514,4 +507,5 @@ decErr FLOAT, \ zp FLOAT, \ + zpFactor FLOAT, \ telluricExt FLOAT, \ airmass FLOAT, \ @@ -554,9 +548,5 @@ tableName = self.getDbFriendlyTableName(region + "." + fileType) - sql = "DROP TABLE " + tableName - try: self.execute(sql) - except: - self.logger.errorPair("Unable to drop table", tableName) - pass + self.dropTable(tableName) # deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable) @@ -583,16 +573,10 @@ sql = "set @@max_heap_table_size = " + str(tableSize) - try: self.execute(sql) - except: - self.logger.errorPair("Unable to set max MEMORY table size for ", self.dvoDetectionTable) - return False + self.execute(sql) # create detections table self.logger.infoPair("Creating table", self.dvoDetectionTable) sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection" - try: self.execute(sql) - except: - self.logger.errorPair("Unable to create table", self.dvoDetectionTable) - return False + self.execute(sql) # XXX changing the dvoDetectionFull table to MEMORY @@ -660,5 +644,5 @@ sql = "SELECT \ - INDEX_, \ + REGION_ID, \ R_MIN+ (R_MAX - R_MIN)/2.0, \ D_MIN + (D_MAX - D_MIN)/2.0, \ @@ -689,5 +673,5 @@ region = "" - sql = "SELECT NAME FROM " + self.dvoSkyTable + " WHERE INDEX_ = " + str(index) + sql = "SELECT NAME FROM " + self.dvoSkyTable + " WHERE REGION_ID = " + str(index) try: rs = self.executeQuery(sql) Index: /trunk/ippToPsps/jython/setupScratchDb.py =================================================================== --- /trunk/ippToPsps/jython/setupScratchDb.py (revision 38836) +++ /trunk/ippToPsps/jython/setupScratchDb.py (revision 38837) @@ -88,5 +88,10 @@ self.logger.infoPair("Installing", "initialization tables") tablepath = self.config.configDir + "tables.IN.vot" - tables = stilts.treads(tablepath) + try: + tables = stilts.treads(tablepath) + except Exception, e: + print "--- problem reading tables.IN.vot" + print "--- " + str(e) + os._exit(3) for table in tables: @@ -94,5 +99,6 @@ self.logger.infoPair("Creating IN table: ", table.name) self.logger.debug("Creating IN table: " + table.name) - table.write(self.scratchDb.url + '#' + table.name) + try: table.write(self.scratchDb.url + '#' + table.name) + except Exception, e: print "--- " + str(e) # create basic DVO tables Index: /trunk/ippToPsps/jython/sqlUtility.py =================================================================== --- /trunk/ippToPsps/jython/sqlUtility.py (revision 38836) +++ /trunk/ippToPsps/jython/sqlUtility.py (revision 38837) @@ -12,10 +12,11 @@ self.fields = [] self.values = [] - self.startString = startString + self.startString = str(startString) return def group(self,field,value): - self.fields.append(field) - self.values.append(value) + self.fields.append(str(field)) + self.values.append(str(value)) + print "* " + str(field) + " = " + str(value) return @@ -30,5 +31,5 @@ output += ", " - output += " " + middleString + " " + output += " " + str(middleString) + " " for i in range(len(self.values)): @@ -37,5 +38,5 @@ output += ", " - output += " " + endString + output += " " + str(endString) return output @@ -51,5 +52,5 @@ output += ", " - output += " " + middleString + " " + output += " " + str(middleString) + " " for i in range(len(self.values)): @@ -58,5 +59,5 @@ output += ", " - output += " " + endString + output += " " + str(endString) return output @@ -72,5 +73,5 @@ output += ", " - output += " " + endString + output += " " + str(endString) return output Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 38836) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 38837) @@ -160,4 +160,5 @@ self.dropTableVerbose(tableName) + self.dropTableVerbose("StackMeta") self.dropTableVerbose("StackObjectThin") self.dropTableVerbose("StackObjectAttributes") @@ -176,4 +177,5 @@ self.dropTableVerbose("StackToImage") + self.dropTableVerbose("StackDetEffMeta") # delete IPP tables @@ -217,19 +219,8 @@ header = self.headerSet[filter] - tablename = filter + "StackMeta" + stackID = self.stackIDs[filter] filterName = filter + ".00000" - self.logger.infoPair("Populating table", tablename) - - fwhm_maj = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') - fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ') - psfmodel_name = self.getKeyValue(header, 'PSFMODEL') - psfmodelID = self.scratchDb.getFitModelID(psfmodel_name) - - ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') - ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = sqrt(ast_cdx^2 + ast_cdy^2) - # make a table filterID = self.scratchDb.getFilterID(filter) @@ -238,52 +229,74 @@ photoCalID = str(self.scratchDb.getPhotoCalID(stackID)) + tablename = filter + "StackMeta" + + sql = "CREATE TABLE " + tablename + " LIKE StackMeta" + self.scratchDb.execute(sql) + + self.logger.infoPair("Populating table", tablename) + + # fwhm_maj = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + # fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ') + + # XXX hard-wired platescale : 0.25 + pltscale = 0.25 + pltscale2 = pltscale * pltscale + + psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) + + psfmodel_name = self.getKeyValue(header, 'PSFMODEL') + psfmodelID = self.scratchDb.getFitModelID(psfmodel_name) + + ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') + ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + # Convert detectionThreshold to appropriate magnitudes detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') - zp = self.getKeyFloat(header, "%.8f", 'FPA.ZP') - # CZW check? + zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + + # XXX zp correction should come from DVO detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) - - # mysql is sensitive to values which are ambiugously float. eg - # a warning is raised if we try to insert '25.' into a float field. - # use getKeyFloat(hdr, format, key) to avoid this problem - - sql = "CREATE TABLE " + tablename + " LIKE StackMeta" - try: self.scratchDb.execute(sql) - except: pass - + # insert stack metadata into table sqlLine = sqlUtility("INSERT INTO " + tablename + " (") - sqlLine.group("stackImageID", str(stackID)) - sqlLine.group("batchID", str(self.batchID)) - sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("filterID", str(filterID)) - sqlLine.group("stackTypeID", str(self.stackTypeID)) - sqlLine.group("tessID", str(self.tessID)) - sqlLine.group("projectionID", str(self.projectionID)) - sqlLine.group("skyCellID", str(self.skycellID)) - sqlLine.group("photoCalID", photoCalID) - sqlLine.group("analysisVer", str(self.analysisVer)) - sqlLine.group("detectionThreshold", detectionThreshold) - sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) - sqlLine.group("psfModelID", psfmodelID) - sqlLine.group("psfFwhm_mean", fwhm_maj) - sqlLine.group("psfFwhm_max", fwhm_maj_uq) - sqlLine.group("astroScat", astroscat) - sqlLine.group("photoZero", self.getKeyFloat(header, "%.5f", 'FPA.ZP')) - sqlLine.group("ctype1", header['CTYPE1']) - sqlLine.group("ctype2", header['CTYPE2']) - sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) - sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) - sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) - sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) - sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) - sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) - sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) - sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) - sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) - sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) - sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + sqlLine.group("stackImageID", str(stackID)) + sqlLine.group("batchID", str(self.batchID)) + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("filterID", str(filterID)) + sqlLine.group("stackTypeID", str(self.stackTypeID)) + sqlLine.group("tessID", str(self.tessID)) + sqlLine.group("projectionID", str(self.projectionID)) + sqlLine.group("skyCellID", str(self.skycellID)) + sqlLine.group("photoCalID", photoCalID) + sqlLine.group("analysisVer", str(self.analysisVer)) + sqlLine.group("detectionThreshold", detectionThreshold) + sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) + sqlLine.group("psfModelID", psfmodelID) + sqlLine.group("psfFWHM", psfFWHM) + sqlLine.group("psfWidMajor", psfFwhmMajor) + sqlLine.group("psfWidMinor", psfFwhmMinor) + sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) +# sqlLine.group("psfFwhm_mean", fwhm_maj) +# sqlLine.group("psfFwhm_max", fwhm_maj_uq) + sqlLine.group("astroScat", astroscat) + sqlLine.group("photoZero", zp) + sqlLine.group("ctype1", header['CTYPE1']) + sqlLine.group("ctype2", header['CTYPE2']) + sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) + sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) + sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) + sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) + sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) + sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) + sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) + sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) + sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) + sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) + sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) sql = sqlLine.make(") VALUES ( ", ")") @@ -536,6 +549,11 @@ continue + cmfTable = filter + "SkyChip_xsrc" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xsrc data for filter" , filter) + continue + # insert all the detections - sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + filter + "SkyChip_xsrc AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -612,5 +630,10 @@ return True - if self.scratchDb.getRowCount(filter+"SkyChip_xfit") <=0 : + cmfTable = filter + "SkyChip_xfit" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xfit data for filter" , filter) + return True + + if self.scratchDb.getRowCount(cmfTable) <=0 : self.logger.infoPair("no extended source information, trying to skip",filter) return True @@ -621,5 +644,5 @@ magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME']))) - sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xfit AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -775,8 +798,12 @@ return True - if self.scratchDb.getRowCount(filter+"SkyChip_xsrc") <=0 : - self.logger.infoPair("no extended source information, trying to skip",filter) + cmfTable = filter + "SkyChip_xsrc" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xsrc data for filter" , filter) return True + if self.scratchDb.getRowCount(cmfTable) <=0 : + self.logger.infoPair("no extended source information, trying to skip", filter) + return True header = self.headerSet[filter] @@ -790,5 +817,5 @@ self.logger.infoPair("Adding", "petrosians for extended sources") - sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xsrc AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -904,5 +931,5 @@ # properly -999 these to start with. the default should take - # care of this, but does not + # care of this, but does not? sqlLine = sqlUtility("UPDATE " + tablename + " SET") @@ -925,4 +952,9 @@ # set the flux values from the cmf file cmfTable = filter + "SkyChip_xrad" + + # skip this table if it was not loaded from the cmf + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xrad data for filter" , filter) + return True # we have variable numbers of these columns. find out which are in use @@ -1057,5 +1089,4 @@ sql += " AND " - self.logger.infoPair('sql',sql) self.scratchDb.execute(sql) @@ -1084,8 +1115,16 @@ self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex(filter + "SkyChip_psf", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xfit", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xrad", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xsrc", "IPP_IDET") + + cmfTable = filter + "SkyChip_psf" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xfit" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xrad" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xsrc" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") return True @@ -1303,4 +1342,6 @@ self.logger.infoPair("Importing ST tables with table match expression: ", tableRE) + self.tablesLoaded = [] + count = 0 for filter in self.filters: @@ -1319,5 +1360,4 @@ return False - # count = 0 for table in tables: @@ -1344,4 +1384,8 @@ self.logger.exception("Problem writing table '" + filter + table.name + "' to the database") + self.tablesLoaded.append(table.name) + + # XXX validate required tables (do we need _psf?, _xsrc? etc) + self.logger.infoPair("Done. Imported", "%d tables" % count) self.indexIppTables()