Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 38826) +++ trunk/ippToPsps/jython/stackbatch.py (revision 38837) @@ -160,4 +160,5 @@ self.dropTableVerbose(tableName) + self.dropTableVerbose("StackMeta") self.dropTableVerbose("StackObjectThin") self.dropTableVerbose("StackObjectAttributes") @@ -176,4 +177,5 @@ self.dropTableVerbose("StackToImage") + self.dropTableVerbose("StackDetEffMeta") # delete IPP tables @@ -217,19 +219,8 @@ header = self.headerSet[filter] - tablename = filter + "StackMeta" + stackID = self.stackIDs[filter] filterName = filter + ".00000" - self.logger.infoPair("Populating table", tablename) - - fwhm_maj = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') - fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ') - psfmodel_name = self.getKeyValue(header, 'PSFMODEL') - psfmodelID = self.scratchDb.getFitModelID(psfmodel_name) - - ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') - ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') - astroscat = sqrt(ast_cdx^2 + ast_cdy^2) - # make a table filterID = self.scratchDb.getFilterID(filter) @@ -238,52 +229,74 @@ photoCalID = str(self.scratchDb.getPhotoCalID(stackID)) + tablename = filter + "StackMeta" + + sql = "CREATE TABLE " + tablename + " LIKE StackMeta" + self.scratchDb.execute(sql) + + self.logger.infoPair("Populating table", tablename) + + # fwhm_maj = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + # fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ') + + # XXX hard-wired platescale : 0.25 + pltscale = 0.25 + pltscale2 = pltscale * pltscale + + psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') + psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN') + psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor) + + psfmodel_name = self.getKeyValue(header, 'PSFMODEL') + psfmodelID = self.scratchDb.getFitModelID(psfmodel_name) + + ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX') + ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY') + astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2) + # Convert detectionThreshold to appropriate magnitudes detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF') expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME') - zp = self.getKeyFloat(header, "%.8f", 'FPA.ZP') - # CZW check? + zp = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') + + # XXX zp correction should come from DVO detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime) - - # mysql is sensitive to values which are ambiugously float. eg - # a warning is raised if we try to insert '25.' into a float field. - # use getKeyFloat(hdr, format, key) to avoid this problem - - sql = "CREATE TABLE " + tablename + " LIKE StackMeta" - try: self.scratchDb.execute(sql) - except: pass - + # insert stack metadata into table sqlLine = sqlUtility("INSERT INTO " + tablename + " (") - sqlLine.group("stackImageID", str(stackID)) - sqlLine.group("batchID", str(self.batchID)) - sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("filterID", str(filterID)) - sqlLine.group("stackTypeID", str(self.stackTypeID)) - sqlLine.group("tessID", str(self.tessID)) - sqlLine.group("projectionID", str(self.projectionID)) - sqlLine.group("skyCellID", str(self.skycellID)) - sqlLine.group("photoCalID", photoCalID) - sqlLine.group("analysisVer", str(self.analysisVer)) - sqlLine.group("detectionThreshold", detectionThreshold) - sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) - sqlLine.group("psfModelID", psfmodelID) - sqlLine.group("psfFwhm_mean", fwhm_maj) - sqlLine.group("psfFwhm_max", fwhm_maj_uq) - sqlLine.group("astroScat", astroscat) - sqlLine.group("photoZero", self.getKeyFloat(header, "%.5f", 'FPA.ZP')) - sqlLine.group("ctype1", header['CTYPE1']) - sqlLine.group("ctype2", header['CTYPE2']) - sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) - sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) - sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) - sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) - sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) - sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) - sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) - sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) - sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) - sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) - sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + sqlLine.group("stackImageID", str(stackID)) + sqlLine.group("batchID", str(self.batchID)) + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("filterID", str(filterID)) + sqlLine.group("stackTypeID", str(self.stackTypeID)) + sqlLine.group("tessID", str(self.tessID)) + sqlLine.group("projectionID", str(self.projectionID)) + sqlLine.group("skyCellID", str(self.skycellID)) + sqlLine.group("photoCalID", photoCalID) + sqlLine.group("analysisVer", str(self.analysisVer)) + sqlLine.group("detectionThreshold", detectionThreshold) + sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) + sqlLine.group("psfModelID", psfmodelID) + sqlLine.group("psfFWHM", psfFWHM) + sqlLine.group("psfWidMajor", psfFwhmMajor) + sqlLine.group("psfWidMinor", psfFwhmMinor) + sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) +# sqlLine.group("psfFwhm_mean", fwhm_maj) +# sqlLine.group("psfFwhm_max", fwhm_maj_uq) + sqlLine.group("astroScat", astroscat) + sqlLine.group("photoZero", zp) + sqlLine.group("ctype1", header['CTYPE1']) + sqlLine.group("ctype2", header['CTYPE2']) + sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) + sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) + sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) + sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) + sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) + sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) + sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) + sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) + sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) + sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) + sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) sql = sqlLine.make(") VALUES ( ", ")") @@ -536,6 +549,11 @@ continue + cmfTable = filter + "SkyChip_xsrc" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xsrc data for filter" , filter) + continue + # insert all the detections - sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + filter + "SkyChip_xsrc AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -612,5 +630,10 @@ return True - if self.scratchDb.getRowCount(filter+"SkyChip_xfit") <=0 : + cmfTable = filter + "SkyChip_xfit" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xfit data for filter" , filter) + return True + + if self.scratchDb.getRowCount(cmfTable) <=0 : self.logger.infoPair("no extended source information, trying to skip",filter) return True @@ -621,5 +644,5 @@ magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME']))) - sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xfit AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -775,8 +798,12 @@ return True - if self.scratchDb.getRowCount(filter+"SkyChip_xsrc") <=0 : - self.logger.infoPair("no extended source information, trying to skip",filter) + cmfTable = filter + "SkyChip_xsrc" + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xsrc data for filter" , filter) return True + if self.scratchDb.getRowCount(cmfTable) <=0 : + self.logger.infoPair("no extended source information, trying to skip", filter) + return True header = self.headerSet[filter] @@ -790,5 +817,5 @@ self.logger.infoPair("Adding", "petrosians for extended sources") - sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xsrc AS b SET") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET") sqlLine.group("a."+filter+"haveData", "'1'") @@ -904,5 +931,5 @@ # properly -999 these to start with. the default should take - # care of this, but does not + # care of this, but does not? sqlLine = sqlUtility("UPDATE " + tablename + " SET") @@ -925,4 +952,9 @@ # set the flux values from the cmf file cmfTable = filter + "SkyChip_xrad" + + # skip this table if it was not loaded from the cmf + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no xrad data for filter" , filter) + return True # we have variable numbers of these columns. find out which are in use @@ -1057,5 +1089,4 @@ sql += " AND " - self.logger.infoPair('sql',sql) self.scratchDb.execute(sql) @@ -1084,8 +1115,16 @@ self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex(filter + "SkyChip_psf", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xfit", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xrad", "IPP_IDET") - self.scratchDb.createIndex(filter + "SkyChip_xsrc", "IPP_IDET") + + cmfTable = filter + "SkyChip_psf" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xfit" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xrad" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") + + cmfTable = filter + "SkyChip_xsrc" + if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET") return True @@ -1303,4 +1342,6 @@ self.logger.infoPair("Importing ST tables with table match expression: ", tableRE) + self.tablesLoaded = [] + count = 0 for filter in self.filters: @@ -1319,5 +1360,4 @@ return False - # count = 0 for table in tables: @@ -1344,4 +1384,8 @@ self.logger.exception("Problem writing table '" + filter + table.name + "' to the database") + self.tablesLoaded.append(table.name) + + # XXX validate required tables (do we need _psf?, _xsrc? etc) + self.logger.infoPair("Done. Imported", "%d tables" % count) self.indexIppTables()