Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py (revision 37167) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py (revision 37168) @@ -178,6 +178,4 @@ if abort or not self.checkClientStatus(): abort = True elif numAttempts > 1 and not self.waitForPollTime(): break - - print "done with wait?" if self.onePassOnly: self.exitProgram("one pass completed") Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37167) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37168) @@ -4,5 +4,5 @@ import glob import sys - +import math import stilts @@ -55,4 +55,5 @@ self.stackType = "DEEP_STACK" # TODO + self.stackTypeID = self.getStackTypeID() self.filters = ['g', 'r', 'i', 'z', 'y'] @@ -158,7 +159,7 @@ self.dropTableVerbose("StackObjectThin") self.dropTableVerbose("StackObjectExtra") - + self.dropTableVerbose("StackApFlx") - + self.dropTableVerbose("StackModelFitThin") self.dropTableVerbose("StackModelFitDeV") @@ -204,4 +205,6 @@ self.logger.infoPair("Projectioncell", "%s" % self.projectioncell) + ### *********************** Utility Functions ************************************ + ''' Drop a table and report the drop @@ -212,27 +215,16 @@ ''' - Updates a table with stackMetaID - ''' - def updateStackMetaID(self, table): - - sql = "UPDATE " + table + " SET " - - for i in range(len(self.filters)): - filter = self.filters[i] - field = filter + "stackMetaID" - sql += field + " = '" + self.stackIDs[filter] + "'" - if (i < len(self.filters) - 1): - sql += ", " - - self.scratchDb.execute(sql) - - ''' - Updates a table with stackTypeID grabbed from FitModel init table - ''' - def updateStackTypeID(self, table): - - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'" - self.scratchDb.execute(sql) - + get stackTypeID corresponding to our stackType + ''' + def getStackTypeID(self): + + sql = "SELECT stackTypeID from StackType WHERE name = '" + self.stackType + "'" + rs = self.scratchDb.executeQuery(sql) + if not rs.next(): + self.logger.errorPair("cannot get stack type ID from database for", self.stackType) + raise + + ID = rs.getInt(1) + return ID def testprint(self): @@ -241,104 +233,5 @@ ''' - Updates aperture fluxes for StackApFkx table - ''' - def updateModelFitFlxs(self, prefix, psfCondition, tablename, filter): - #properly -999 these to start with: - f = filter - if tablename == 'StackModelFit': - sql = "UPDATE " + tablename + " AS a, "+f+"SkyChip_xrad AS b SET \ - "+f+prefix+"flxR1=-999, \ - "+f+prefix+"flxR1Err=-999, \ - "+f+prefix+"flxR1Std=-999, \ - "+f+prefix+"flxR1Fill=-999, \ - "+f+prefix+"flxR2=-999, \ - "+f+prefix+"flxR2Err=-999, \ - "+f+prefix+"flxR2Std=-999, \ - "+f+prefix+"flxR2Fill=-999, \ - "+f+prefix+"flxR3=-999, \ - "+f+prefix+"flxR3Err=-999, \ - "+f+prefix+"flxR3Std=-999, \ - "+f+prefix+"flxR3Fill=-999, \ - "+f+prefix+"flxR4=-999, \ - "+f+prefix+"flxR4Err=-999, \ - "+f+prefix+"flxR4Std=-999, \ - "+f+prefix+"flxR4Fill=-999, \ - "+f+prefix+"flxR5=-999, \ - "+f+prefix+"flxR5Err=-999, \ - "+f+prefix+"flxR5Std=-999, \ - "+f+prefix+"flxR5Fill=-999, \ - "+f+prefix+"flxR6=-999, \ - "+f+prefix+"flxR6Err=-999, \ - "+f+prefix+"flxR6Std=-999, \ - "+f+prefix+"flxR6Fill=-999, \ - "+f+prefix+"flxR7=-999, \ - "+f+prefix+"flxR7Err=-999, \ - "+f+prefix+"flxR7Std=-999, \ - "+f+prefix+"flxR7Fill=-999, \ - "+f+prefix+"flxR8=-999, \ - "+f+prefix+"flxR8Err=-999, \ - "+f+prefix+"flxR8Std=-999, \ - "+f+prefix+"flxR8Fill=-999, \ - "+f+prefix+"flxR9=-999, \ - "+f+prefix+"flxR9Err=-999, \ - "+f+prefix+"flxR9Std=-999, \ - "+f+prefix+"flxR9Fill=-999, \ - "+f+prefix+"flxR10=-999, \ - "+f+prefix+"flxR10Err=-999, \ - "+f+prefix+"flxR10Std=-999, \ - "+f+prefix+"flxR10Fill=-999 \ - WHERE a."+f+"ippDetectID=b.IPP_IDET " - #self.logger.infoPair("sql", sql) - self.scratchDb.execute(sql) - - - #we have variable numbers of these columns. find out which are in use - sql = "DESCRIBE "+f+"SkyChip_xrad 'aper_fill_%'" - rs = self.scratchDb.executeQuery(sql) - rs.first() - columns = [] - column = rs.getString(1) - columns.append(column) - while (rs.next()): - column = rs.getString(1) - columns.append(column) - rs.close() - - #use regex to leave just the numbers - - numbers = [] - min = 1 - max = 10 - if tablename == 'StackApFlx': - min = 3 - max = 5 - for column in columns: - number = re.sub("APER_FILL_","",column) - if (int(number) >= min and int(number) <=max): - numbers.append(number) - self.logger.infoPair("Aperture Numbers",number) - - #generate the sql to do the necessary ops on the columns - sql = "UPDATE " + tablename + " AS a, "+f+"SkyChip_xrad AS b SET " - cnt =0 - - # XXX : I need to worry about which header this is - exptime = self.safeDictionaryAccess(self.header, 'EXPTIME') - for number in numbers: - if (cnt>0): - sql=sql+" , " - sql=sql+f+prefix+"flxR"+number+"=b.APER_FLUX_"+number +" / " + str(exptime) + ", " - sql=sql+f+prefix+"flxR"+number+"Err=b.APER_FLUX_ERR_"+number +" / " + str(exptime) + ", " - sql=sql+f+prefix+"flxR"+number+"Std=b.APER_FLUX_STDEV_"+number +" / " + str(exptime) + ", " - sql=sql+f+prefix+"flxR"+number+"Fill=b.APER_FILL_"+number + " " - cnt=cnt+1 - - sql=sql+"WHERE a."+f+"ippDetectID=b.IPP_IDET AND "+psfCondition - self.logger.infoPair("sql",sql) - self.scratchDb.execute(sql) - - - ''' - Populates the StackMeta table, mainly from dictionary values found in IPP FITS header + Populates the StackMeta table, mainly from header values found in stack cmf FITS header ''' def populateStackMeta(self,filter): @@ -374,16 +267,16 @@ sqlLine = sqlUtility("INSERT INTO " + tablename + " (") + sqlLine.group("stackMetaID", str(stackID)) sqlLine.group("batchID", str(self.batchID)) sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("dataRelease", str(self.skychunk.dataRelease)) - sqlLine.group("stackMetaID", str(stackID)) sqlLine.group("filterID", str(filterID)) + sqlLine.group("stackTypeID", str(self.stackTypeID)) sqlLine.group("skyCellID", self.skycell) sqlLine.group("photoCalID", photoCalID) - sqlLine.group("photoZero", self.getKeyFloat(header, "%.5f", 'FPA.ZP')) sqlLine.group("expTime", self.getKeyFloat(header, "%.5f", 'EXPTIME')) sqlLine.group("psfModelID", psfmodel) sqlLine.group("psfFwhm_mean", fwhm_maj) sqlLine.group("psfFwhm_max", fwhm_maj_uq) + sqlLine.group("photoZero", self.getKeyFloat(header, "%.5f", 'FPA.ZP')) sqlLine.group("ctype1", header['CTYPE1']) sqlLine.group("ctype2", header['CTYPE2']) @@ -398,4 +291,5 @@ sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) + sqlLine.group("dataRelease", str(self.skychunk.dataRelease)) sql = sqlLine.make(") VALUES ( ", ")") @@ -404,8 +298,4 @@ self.scratchDb.execute(sql) - # set stackTypeID in table based on lookup table & self.stackType - # XXX include in the INSERT above? - self.updateStackTypeID(tablename) - self.tablesToExport.append(tablename) @@ -419,4 +309,5 @@ # XXX EAM 20140731 : if we are going to add indexes (objID?) to StackObjectThin, do it HERE + self.scratchDb.createIndex("StackObjectThin", "objID") self.logger.infoPair("inserting filter dependent items into", "StackObjectThin") @@ -491,4 +382,6 @@ self.logger.errorPair('failed sql: ', sql) raise + + self.tablesToExport.append("StackObjectThin") ''' @@ -576,4 +469,6 @@ return + self.tablesToExport.append("StackObjectExtra") + ''' Populates the StackModelFitThin table @@ -587,4 +482,5 @@ # self.scratchDb.createIndex("StackObjectExtra", "objID") + # currently, none of the fields defined by the table are generated ''' @@ -603,4 +499,6 @@ for filter in self.filters: self.populateStackModelFitFilter(model, ippModelType, filter) + + self.tablesToExport.append(table) ''' @@ -616,14 +514,15 @@ header = self.headerSet[filter] + print "exptime in header: " + header['EXPTIME'] + + magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME']))) exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") + print "magtime: " + magtime + ", exptime: " + exptime + sqlLine = sqlUtility("UPDATE " + table + " AS a, " + filter + "SkyChip_xfit AS b SET") - # these lines yield the flux, flux error values, but I do not think we want those in the database... - # sqlLine.group("a." + filter + model + "Flux", "POW(10, -0.4 * b.EXT_INST_MAG) / " + str(exptime)) - # sqlLine.group("a." + filter + model + "FluxErr", "ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(exptime) + " / 1.085736") - - sqlLine.group("a." + filter + model + "Mag", "b.EXT_INST_MAG") # need to modify by zero point and 2.5*log10(exptime) - sqlLine.group("a." + filter + model + "MagErr", "b.EXT_INST_MAG_SIG") # need to modify by zero point and 2.5*log10(exptime) + sqlLine.group("a." + filter + model + "Mag", "b.EXT_INST_MAG + " + magtime) # need to modify by zero point and 2.5*log10(exptime) + sqlLine.group("a." + filter + model + "MagErr", "b.EXT_INST_MAG_SIG") sqlLine.group("a." + filter + model + "Radius", "b.EXT_WIDTH_MAJ") @@ -655,73 +554,214 @@ self.scratchDb.execute(sql) - - # XXX old code for flux tables - # self.logger.infoPair("Adding fluxes", "un-convolved") - # self.updateModelFitFlxs("", "((b.table_index -1) % 3 = 0)","StackModelFit",f) - # self.logger.infoPair("Adding fluxes", "1st convolved") - # self.updateModelFitFlxs("c1", "((b.table_index -1) % 3 = 1)","StackModelFit",f) - # self.logger.infoPair("Adding fluxes", "2nd convolved") - # self.updateModelFitFlxs("c2", "((b.table_index -1) % 3 = 2)","StackModelFit",f) - - ''' - Populates the StackApFlx table, mainly from dictionary values found in IPP FITS header - ''' - # def populateStackApFlx(self,filter): - - ## self.logger.infoPair("Adding", "petrosians for extended sources") - ## sql = "UPDATE StackModelFit as a, "+f+"SkyChip_xsrc AS b SET \ - ## "+f+"petRadius=b.PETRO_RADIUS \ - ## ,"+f+"petRadiusErr=b.PETRO_RADIUS_ERR \ - ## ,"+f+"petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.header['EXPTIME']) + " \ - ## ,"+f+"petFluxErr=-999 \ - ## ,"+f+"petR50=b.PETRO_RADIUS_50 \ - ## ,"+f+"petR50Err=b.PETRO_RADIUS_50_ERR \ - ## ,"+f+"petR90=b.PETRO_RADIUS_90 \ - ## ,"+f+"petR90Err=b.PETRO_RADIUS_90_ERR \ - ## ,"+f+"petCf=b.PETRO_FILL \ - ## WHERE a."+f+"ippDetectID=b.IPP_IDET" - ## self.scratchDb.execute(sql) - ## - ## self.logger.infoPair("Adding", "petFluxErr for non nulls") - ## sql = "UPDATE StackModelFit as a, "+f+"SkyChip_xsrc AS b SET \ - ## "+f+"petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.header['EXPTIME']) + " / 1.085736 \ - ## WHERE a."+f+"ippDetectID=b.IPP_IDET \ - ## AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)" - ## # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc) - ## self.logger.infoPair("sql", sql) - ## self.scratchDb.execute(sql) - - - # self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) - # self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.skychunk.dataRelease)) - - # self.logger.infoPair("not adding"," PrimaryF to stackApFlx") - # sql = "UPDATE StackModelFit AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ - # a.primaryF = ((b.flags & 0x10000) >> 16 ) \ - # WHERE a.ippDetectID = b.ippDetectID \ - # AND b.imageID = " + str(imageID) + # insert detection information for each filter based on the DVO contents + sqlLine = sqlUtility("UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET") + sqlLine.group("a." + filter + model + "Mag", "a." + filter + model + "Mag + b.zp") + sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID") + + self.logger.infoPair('sql', sql) + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + ''' + Populates the StackModelFitPet table, mainly from dictionary values found in IPP FITS header + ''' + def populateStackPetrosian(self): + + table = "StackModelFitPet" + + self.logger.infoPair("Procesing table", table) + self.insertDvoIDs(table, "StackObjectThin") + + # if we are going to add a key, do it here so it is useful + # self.scratchDb.createIndex(table, "objID") + + for filter in self.filters: + self.populateStackPetrosianFilter(filter) + + self.tablesToExport.append(table) + + ''' + Populates the StackModelFitPet table, mainly from dictionary values found in IPP FITS header + ''' + def populateStackPetrosianFilter(self, filter): + + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no stack data for filter" , filter) + return True + + self.logger.infoPair("Adding", "petrosians for extended sources") + sqlLine = sqlUtility("UPDATE StackModelFitPet AS a, " + filter + "SkyChip_xsrc AS b SET") + + sqlLine.group("a." + filter + "petRadius", "b.PETRO_RADIUS") + sqlLine.group("a." + filter + "petRadiusErr", "b.PETRO_RADIUS_ERR") + sqlLine.group("a." + filter + "petFlux", "b.PETRO_MAG") # XXX EAM 20140732 : still need to decide about flux vs mag + sqlLine.group("a." + filter + "petFluxErr", "b.PETRO_MAG_ERR") # XXX EAM 20140732 + sqlLine.group("a." + filter + "petR50", "b.PETRO_RADIUS_50") + sqlLine.group("a." + filter + "petR50Err", "b.PETRO_RADIUS_50_ERR") + sqlLine.group("a." + filter + "petR90", "b.PETRO_RADIUS_90") + sqlLine.group("a." + filter + "petR90Err", "b.PETRO_RADIUS_90_ERR") + sqlLine.group("a." + filter + "petCf", "b.PETRO_FILL") + + sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET") + + print "SQL: " + sql + self.scratchDb.execute(sql) ''' Populates the StackApFlx table ''' - def populateStackApFlx(self): + def populateStackApFlxThin(self): + + tablename = "StackApFlx" + + self.logger.infoPair("Procesing table", tablename) + self.insertDvoIDs(tablename, "StackObjectThin") + + # if we are going to add a key, do it here so it is useful + # self.scratchDb.createIndex(tablename, "objID") for filter in self.filters: - if self.stackIDs[filter] <= 0: - continue - - # XXX is this a reasonable solution? - self.header = self.headerSet[filter] - - self.logger.infoPair("Adding fluxes", "un-convolved") - self.updateModelFitFlxs("", "((b.table_index -1) % 3 = 0)", "StackApFlx", filter) - self.logger.infoPair("Adding fluxes", "1st convolved") - self.updateModelFitFlxs("c1", "((b.table_index -1) % 3 = 1)","StackApFlx", filter) - self.logger.infoPair("Adding fluxes", "2nd convolved") - self.updateModelFitFlxs("c2", "((b.table_index -1) % 3 = 2)","StackApFlx", filter) - - self.deleteDetectionsNotInStackObject("StackApFlx",f) - - self.scratchDb.reportAndDeleteRowsWithNULLS("StackApFlx", "objID") + self.populateStackApFlxFilter(tablename, "", filter) + self.populateStackApFlxFilter(tablename, "C1", filter) + self.populateStackApFlxFilter(tablename, "C2", filter) + + self.tablesToExport.append(tablename) + + ''' + Populates the StackApFlx table + ''' + def populateStackApFlx(self, version): + + tablename = "StackModelFitFlx" + version + + self.logger.infoPair("Procesing table", tablename) + self.insertDvoIDs(tablename, "StackObjectThin") + + # if we are going to add a key, do it here so it is useful + # self.scratchDb.createIndex(tablename, "objID") + + for filter in self.filters: + self.populateStackApFlxFilter(tablename, version, filter) + + self.tablesToExport.append(tablename) + + ''' + Updates aperture fluxes for StackApFkx table + ''' + def populateStackApFlxFilter(self, tablename, version, filter): + + prefix = "none" + + # order of the 3 convolutions is fixed (raw, C1, C2) = (1, 2, 3) + if version == "": + psfCondition = "(b.table_index % 3 = 1)" + prefix = "" + if version == "C1": + psfCondition = "(b.table_index % 3 = 2)" + prefix = "c1" + if version == "C2": + psfCondition = "(b.table_index % 3 = 0)" + prefix = "c2" + + if prefix == "none": + self.logger.errorPair("invalid stack ap flux version", version) + raise + + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no stack data for filter" , filter) + return True + + minRadius = 1 + maxRadius = 10 + if tablename == 'StackApFlx': + minRadius = 3 + maxRadius = 5 + + # properly -999 these to start with: + # XXX the default should take care of this, but does not + + sqlLine = sqlUtility("UPDATE " + tablename + " SET") + for radius in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]: + if radius < minRadius: continue + if radius > maxRadius: continue + sqlLine.group(filter + prefix + "flxR" + str(radius), "-999") + sqlLine.group(filter + prefix + "flxR" + str(radius) + "Err", "-999") + sqlLine.group(filter + prefix + "flxR" + str(radius) + "Std", "-999") + sqlLine.group(filter + prefix + "flxR" + str(radius) + "Fill", "-999") + + sql = sqlLine.makeEquals("") + + print "SQL: " + sql + self.scratchDb.execute(sql) + + # set the flux values from the cmf file + cmfTable = filter + "SkyChip_xrad" + + # we have variable numbers of these columns. find out which are in use + sql = "DESCRIBE " + cmfTable + " 'aper_fill_%'" + rs = self.scratchDb.executeQuery(sql) + rs.first() + columns = [] + column = rs.getString(1) + columns.append(column) + while (rs.next()): + column = rs.getString(1) + columns.append(column) + rs.close() + + # use regex to select the aperture number (APER_FILL_N) + numbers = [] + for column in columns: + number = re.sub("APER_FILL_","",column) + if (int(number) >= minRadius and int(number) <= maxRadius): + numbers.append(number) + self.logger.infoPair("Aperture Numbers",number) + + # XXX : I need to worry about which header this is + header = self.headerSet[filter] + exptime = self.getKeyFloat(header, "%.5f", "EXPTIME") + + # generate the sql to do the necessary ops on the columns + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET ") + + for number in numbers: + sqlLine.group(filter + prefix + "flxR" + number, "b.APER_FLUX_" + number + " / " + exptime) + sqlLine.group(filter + prefix + "flxR" + number + "Err", "b.APER_FLUX_ERR_" + number + " / " + exptime) + sqlLine.group(filter + prefix + "flxR" + number + "Std", "b.APER_FLUX_STDEV_" + number + " / " + exptime) + sqlLine.group(filter + prefix + "flxR" + number + "Fill", "b.APER_FILL_" + number + " / " + exptime) + + sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET AND " + psfCondition) + self.logger.infoPair("sql",sql) + self.scratchDb.execute(sql) + + ''' + Populates the StackToImage table + ''' + def populateStackToImage(self): + + self.logger.infoPair("Procesing table", "StackToImage") + + for filter in self.filters: + + stackmetaID = self.stackIDs[filter] + if stackmetaID > 0: + + imageIDs = self.gpc1Db.getImageIDsForThisStackID(stackmetaID) + + for imageID in imageIDs: + + sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ + VALUES (\ + " + str(stackmetaID) + ", " + imageID + ")" + self.scratchDb.execute(sql) + + # now update StackMeta with correct number of inputs + sql = "UPDATE "+filter+"StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage where stackMetaID = "+str(stackmetaID)+")" + self.scratchDb.execute(sql) + + self.tablesToExport.append("StackToImage") ''' @@ -743,107 +783,4 @@ - ''' - Populates the StackToImage table - ''' - def populateStackToImage(self): - - self.logger.infoPair("Procesing table", "StackToImage") - - for filter in self.filters: - - stackmetaID = self.stackIDs[filter] - if stackmetaID > 0: - - imageIDs = self.gpc1Db.getImageIDsForThisStackID(stackmetaID) - - for imageID in imageIDs: - - sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ - VALUES (\ - " + str(stackmetaID) + ", " + imageID + ")" - self.scratchDb.execute(sql) - - # now update StackMeta with correct number of inputs - sql = "UPDATE "+filter+"StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage where stackMetaID = "+str(stackmetaID)+")" - self.scratchDb.execute(sql) - - ''' - Generates the randomStackIds - ''' - def populateRandomsAndSkyCellID(self): - - self.logger.infoPair("Generating Randoms","for StackObject") - sql = "update StackObject set randomStackObjID = FLOOR(RAND()*9223372036854775807) \ - ,skyCellID = " + str(self.skycell)+ " \ - ,surveyID = " + str(self.surveyID) - try: - self.scratchDb.execute(sql) - except: - self.logger.infoPair("failed sql", sql) - - ''' - copies randomStackIds from StackObject to tableName - ''' - def copyRandomsAndSkyCellIDFromStackObject(self, tableName): - - self.logger.infoPair("copying Randoms from StackObject to ", tableName) - sql = "update " + tableName + " join StackObject using (objID) \ - set " + tableName + ".randomStackObjID = StackObject.randomStackObjID \ - , " + tableName + ".skyCellID = StackObject.skyCellID \ - , " + tableName + ".surveyID = StackObject.surveyID" - self.logger.infoPair("sql", sql) - try: - self.scratchDb.execute(sql) - except: - self.logger.infoPair("failed sql", sql) - - - - ''' - I think we are no longer using this function: this have been - replaced with the code in 'selectDvoObjIDs' and 'selectDvoFilterIDs' - ''' - def populateStackIPPDetectIDFromDvo(self, tableName, all): - - #tableName = "StackObject" - self.logger.infoPair("Processing table", tableName) - - filters = ['g','r','i','z','y'] - - for filter in self.filters: - if self.stackIDs[filter] < 0: - self.logger.infoPair("skipping ingest of dvo for filter",f) - continue - - self.header = self.headerSet[filter] - stackmetaID = self.stackIDs[filter] - - imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) - self.logger.infoPair("obtained","imageID") - addsql = "" - if (all == 1): - addsql = "a." + filter + "ra = b.ra, \ - a." + filter + "dec = b.dec_, \ - a." + filter + "raErr = b.raErr, \ - a." + filter + "decErr = b.decErr, \ - a." + filter + "zp = b.zp, \ - a." + filter + "expTime = b.expTime," - - - sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \ - SET a." + filter + "ippDetectID = b.ippDetectID, \ - " + addsql + " \ - a." + filter + "stackDetectID = b.detectID, \ - a." + filter + "ippObjID = b.ippObjID, \ - a." + filter + "stackMetaID = " + str(stackmetaID) + " \ - WHERE a.objID = b.objID AND b.imageID = " + str(imageID) - - self.logger.infoPair('sql',sql) - self.scratchDb.execute(sql) - - # out of the loops - give dataRelase to all - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) - - def deleteNULLippDetectIDs(self, tableName): @@ -863,11 +800,11 @@ def alterPspsTables(self): - self.logger.debug("Altering PSPS tables") + self.logger.debug("Altering PSPS tables (currently does nothing in stackbatch)") # heather uncommented the unique line -- well that no worky #self.scratchDb.makeColumnUnique("StackObject", "objID") - self.scratchDb.createIndex("StackObject", "ippDetectID") - self.scratchDb.createIndex("StackObject", "stackDetectID") - self.scratchDb.createIndex("StackApFlx", "ippDetectID") - self.scratchDb.createIndex("StackModelFit", "ippDetectID") + #self.scratchDb.createIndex("StackObject", "ippDetectID") + #self.scratchDb.createIndex("StackObject", "stackDetectID") + #self.scratchDb.createIndex("StackApFlx", "ippDetectID") + #self.scratchDb.createIndex("StackModelFit", "ippDetectID") return True @@ -878,9 +815,14 @@ def indexIppTables(self): - self.logger.infoPair("Creating indexes on", "IPP tables") - self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") - self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") + for filter in self.filters: + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no files for filter" , filter) + continue + + self.logger.infoPair("Creating indexes on", "IPP tables") + self.scratchDb.createIndex(filter + "SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex(filter + "SkyChip_xfit", "IPP_IDET") + self.scratchDb.createIndex(filter + "SkyChip_xrad", "IPP_IDET") + self.scratchDb.createIndex(filter + "SkyChip_xsrc", "IPP_IDET") return True @@ -892,18 +834,15 @@ def selectDvoObjIDs(self): - print "starting selectDvoObjIDs" - - sql = "INSERT INTO StackObjectThin (objID, ippObjID, surveyID, skyCellID, dvoRegionID, dataRelease) " - sql += "SELECT DISTINCT objID, ippObjID, " - sql += str(self.surveyID) + ", " - sql += str(self.skycell) + ", " - sql += str(self.dvoRegionID) + ", " - sql += str(self.skychunk.dataRelease) - - sql += " FROM " + self.scratchDb.dvoDetectionTable - sql += " WHERE (" - - print "sql so far: " + sql - + sqlLine = sqlUtility("INSERT INTO StackObjectThin (") + + sqlLine.group("objID", "objID") + sqlLine.group("ippObjID", "ippObjID") + sqlLine.group("surveyID", "'" + str(self.surveyID) + "'") + sqlLine.group("skyCellID", "'" + str(self.skycell) + "'") + sqlLine.group("dvoRegionID", "'" + str(self.dvoRegionID) + "'") + sqlLine.group("dataRelease", "'" + str(self.skychunk.dataRelease) + "'") + sqlLine.group("randomStackObjID", "FLOOR(RAND()*9223372036854775807)") + + whereClause = " WHERE (" imageIDs = self.imageIDs.values() for i in range(len(imageIDs)): @@ -911,44 +850,16 @@ self.logger.infoPair("selecting imageID", imageID) - print "table: self.scratchDb.dvoDetectionTable" - print "imageID: " + str(imageID) - - sql += "(" + self.scratchDb.dvoDetectionTable + ".imageID = " + str(imageID) + ")" - print "sql: " + sql + whereClause += "(" + self.scratchDb.dvoDetectionTable + ".imageID = " + str(imageID) + ")" if (i < len(imageIDs) - 1): - sql += " or " - - print "done segment" - - sql += ")" - + whereClause += " or " + + whereClause += ")" + + sql = sqlLine.makeRaw(") SELECT DISTINCT ", "FROM " + self.scratchDb.dvoDetectionTable + whereClause) print "SQL: " + sql + self.scratchDb.execute(sql) - ## XXX ''' - ## XXX select objid (psps object ID) from dvo restricted to this set of imageIDs - ## XXX ''' - ## XXX def selectDvoFilterIDs(self, table): - ## XXX - ## XXX # XXX I am using imageID a lot: generate a filter-index list - ## XXX for filter in self.filters: - ## XXX self.header = self.headerSet[filter] - ## XXX imageIDgpc = self.header['IMAGEID'] - ## XXX imageIDdvo = self.scratchDb.getImageIDFromExternID(imageIDgpc) - ## XXX self.logger.infoPair("selecting imageID", imageIDdvo) - ## XXX - ## XXX sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable " AS b SET " - ## XXX - ## XXX sql += "a." + filter + "ippDetectID = b.ippDetectID, " # eg, gippDetectID (from dvo.measure.detID) - ## XXX sql += "a." + filter + "stackDetectID = b.detectID " # eg, gstackDetectID (from dvo.measure.extID) - ## XXX sql += "a." + filter + "stackMetaID = " + imageIDgpc # eg, gstackMetaID (from cmf.IMAGEID) - ## XXX - ## XXX sql += "WHERE a.objID = b.objID " - ## XXX sql += "AND b.imageID = " + imageIDdvo - ## XXX - ## XXX print "SQL: " + sql - ## XXX self.scratchDb.execute(sql) - ''' Insert objid (from dvo) into table from main table @@ -956,5 +867,5 @@ def insertDvoIDs(self, table, mainTable): - fields = "objID, ippObjID, surveyID, skyCellID, dvoRegionID, dataRelease, " + fields = "objID, ippObjID, surveyID, skyCellID, randomStackObjID, primaryDetection, dvoRegionID, dataRelease, " for i in range(len(self.filters)): @@ -1007,5 +918,4 @@ self.scratchDb.execute(sql) - ''' Does the processing, i.e. pulling stuff from IPP tables into PSPS tables @@ -1015,11 +925,5 @@ self.skipBatch = False - # get the IDs from the DVO tables - # XXX deprecated along with dvograbber - # if not self.getIDsFromDVO(): return False - - # do some basics first: - # populate stack meta per filter - # insert objid/stackdetectid per filter to each table + # each of the "populate*" methods below add their tables to the list: self.tablesToExport=[] @@ -1060,4 +964,7 @@ if self.stackType != "NIGHTLY_STACK": + self.logger.infoPair("populating ","StackModelFitThin") + self.populateStackModelFitThin() + self.logger.infoPair("populating ","StackModelFit") self.populateStackModelFit("Exp", "PS_MODEL_EXP") @@ -1065,20 +972,20 @@ self.populateStackModelFit("Ser", "PS_MODEL_SERSIC") - #self.populateStackModelPet("Ser", "PS_MODEL_SERSIC") - - #self.logger.infoPair("populating ","StackApFlx") - #self.populateStackApFlx() - + self.populateStackPetrosian() + + self.populateStackModelFit("Ser", "PS_MODEL_SERSIC") + + self.logger.infoPair("populating ","StackApFlx Tables") + self.populateStackApFlxThin() + + self.populateStackApFlx("") + self.populateStackApFlx("C1") + self.populateStackApFlx("C2") + self.logger.infoPair("populating","StackToImage") self.populateStackToImage() # self.setMinMaxObjID(["StackObject"]) - # self.updateStackTypeID("StackModelFit") - - self.tablesToExport.append("StackToImage") - self.tablesToExport.append("StackObject") - self.tablesToExport.append("StackModelFit") - self.tablesToExport.append("StackApFlx") - self.tablesToExport.append("SkinnyObject") + self.logger.infoPair("finishing","populatePspsTables"); return True @@ -1134,5 +1041,5 @@ # IPP FITS files are littered with infinities, so remove these - self.logger.debug("Removing Infinity values from all columns") + self.logger.info("Removing Infinity values from all columns") table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') table = stilts.tpipe(table, cmd='replaceval -Infinity null *') Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/testCode.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/testCode.py (revision 37167) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/testCode.py (revision 37168) @@ -6,8 +6,8 @@ import sys # import os -# import math +import math # import logging.config -from sqlUtility import sqlUtility +# from sqlUtility import sqlUtility # testCode class @@ -19,4 +19,8 @@ def run(self): + + myvalue = "10.0" + print "test line" + print "log of 5.0: " + str(math.log10(float(myvalue))) names = {}