Index: trunk/ippToPsps/jython/batch.py =================================================================== --- trunk/ippToPsps/jython/batch.py (revision 31108) +++ trunk/ippToPsps/jython/batch.py (revision 31109) @@ -16,6 +16,7 @@ Constructor ''' - def __init__(self, batchType, inputFitsPath, outputFitsPath, dbHost, dbName, dbUser, dbPass): + def __init__(self, batchType, inputFitsPath, outputFitsPath, dbHost, dbName, dbUser, dbPass, survey): + # set up class variables self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" self.inputFitsPath = inputFitsPath @@ -25,12 +26,23 @@ self.dbUser = dbUser self.dbPass = dbPass + self.survey = survey + + # set up JDBC connection self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass self.con = DriverManager.getConnection(self.url) + self.stmt = self.con.createStatement() + # get survey ID from init table + sql = "SELECT surveyID from Survey WHERE name = '" + survey + "'" + rs = self.stmt.executeQuery(sql) + rs.first() + self.surveyID = rs.getInt(1) + # store today's date now = datetime.datetime.now(); self.dateStr = now.strftime("%Y-%m-%d") - self.parseFitsHeader() + if self.inputFitsPath != "": + self.parseFitsHeader() ''' @@ -38,7 +50,24 @@ ''' def __del__(self): + print "Batch destructor" + self.stmt.close() self.con.close() + ''' + Updates a table with surveyID + ''' + def updateSurveyID(self, table): + + sql = "UPDATE " + table + " SET surveyID=%d" % self.surveyID + self.stmt.execute(sql) + + ''' + Updates a table with filterID grabbed from Filter init table + ''' + def updateFilterID(self, table): + + sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + self.filter + "'" + self.stmt.execute(sql) ''' @@ -87,4 +116,34 @@ table.write(self.url + '#' + table.name) + self.indexPspsTables() + + ''' + Subclass should implement this to index IPP tables + ''' + def indexIppTables(self): + print "indexIppTables not implemented" + + + ''' + Adds an index to the supplied table and column + ''' + def createIndex(self, table, column): + + print "Creating index on column '"+column+"' for table '"+table+"'" + + sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")" + try: + self.stmt.execute(sql) + except: + print "Index already in place on '" + column + "' for table '" + table + "'" + + + ''' + Subclass should implement this to index PSPS tables + ''' + def indexPspsTables(self): + print "indexPspsTables not implemented" + + ''' Imports IPP tables from FITS file @@ -101,4 +160,6 @@ except: print "ERROR table " + table.name + " is busted" + + self.indexIppTables() ''' Index: trunk/ippToPsps/jython/initbatch.py =================================================================== --- trunk/ippToPsps/jython/initbatch.py (revision 31108) +++ trunk/ippToPsps/jython/initbatch.py (revision 31109) @@ -6,4 +6,7 @@ from batch import Batch +''' +Class encapsulating the init batch type (IN) +''' class InitBatch(Batch): @@ -14,5 +17,4 @@ super(InitBatch, self).__init__( "init", - #"/data/ipp053.0/eugene/md04.20110320/staticsky/MD04.V2/skycell.087/MD04.V2.skycell.087.stk.280.000.cmf", "", "init.fits", Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 31108) +++ trunk/ippToPsps/jython/stackbatch.py (revision 31109) @@ -16,9 +16,28 @@ #"/data/ipp053.0/eugene/md04.20110320/staticsky/MD04.V2/skycell.087/MD04.V2.skycell.087.stk.280.000.cmf", "demo.fits", - "multi.fits", + "stack.fits", "ippdb01", "ippToPsps", "ipp", - "ipp") + "ipp", + "MD04") # TODO + + + + ''' + Updates a table with stackMetaID + ''' + def updateStackMetaID(self, table): + + sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] + self.stmt.execute(sql) + + ''' + Updates a table with stackTypeID grabbed from FitModel init table + ''' + def updateStackTypeID(self, table): + + sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" + self.stmt.execute(sql) @@ -26,5 +45,5 @@ Updates aperture fluxes for StackApFkx table ''' - def updateApFlxs(self, stmt, prefix, psfCondition): + def updateApFlxs(self, prefix, psfCondition): sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \ @@ -71,58 +90,68 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition - stmt.execute(sql) + self.stmt.execute(sql) ''' Updates parameters for a particular model in the StackModelFit table ''' - def updateModelFit(self, stmt, model, ippModelType): + def updateModelFit(self, model, ippModelType): + + if model == "ser": + modelLong = "sersic" + else: + modelLong = model; sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ - "+model+"Covar11=b.EXT_COVAR_00_00, \ - "+model+"Covar12=b.EXT_COVAR_00_01, \ - "+model+"Covar13=b.EXT_COVAR_00_02, \ - "+model+"Covar14=b.EXT_COVAR_00_03, \ - "+model+"Covar15=b.EXT_COVAR_00_04, \ - "+model+"Covar16=b.EXT_COVAR_00_05, \ - "+model+"Covar17=b.EXT_COVAR_00_06, \ - "+model+"Covar22=b.EXT_COVAR_01_01, \ - "+model+"Covar23=b.EXT_COVAR_01_02, \ - "+model+"Covar24=b.EXT_COVAR_01_03, \ - "+model+"Covar25=b.EXT_COVAR_01_04, \ - "+model+"Covar26=b.EXT_COVAR_01_05, \ - "+model+"Covar27=b.EXT_COVAR_01_06, \ - "+model+"Covar33=b.EXT_COVAR_02_02, \ - "+model+"Covar34=b.EXT_COVAR_02_03, \ - "+model+"Covar35=b.EXT_COVAR_02_04, \ - "+model+"Covar36=b.EXT_COVAR_02_05, \ - "+model+"Covar37=b.EXT_COVAR_02_06, \ - "+model+"Covar44=b.EXT_COVAR_03_03, \ - "+model+"Covar45=b.EXT_COVAR_03_04, \ - "+model+"Covar46=b.EXT_COVAR_03_05, \ - "+model+"Covar47=b.EXT_COVAR_03_06, \ - "+model+"Covar55=b.EXT_COVAR_04_04, \ - "+model+"Covar56=b.EXT_COVAR_04_05, \ - "+model+"Covar57=b.EXT_COVAR_04_06, \ - "+model+"Covar66=b.EXT_COVAR_05_05, \ - "+model+"Covar67=b.EXT_COVAR_05_06, \ - "+model+"Covar77=b.EXT_COVAR_06_06 \ + "+model+"Radius=b.EXT_WIDTH_MAJ, \ + "+model+"Mag=b.EXT_INST_MAG, \ + "+model+"MagErr=b.EXT_INST_MAG_SIG, \ + "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ + "+model+"Phi=b.EXT_THETA, \ + "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ + "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ + "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ + "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ + "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ + "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ + "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ + "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ + "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ + "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ + "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ + "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ + "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ + "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ + "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ + "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ + "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ + "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ + "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ + "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ + "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ + "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ + "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ + "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ + "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ + "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ + "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ + "+modelLong+"Covar77=b.EXT_COVAR_06_06 \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - stmt.execute(sql) + self.stmt.execute(sql) # sersic fit has an extra parameter if ippModelType == "PS_MODEL_SERSIC": sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ - "+model+"Covar18=b.EXT_COVAR_00_07, \ - "+model+"Covar28=b.EXT_COVAR_01_07, \ - "+model+"Covar38=b.EXT_COVAR_02_07, \ - "+model+"Covar48=b.EXT_COVAR_03_07, \ - "+model+"Covar58=b.EXT_COVAR_04_07, \ - "+model+"Covar68=b.EXT_COVAR_05_07, \ - "+model+"Covar78=b.EXT_COVAR_06_07, \ - "+model+"Covar88=b.EXT_COVAR_07_07 \ + "+modelLong+"Covar18=b.EXT_COVAR_00_07, \ + "+modelLong+"Covar28=b.EXT_COVAR_01_07, \ + "+modelLong+"Covar38=b.EXT_COVAR_02_07, \ + "+modelLong+"Covar48=b.EXT_COVAR_03_07, \ + "+modelLong+"Covar58=b.EXT_COVAR_04_07, \ + "+modelLong+"Covar68=b.EXT_COVAR_05_07, \ + "+modelLong+"Covar78=b.EXT_COVAR_06_07, \ + "+modelLong+"Covar88=b.EXT_COVAR_07_07 \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - stmt.execute(sql) + self.stmt.execute(sql) @@ -130,5 +159,5 @@ Populates the StackMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateStackMeta(self, stmt): + def populateStackMeta(self): print "Procesing StackMeta table"; @@ -169,48 +198,72 @@ )" print sql - stmt.execute(sql) + self.stmt.execute(sql) + + self.updateSurveyID("StackMeta") + self.updateFilterID("StackMeta") + self.updateStackTypeID("StackMeta") ''' Populates the StackDetection table ''' - def populateStackDetection(self, stmt): + def populateStackDetection(self): print "Procesing StackDetection table"; # insert all the detections - sql = "INSERT INTO StackDetection \ - (ippDetectID, xPos, yPos, xPosErr, yPosErr, instFlux, instFluxErr, sky, skyErr, sgSep, psfWidMajor, psfWidMinor, psfTheta, psfCf, nFrames) \ + sql = "INSERT INTO StackDetection (\ + ippDetectID \ + ,xPos \ + ,yPos \ + ,xPosErr \ + ,yPosErr \ + ,instFlux \ + ,instFluxErr \ + ,peakFlux \ + ,sky \ + ,skyErr \ + ,sgSep \ + ,psfWidMajor \ + ,psfWidMinor \ + ,psfTheta \ + ,psfCf \ + ,nFrames \ + ) \ SELECT \ - IPP_IDET, X_PSF, Y_PSF, X_PSF_SIG, Y_PSF_SIG, PSF_INST_FLUX, PSF_INST_FLUX_SIG, SKY, SKY_SIGMA, EXT_NSIGMA, PSF_MAJOR, PSF_MINOR, PSF_THETA, PSF_QF, N_FRAMES \ + IPP_IDET \ + ,X_PSF \ + ,Y_PSF \ + ,X_PSF_SIG \ + ,Y_PSF_SIG \ + ,PSF_INST_FLUX \ + ,PSF_INST_FLUX_SIG \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ + ,SKY \ + ,SKY_SIGMA \ + ,EXT_NSIGMA \ + ,PSF_MAJOR \ + ,PSF_MINOR \ + ,PSF_THETA \ + ,PSF_QF \ + ,N_FRAMES \ FROM SkyChip_psf" - stmt.execute(sql) + self.stmt.execute(sql) + + if self.header['STK_TYPE'] != "NIGHTLY_STACK": + sql = "UPDATE StackDetection SET obsTime = " + self.header['MJD-OBS'] + self.stmt.execute(sql) sql = "UPDATE StackDetection SET skycellID = " + self.skycell + ", assocDate = '"+self.dateStr+"'" - stmt.execute(sql) - - ''' - Populates the StackModelFit table - ''' - def populateStackModelFit(self, stmt): - print "Procesing StackModelFit table"; - - # insert all the detections - sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" - stmt.execute(sql) - - # populate model parameters - print " Adding deVaucouleurs fit" - self.updateModelFit(stmt, "deV", "PS_MODEL_DEV") - print " Adding exponential fit" - self.updateModelFit(stmt, "exp", "PS_MODEL_EXP") - print " Adding sersic fit" - self.updateModelFit(stmt, "sersic", "PS_MODEL_SERSIC") - + self.stmt.execute(sql) + self.updateSurveyID("StackDetection") + self.updateFilterID("StackDetection") + self.updateStackMetaID("StackDetection") + self.updateStackTypeID("StackDetection") ''' Populates the StackApFlx table ''' - def populateStackApFlx(self, stmt): + def populateStackApFlx(self): print "Procesing StackApFlx table"; @@ -220,15 +273,28 @@ DISTINCT IPP_IDET \ FROM SkyChip_xrad" - stmt.execute(sql) + self.stmt.execute(sql) print " Adding 1st convolved fluxes" - #self.updateApFlxs(stmt, "c1", "< 7.0") + self.updateApFlxs("c1", "< 7.0") print " Adding 2nd convolved fluxes" -# self.updateApFlxs(stmt, "c2", "> 7.0") - - - # now add in the extended source stuff - sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET petRadius=b.PETRO_RADIUS WHERE a.ippDetectID=b.IPP_IDET" - stmt.execute(sql) + self.updateApFlxs("c2", "> 7.0") + + print " Adding petrosian stuff for extended sources" + sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \ + petRadius=b.PETRO_RADIUS \ + ,petRadiusErr=b.PETRO_RADIUS_ERR \ + ,petMag=b.PETRO_MAG \ + ,petMagErr=b.PETRO_MAG_ERR \ + ,petR50=b.PETRO_RADIUS_50 \ + ,petR50Err=b.PETRO_RADIUS_50_ERR \ + ,petR90=b.PETRO_RADIUS_90 \ + ,petR90Err=b.PETRO_RADIUS_90_ERR \ + WHERE a.ippDetectID=b.IPP_IDET" + self.stmt.execute(sql) + + self.updateSurveyID("StackApFlx") + self.updateFilterID("StackApFlx") + self.updateStackMetaID("StackApFlx") + self.updateStackTypeID("StackApFlx") #rs = stmt.executeQuery(sql) @@ -240,4 +306,49 @@ #rs.close() + + ''' + Populates the StackModelFit table + ''' + def populateStackModelFit(self): + print "Procesing StackModelFit table"; + + # insert all the detections + sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" + self.stmt.execute(sql) + + # populate model parameters + print " Adding deVaucouleurs fit" + self.updateModelFit("deV", "PS_MODEL_DEV") + print " Adding exponential fit" + self.updateModelFit("exp", "PS_MODEL_EXP") + print " Adding sersic fit" + self.updateModelFit("ser", "PS_MODEL_SERSIC") + + self.updateSurveyID("StackModelFit") + self.updateFilterID("StackModelFit") + self.updateStackMetaID("StackModelFit") + self.updateStackTypeID("StackModelFit") + + + ''' + Applies indexes to the PSPS tables + ''' + def indexPspsTables(self): + + print "Creating indexes on PSPS tables" + self.createIndex("StackDetection", "ippDetectID") + self.createIndex("StackApFlx", "ippDetectID") + self.createIndex("StackModelFit", "ippDetectID") + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + print "Creating indexes on IPP tables" + self.createIndex("SkyChip_psf", "IPP_IDET") + self.createIndex("SkyChip_xfit", "IPP_IDET") + self.createIndex("SkyChip_xrad", "IPP_IDET") + self.createIndex("SkyChip_xsrc", "IPP_IDET") ''' @@ -250,10 +361,12 @@ self.skycell = self.skycell[8:] - stmt = self.con.createStatement() - self.populateStackMeta(stmt) - self.populateStackDetection(stmt) - self.populateStackModelFit(stmt) - self.populateStackApFlx(stmt) - stmt.close() + # get filterID using init table + self.filter = self.header['FPA.FILTER'] + self.filter = self.filter[0:1] + + self.populateStackMeta() + self.populateStackDetection() + self.populateStackModelFit() + self.populateStackApFlx() @@ -263,3 +376,2 @@ stackBatch.process() stackBatch.exportPspsTablesToFits() -stackBatch