Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 34879) +++ trunk/ippToPsps/jython/stackbatch.py (revision 35097) @@ -2,4 +2,5 @@ import os.path +import glob import sys @@ -32,4 +33,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -42,4 +44,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -48,6 +51,9 @@ batchID, "ST", - gpc1Db.getStackStageCmf(config.dvoLabel, stackID), + gpc1Db.getStackStageCmf(skychunk.dvoLabel, stackID), useFullTables) + + # self.printline = 0 + # # self.testprint() self.stackType = "DEEP_STACK" # TODO @@ -63,4 +69,6 @@ self.filterID = self.scratchDb.getFilterID(self.filter) self.skycell = meta[1]; + + # self.testprint() # skycell is, eg "skycell.1133.081" @@ -76,4 +84,6 @@ self.skycell = self.skycell[8:12] self.projectioncell = self.skycell + + # self.testprint() # proposed new values. Need to coordinate with the SkyCell table @@ -96,4 +106,6 @@ self.scratchDb.dropTable("StackDetectionCalib") + # self.testprint() + # delete IPP tables self.scratchDb.dropTable("SkyChip_psf") @@ -115,4 +127,6 @@ if not self.useFullTables: self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID']) + + # self.testprint() # dump stuff to log @@ -139,4 +153,8 @@ self.scratchDb.execute(sql) + + def testprint(self): + print "here ", self.printline + self.printline += 1 ''' @@ -348,4 +366,14 @@ self.logger.infoPair("Procesing table", "StackMeta") + self.fwhm_maj = self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + self.fwhm_maj_uq = self.safeDictionaryAccess(self.header, 'FW_MJ_UQ') + self.psfmodel = self.safeDictionaryAccess(self.header, 'PSFMODEL') + if (self.fwhm_maj == "NULL"): self.fwhm_maj = -999 + if (self.fwhm_maj_uq == "NULL"): self.fwhm_maj_uq = -999 + + # print "fwhm_maj = ", self.fwhm_maj + # print "fwhm_maj_uq = ", self.fwhm_maj_uq + # print "psfmodel = ", self.psfmodel + sql = "INSERT INTO StackMeta (\ stackMetaID \ @@ -375,7 +403,7 @@ ," + self.header['FPA.ZP'] + " \ ," + self.expTime + " \ - ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ - ,'" + self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + "' \ - ,'" + self.safeDictionaryAccess(self.header, 'FW_MJ_UQ') + "' \ + ,'" + self.psfmodel + "' \ + ," + str(self.fwhm_maj) + " \ + ," + str(self.fwhm_maj_uq) + " \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -395,5 +423,5 @@ self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackMeta", self.filter) - self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.skychunk.dataRelease)) self.updateStackTypeID("StackMeta") @@ -486,6 +514,10 @@ ," + self.historyModNum + " \ FROM SkyChip_psf" - self.scratchDb.execute(sql) - + + # print "sql: ", sql + # response = raw_input("ready to insert stack det ") + + self.scratchDb.execute(sql) + #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid #we drop those... @@ -498,5 +530,5 @@ self.scratchDb.updateFilterID("StackDetection", self.filter) - self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") @@ -509,8 +541,10 @@ self.updateDvoIDsAndFlags("StackDetection") + # response = raw_input("updated dvo ") sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" self.scratchDb.execute(sql) + # response = raw_input("add primary key? ") if self.stackType == "DEEP_STACK": @@ -524,4 +558,5 @@ self.scratchDb.execute(sql) + # response = raw_input("add psf flux ") #leave null instflux in @@ -529,9 +564,11 @@ self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") + # response = raw_input("delete nulls ") sql="DELETE FROM StackDetection where objID = 0" self.scratchDb.execute(sql) - self.logger.infoPair("Delleting", "entries with StackDetection.objID = 0") + self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") + # response = raw_input("deleted objID is 0 ") @@ -577,5 +614,5 @@ self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackApFlx", self.filter) - self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") @@ -611,5 +648,5 @@ self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackModelFit", self.filter) - self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") @@ -676,10 +713,12 @@ self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) - self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.skychunk.dataRelease)) ''' Populates the StackDetectionCalib table - ''' - def populateStackDetectionCalib(self): + XXX this can probably get a big speed increase by using 'SELECT () INTO OUTFILE '/tmp/name' and then + calling the load data infile '/tmp/name' into table; + ''' + def populateStackDetectionCalibInsertUpdate(self): self.logger.infoPair("Processing table", "StackDetectionCalib") tableName = "StackDetectionCalib" @@ -702,4 +741,8 @@ FROM StackDetection" self.scratchDb.execute(sql) + + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) + self.logger.infoPair("obtained","imageID") + # insert calibration information from dvoDetections into the Table sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \ @@ -712,10 +755,64 @@ a.expTime = b.expTime, \ a.airMass = b.airMass \ - WHERE a.stackDetectID = b.detectID" - self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) - - - + WHERE a.stackDetectID = b.detectID \ + AND b.imageID = " + str(imageID) + self.scratchDb.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) + + def populateStackDetectionCalib(self): + + tableName = "StackDetectionCalib" + self.logger.infoPair("Processing table", tableName) + + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) + self.logger.infoPair("obtained","imageID") + + # check for & create output directory first + datadumpDir = "/tmp/datadump" + try: + statinfo = os.stat(datadumpDir) + # check on the stat results? + except: + print "making the data dump directory ", datadumpDir + os.mkdir(datadumpDir) + os.chmod(datadumpDir, 0777) + statinfo = os.stat(datadumpDir) + + dumpFile = datadumpDir + "/genetest.xx.dat" + files = glob.glob(dumpFile) + if len(files) > 0: + os.unlink(dumpFile) + + # insert all detections into table + sql = "SELECT \ + a.objID, \ + a.stackDetectID, \ + a.ippObjID, \ + a.ippDetectID, \ + a.filterID, \ + a.surveyID, \ + b.ra, \ + b.dec_, \ + b.raErr, \ + b.decErr, \ + b.zp, \ + b.zpErr, \ + b.expTime, \ + b.airMass, \ + " + str(self.skychunk.dataRelease) + " \ + FROM \ + StackDetection as a \ + JOIN " + self.scratchDb.dvoDetectionTable + " as b \ + ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \ + " INTO OUTFILE '" + dumpFile + "'" + print "sql: ", sql + self.scratchDb.execute(sql) + + sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName + print "sql: ", sql + self.scratchDb.execute(sql) + + ## XXX write this with the select/insert + # self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -737,5 +834,5 @@ self.scratchDb.updateFilterID("ObjectCalColor", self.filter) - self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.skychunk.dataRelease)) @@ -802,4 +899,7 @@ WHERE a.ippDetectID = b.ippDetectID \ AND b.imageID = " + str(imageID) + + # print "update dvo sql: ", sql + # response = raw_input("update dvo...") self.scratchDb.execute(sql)