Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 35994) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 36706) @@ -257,7 +257,13 @@ ,psfWidMinor \ ,psfTheta \ - ,momentFwhm \ - ,momentWidMajor \ - ,momentWidMinor \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ + ,momentR1 \ + ,momentRH \ ,apResid \ ,dapResid \ @@ -319,7 +325,13 @@ ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ - , ( " + self.safeDictionaryAccess(header, 'IQ_FW1') + " + " + self.safeDictionaryAccess(header, 'IQ_FW2') + " ) / 2.0 \ - ," + self.safeDictionaryAccess(header, 'IQ_FW1') + " \ - ," + self.safeDictionaryAccess(header, 'IQ_FW2') + " \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ @@ -407,8 +419,8 @@ ,psfFlux \ ,psfFluxErr \ - ,peakADU \ - ,psfWidMajor \ - ,psfWidMinor \ + ,psfMajorFWHM \ + ,psfMinorFWHM \ ,psfTheta \ + ,psfCore \ ,psfLikelihood \ ,psfQf \ @@ -416,4 +428,10 @@ ,momentXY \ ,momentYY \ + ,momentR1 \ + ,momentRH \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ ,apFlux \ ,kronFlux \ @@ -424,8 +442,4 @@ ,sky \ ,skyErr \ - ,sgSep \ - ,activeFlag \ - ,assocDate \ - ,historyModNum \ ,dataRelease \ ) \ @@ -441,8 +455,8 @@ ,PSF_INST_FLUX / " + self.header['EXPTIME'] + " \ ,PSF_INST_FLUX_SIG / " + self.header['EXPTIME'] + " \ - ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \ ,PSF_MAJOR \ ,PSF_MINOR \ ,PSF_THETA \ + ,-999 \ ,psfLikelihood(EXT_NSIGMA) \ ,PSF_QF \ @@ -450,4 +464,10 @@ ,MOMENTS_XY \ ,MOMENTS_YY \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ @@ -458,8 +478,4 @@ ,SKY / " + self.header['EXPTIME'] + " \ ,SKY_SIGMA / " + self.header['EXPTIME'] + " \ - ,EXT_NSIGMA \ - , 0 \ - , '" + self.dateStr + "' \ - , 0 \ , " + str(self.skychunk.dataRelease) + "\ FROM " + ippTableName @@ -496,5 +512,5 @@ # we don't delete these anymore results['NULLINSTFLUX'] = 0; - results['NULLPEAKADU'] = 0; + ''' @@ -531,10 +547,10 @@ def populateDetectionCalibTableUpdateInsert(self, ota): - tableName = "DetectionCalib_" + ota + tableName = "Detection_" + ota # drop then re-create table - self.scratchDb.dropTable(tableName) - sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" - try: self.scratchDb.execute(sql) - except: pass + #self.scratchDb.dropTable(tableName) + #sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" + #try: self.scratchDb.execute(sql) + #except: pass # insert all detections into table @@ -555,5 +571,5 @@ ,surveyID \ FROM Detection_" + ota - self.scratchDb.execute(sql) +# self.scratchDb.execute(sql) # insert calibration information from dvoDetections into the Table @@ -564,5 +580,4 @@ a.decErr = b.decErr, \ a.zp = b.zp, \ - a.zpErr = b.zpErr, \ a.expTime = b.expTime, \ a.airMass = b.airMass \ @@ -577,66 +592,33 @@ # target table name: - tableName = "DetectionCalib_" + ota + tableName = "Detection_" + ota # drop then re-create table - self.scratchDb.dropTable(tableName) - sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" - try: self.scratchDb.execute(sql) - except: pass + #self.scratchDb.dropTable(tableName) + #sql = "CREATE TABLE " + tableName + " LIKE Detection" + #try: self.scratchDb.execute(sql) + #except: pass externID = self.imageIDs[ota] imageID = self.scratchDb.getImageIDFromExternID(externID) - self.logger.infoPair("obtained","imageID") - - # check for & create output directory first - datadumpDir = "/tmp/datadump" - try: - statinfo = os.stat(datadumpDir) - # check on the stat results? - except: - print "making the data dump directory ", datadumpDir - os.mkdir(datadumpDir) - os.chmod(datadumpDir, 0777) - statinfo = os.stat(datadumpDir) - - dumpFile = datadumpDir + "/genetest.xx.dat" - files = glob.glob(dumpFile) - if len(files) > 0: - os.unlink(dumpFile) + self.logger.infoPair("obtained imageID",imageID) + self.logger.infoPair("obtained externID",externID) # insert all detections into table - sql = "SELECT \ - a.objID, \ - a.detectID, \ - a.ippObjID, \ - a.ippDetectID, \ - a.filterID, \ - a.surveyID, \ - b.ra, \ - b.dec_, \ - b.raErr, \ - b.decErr, \ - b.zp, \ - b.zpErr, \ - b.expTime, \ - b.airMass, \ - " + str(self.skychunk.dataRelease) + " \ - FROM \ - Detection_" + ota + " as a \ - JOIN " + self.scratchDb.dvoDetectionTable + " as b \ - ON (a.objID = b.objID AND a.detectID = b.detectID) \ - WHERE b.imageID = " + str(imageID) + \ - " INTO OUTFILE '" + dumpFile + "'" + sql = "Update Detection_" + ota + " AS a, "+ self.scratchDb.dvoDetectionTable + " as b \ + set a.ra=b.ra, \ + a.dec = b.dec_, \ + a.raErr = b.raErr, \ + a.decErr = b.decErr, \ + a.zp = b.zp, \ + a.expTime = b.expTime, \ + a.airMass = b.airMass \ + WHERE (a.objID = b.objID AND a.detectID = b.detectID) \ + AND b.imageID = " + str(imageID) \ + try: self.scratchDb.execute(sql) except: - self.logger.info("failed to select data for detectionCalib") - self.logger.infoPair("sql: ", sql) - raise - - sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName - try: self.scratchDb.execute(sql) - except: - self.logger.info("failed to load data from infile for detectionCalib") + self.logger.info("failed to update data for detectionCalib") self.logger.infoPair("sql: ", sql) raise @@ -751,5 +733,5 @@ # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of # the column in PSPS - self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double") + self.scratchDb.execute("ALTER TABLE Detection CHANGE dec_ `dec` double") return True @@ -818,23 +800,22 @@ return False - self.logger.info("populate stuff "); + #self.logger.info("populate stuff "); # populate remainder of tables self.populateDetectionTable(chipname, results) - self.logger.info("successful populate "); + #self.logger.info("successful populate "); # now add DVO IDs self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) - self.logger.info("updated dvoids") + #self.logger.info("updated dvoids") results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") #self.logger.info("deleted nulls") self.updateImageID("Detection_" + chipname, x, y) - self.logger.info("updateImageId") + #self.logger.info("updateImageId") rowCount = self.scratchDb.getRowCount("Detection_" + chipname) - self.logger.info("got row count") - self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", + #self.logger.info("got row count") + self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", chipname, results['ORIGINALTOTAL'], results['SATDET'], results['NULLINSTFLUX'], - results['NULLPEAKADU'], results['NULLOBJID'], rowCount) @@ -842,5 +823,4 @@ self.totalSatDet = self.totalSatDet + results['SATDET'] self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX'] - self.totalNullPeakFlux = self.totalNullPeakFlux + results['NULLPEAKADU'] self.totalNullObjID = self.totalNullObjID + results['NULLOBJID'] self.totalDetections = self.totalDetections + rowCount @@ -867,11 +847,8 @@ self.scratchDb.execute(sql) - self.logger.info("updated imagedata") - self.populateSkinnyObjectTable(chipname) - self.logger.info("updated skinnyobject") - #self.populateObjectCalColorTable(chipname) - #self.logger.info("updated objectcalcolor") + #self.logger.info("updated imagedata") + self.populateDetectionCalibTable(chipname) - self.logger.info("updated detectioncalibtable") + #self.logger.info("updated detectioncalibtable") # add these to list of tables to export later @@ -882,11 +859,5 @@ self.logger.info("export Detection") - self.tablesToExport.append("SkinnyObject_" + chipname) - self.logger.info("export Skinny") - - #self.tablesToExport.append("ObjectCalColor_" + chipname) - self.tablesToExport.append("DetectionCalib_" + chipname) - self.logger.info("export DetectionCalib") - + tables.append("Detection_" + chipname) self.logger.info("updated detectioncalibtable") @@ -910,7 +881,7 @@ results = {} self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0 - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") - self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL obj ID | Remainder |") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") for x in range(self.startX, self.endX): for y in range(self.startY, self.endY): @@ -937,5 +908,5 @@ # print totals - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", self.totalOriginal, @@ -945,5 +916,5 @@ self.totalNullObjID, self.totalDetections) - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)