Index: /trunk/ippToPsps/config/tables.DF.vot
===================================================================
--- /trunk/ippToPsps/config/tables.DF.vot (revision 37747)
+++ /trunk/ippToPsps/config/tables.DF.vot (revision 37748)
@@ -36,5 +36,5 @@
saturation magnitude level
-
+
analysis version index
Index: /trunk/ippToPsps/jython/diffbatch.py
===================================================================
--- /trunk/ippToPsps/jython/diffbatch.py (revision 37747)
+++ /trunk/ippToPsps/jython/diffbatch.py (revision 37748)
@@ -0,0 +1,621 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+import glob
+import time
+import stilts
+
+from java.lang import *
+from java.sql import *
+
+from java.lang import Math
+from org.apache.commons.math.special import Erf
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from sqlUtility import sqlUtility
+
+import logging.config
+
+'''
+DiffBatch class
+
+This class, a sub-class of Batch, processes single exposures in the form of IPP smf files
+
+'''
+class DiffBatch(Batch):
+
+ '''
+ Constructor
+ '''
+ def __init__(self,
+ logger,
+ config,
+ skychunk,
+ gpc1Db,
+ ippToPspsDb,
+ scratchDb,
+ diffSkyFileID,
+ batchID):
+
+ super(DiffBatch, self).__init__(
+ logger,
+ config,
+ skychunk,
+ gpc1Db,
+ ippToPspsDb,
+ scratchDb,
+ diffSkyFileID,
+ batchID,
+ "DF",
+ None)
+ # gpc1Db.getDiffStageCmf(skychunk.dvoLabel,diffSkyFileID))
+
+ # get diff meta data
+ # if error about catID you are usings an old ipptopsps_scratch db
+ self.logger.infoPair("getting diff" ," metadata")
+ meta = self.gpc1Db.getDiffStageMeta(diffSkyFileID)
+ self.logger.infoPair("ok, I guess that worked","now doing this")
+ self.diffSkyFileID = diffSkyFileID
+ if not meta:
+ self.logger.errorPair("Could not get", "diff metadata")
+ raise
+ self.logger.infoPair("so meta", "much amaze")
+ self.filterName = meta[0];
+ self.difftypeID = meta[1];
+ self.posImageID = meta[2];
+ self.negImageID = meta[3];
+ self.skycellName = meta[4];
+ self.analVer = meta[5];
+ self.expTime = meta[6];
+ self.fits = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, diffSkyFileID)
+ self.header = self.fits.getPrimaryHeader()
+ tessName = self.getKeyValue(self.header, 'TESS_ID')
+ self.tessID = self.scratchDb.getTessID(tessName)
+
+
+ # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL
+ if tessName == 'RINGS.V3':
+ # skycell is, eg "skycell.1133.081"
+ # 0123456789012345
+ self.projectionID = self.skycellName[8:12]
+ self.skycellID = self.skycellName[13:]
+ else:
+ # skycell is, eg "skycell.081"
+ # 01234567890
+ self.projectionID = 0
+ self.skycellID = self.skycellName[8:11]
+ if self.skycellID == "":
+ self.skycellID = -1
+
+
+ # drop the existing tables
+
+ self.dropTableVerbose("DiffMeta")
+ self.dropTableVerbose("DiffToImage")
+ self.dropTableVerbose("DiffDetection")
+
+
+
+
+ self.logger.infoPair("I can haz",diffSkyFileID)
+
+
+
+ if not self.analVer: self.analVer = -999
+
+ # create an output filename, which is {expID}.FITS
+ self.outputFitsFile = "%08d.FITS" % self.diffSkyFileID
+ self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+ print "got here"
+ self.photoCalID = str(self.scratchDb.getPhotoCalID(self.diffSkyFileID))
+ print "got photocalid"
+ self.calibModNum = 0;
+ self.dataRelease = skychunk.dataRelease;
+
+ self.filterID = self.scratchDb.getFilterID(self.filterName[0])
+
+ print self.filterName
+ print self.filterName[0]
+ print self.filterID
+ # insert what we know about this stack batch into the stack table
+ self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[0], self.difftypeID)
+
+ # dump stuff to log
+ self.logger.infoPair("diff_skyfile_ID", "%d" % self.diffSkyFileID)
+
+
+ '''
+ Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header
+ '''
+ def populateDiffMeta(self):
+ self.logger.infoPair("Proccesing table", "DiffMeta")
+
+ sqlLine = sqlUtility("INSERT INTO DiffMeta (")
+
+ sqlLine.group("diffMetaID", str(self.diffSkyFileID));
+ sqlLine.group("difftypeID", self.difftypeID);
+ sqlLine.group("posImageID", self.posImageID);
+ sqlLine.group("negImageID", self.negImageID);
+ sqlLine.group("skycellID", self.skycellID);
+ sqlLine.group("photoCalID", self.photoCalID);
+ sqlLine.group("magSat", self.getKeyFloat(self.header, "%.8f", 'FSATUR'));
+ sqlLine.group("analVer", str(self.analVer));
+ sqlLine.group("expTime", self.expTime);
+ sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1'));
+ sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2'));
+ sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1'));
+ sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2'));
+ sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1'));
+ sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2'));
+ sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1'));
+ sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2'));
+ sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001'));
+ sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002'));
+ sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001'));
+ sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002'));
+
+ sql = sqlLine.make(") VALUES ( ", ")")
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair('failed sql: ', sql)
+ raise
+ self.logger.infoPair("DiffMeta","final stuff")
+ self.scratchDb.updateAllRows("DiffMeta", "batchID", str(self.batchID))
+ self.scratchDb.updateAllRows("DiffMeta", "surveyID", str(self.surveyID))
+ self.scratchDb.updateFilterID("DiffMeta", self.filterName[0])
+ self.scratchDb.updateAllRows("DiffMeta", "calibModNum", str(self.calibModNum))
+ self.scratchDb.updateAllRows("DiffMeta", "dataRelease", str(self.skychunk.dataRelease))
+ self.logger.infoPair("DiffMeta","done")
+
+ '''
+ Populates the Diff table for this OTA
+ '''
+ def populateDiffTable(self, ota, results):
+
+ pspsTableName = "DiffDetection"
+ ippTableName = "diff_psf"
+
+ results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
+
+ # drop then re-create table
+ self.scratchDb.dropTable(pspsTableName)
+ sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
+ try: self.scratchDb.execute(sql)
+ except: pass
+
+ # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5)
+ # for examples of code which do this, go to r35097 or earlier
+
+ BEFORE = self.scratchDb.getRowCount(ippTableName)
+ results['SATDET'] = 0
+
+ extTimeString = str(self.expTime)
+
+ # insert all detections into table : do NOT include the IGNORE unless we have to
+ # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (")
+ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
+
+ # XXX WARNING: need to apply platescale to convert sizes to arcsec
+ # XXX apFluxF or apFluxNpix + apFluxRadius?
+
+ sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("randomDiffID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")
+ sqlLine.group("filterID", str(self.filterID))
+ sqlLine.group("surveyID", str(self.surveyID))
+ ##sqlLine.group("obsTime", str(self.obsTime)) I have expTime
+ sqlLine.group("expTime", str(self.expTime))
+ sqlLine.group("xPos", "X_PSF")
+ sqlLine.group("yPos", "Y_PSF")
+ sqlLine.group("xPosErr", "X_PSF_SIG")
+ sqlLine.group("yPosErr", "Y_PSF_SIG")
+ if (self.id >= 982483):
+ sqlLine.group("pltScale", "PLTSCALE")
+ sqlLine.group("posAngle", "POSANGLE")
+# sqlLine.group("raErr", "X_PSF_SIG * abs(PLTSCALE)")
+# sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)")
+ else:
+ sqlLine.group("pltScale", "0.257")
+ sqlLine.group("posAngle", "-999")
+# sqlLine.group("raErr", "X_PSF_SIG * 0.257")
+# sqlLine.group("decErr", "Y_PSF_SIG * 0.257")
+ sqlLine.group("psfFlux", "PSF_INST_FLUX / " + extTimeString)
+ sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / " + extTimeString)
+ if (self.id >= 982483):
+ sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ")
+ sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN")
+ sqlLine.group("psfCore", "PSF_CORE")
+ sqlLine.group("psfTheta", "PSF_THETA")
+ sqlLine.group("psfQf", "PSF_QF")
+ sqlLine.group("psfQfPerfect", "PSF_QF_PERFECT")
+ sqlLine.group("psfChiSq", "PSF_CHISQ")
+ sqlLine.group("psfLikelihood", "psfLikelihood(EXT_NSIGMA)")
+ sqlLine.group("momentXX", "MOMENTS_XX")
+ sqlLine.group("momentXY", "MOMENTS_XY")
+ sqlLine.group("momentYY", "MOMENTS_YY")
+ sqlLine.group("momentR1", "MOMENTS_R1")
+ sqlLine.group("momentRH", "MOMENTS_RH")
+# sqlLine.group("momentM3C", "MOMENTS_M3C")
+# sqlLine.group("momentM3S", "MOMENTS_M3S")
+# sqlLine.group("momentM4C", "MOMENTS_M4C")
+# sqlLine.group("momentM4S", "MOMENTS_M4S")
+ if (self.id >= 982483):
+ sqlLine.group("apFlux", "AP_FLUX / " + extTimeString)
+ sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString)
+ sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
+ # sqlLine.group("apRadius", "AP_MAG_RADIUS")
+ sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString)
+ sqlLine.group("kronFluxErr", "KRON_FLUX_ERR / " + extTimeString)
+ sqlLine.group("kronRad", "MOMENTS_R1 * 2.5")
+ sqlLine.group("diffNPos", "DIFF_NPOS")
+ sqlLine.group("diffFRatio", "DIFF_FRATIO")
+ sqlLine.group("diffNBad", "DIFF_NRATIO_BAD")
+ sqlLine.group("diffNMask", "DIFF_NRATIO_MASK")
+ sqlLine.group("diffNAll", "DIFF_NRATIO_ALL")
+ sqlLine.group("diffPdist", "DIFF_R_P")
+ sqlLine.group("diffNdist", "DIFF_R_M")
+ sqlLine.group("diffPSN", "DIFF_SN_P")
+ sqlLine.group("diffNSN", "DIFF_SN_M")
+ sqlLine.group("sky", "SKY / " + extTimeString)
+ sqlLine.group("skyErr", "SKY_SIGMA / " + extTimeString)
+ sqlLine.group("infoFlag", "FLAGS")
+ sqlLine.group("infoFlag2", "FLAGS2")
+ sqlLine.group("dataRelease", str(self.skychunk.dataRelease))
+
+ sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+ # NOTE : Flux limits : in the current PSPS schema, negative fluxes
+ # cause problems for sql queries which work in mags as SQL cannot do
+ # something like (f < 0.0) ? -999 : -2.5*log10(f)
+ # as a result, the negative fluxes here result in floating point errors
+ # XXX EAM 2014072 : Is this still a problem?
+ # sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0"
+ # self.scratchDb.execute(sql)
+
+ # sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0"
+ # self.scratchDb.execute(sql)
+
+ # sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0"
+ # self.scratchDb.execute(sql)
+
+ # we don't delete these anymore
+ results['NULLINSTFLUX'] = 0;
+
+
+# insert stuff from dvo
+
+
+
+ '''
+ Applies indexes and other constraints to the PSPS tables
+ '''
+ def alterPspsTables(self):
+
+ self.logger.debug("Altering PSPS tables")
+ self.logger.debug("Creating indexes on PSPS tables")
+ self.scratchDb.makeColumnUnique("DiffDetection", "objID")
+ self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
+
+ self.populateDiffMeta()
+
+ # dictionary objects to hold imageIDs for later
+ self.imageIDs = {}
+ self.sourceIDs = {}
+
+ # loop through all OTAs and populate ImageMeta extensions
+ # the column in PSPS
+ self.scratchDb.execute("ALTER TABLE DiffDetection CHANGE dec_ `dec` double")
+
+ return True
+
+
+ '''
+ Applies indexes to the IPP tables
+ '''
+ def indexIppTables(self):
+
+ self.logger.infoPair("Creating indexes on", "IPP tables")
+
+ self.scratchDb.createIndex(extension, "IPP_IDET")
+ # try the test Chip
+ self.scratchDb.createIndex("Chip_psf", "IPP_IDET")
+
+ self.logger.infoPair("created indexes on", "IPP tables")
+
+ '''
+ Updates provided table with DVO IDs from DVO table
+ '''
+ def updateDvoIDs(self, table, externID):
+
+ imageID = self.scratchDb.getImageIDFromExternID(externID)
+ self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+
+ sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
+ a.objID = b.objID, \
+ a.detectID = b.detectID, \
+ a.ippObjID = b.ippObjID, \
+ a.dvoRegionID = b.catID, \
+ a.ra = b.ra, \
+ a.dec = b.dec_, \
+ a.zp = b.zp, \
+ a.telluricExt = b.telluricExt, \
+ a.airmass = b.airmass, \
+ a.expTime = b.expTime, \
+ a.infoFlag2 = (b.flags << 13) | a.infoFlag2 \
+ WHERE a.ippDetectID = b.ippDetectID \
+ AND b.imageID = " + str(imageID)
+
+ self.scratchDb.execute(sql)
+
+ '''
+ Updates table and generates pspsuniqueids
+ '''
+
+ def updatePspsUniqueIDs(self,table):
+ sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
+ (select @r:=0) r, "+table+" t) as foo using (objID) set \
+ uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair('failed sql',sql)
+ return
+
+ '''
+ Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+ '''
+ def populatePspsTablesChip(self, chipname, x, y, results, tables):
+ # XXX EAM NOTE: drop tables Detection_* here so
+ # they do not polute the db?
+ # XXX or put an explicit drop at the end of the loop?
+
+ # self.logger.infoTitle("Processing " + chipname)
+ # this is a bit crude: if the chip is not present, this test will fail and the chip
+ # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing")
+
+ # XXX keep this in or not?
+ # if not self.scratchDb.astrometricSolutionOK(chipname):
+ # self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname)
+ # return False
+
+ # does this chip exist in the DVO image table (if NOT, then skip it)
+ if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
+ self.logger.info("| %5s | ------------- Chip not in DVO : rejecting ------------ |" % chipname)
+ return False
+
+ #self.logger.info("populate stuff ");
+ # populate remainder of tables
+ self.populateDetectionTable(chipname, results)
+ #self.logger.info("successful populate ");
+ # now add DVO IDs
+ self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
+ #self.logger.info("updated dvoids")
+ results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
+ #self.logger.info("deleted nulls")
+ self.logger.info("add psps unique p2 ids")
+ self.updatePspsUniqueIDs("Detection_" + chipname, self.imageIDs[chipname])
+ self.updateImageID("Detection_" + chipname, x, y)
+ #self.logger.info("updateImageId")
+ rowCount = self.scratchDb.getRowCount("Detection_" + chipname)
+ #self.logger.info("got row count")
+ self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |",
+ chipname,
+ results['ORIGINALTOTAL'],
+ results['SATDET'],
+ results['NULLINSTFLUX'],
+ results['NULLOBJID'],
+ rowCount)
+ self.totalOriginal = self.totalOriginal + results['ORIGINALTOTAL']
+ self.totalSatDet = self.totalSatDet + results['SATDET']
+ self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX']
+ self.totalNullObjID = self.totalNullObjID + results['NULLOBJID']
+ self.totalDetections = self.totalDetections + rowCount
+
+ fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
+ # print "kept ", str(fractionKept), " detections from ", chipname
+
+ if fractionKept < 0.8:
+ # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
+ self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
+ self.skipBatch = True
+ return False;
+
+ #self.logger.info("updated totals")
+ # check we have something in this Detection table TODO add this to table above
+ if rowCount < 1:
+ self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
+ self.skipBatch = True
+ return False;
+
+ # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
+ sql = "UPDATE ImageMeta_" + chipname + " \
+ SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
+ self.scratchDb.execute(sql)
+
+ # add these to list of tables to export later
+ self.tablesToExport.append("ImageMeta_" + chipname)
+ self.logger.info("export ImageMeta")
+
+ self.tablesToExport.append("Detection_" + chipname)
+ self.logger.info("export Detection")
+
+ tables.append("Detection_" + chipname)
+ self.logger.info("updated detection")
+
+ self.validChips.append(chipname)
+
+ return True
+
+ '''
+ Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+ '''
+ def populatePspsTables(self):
+
+ # loop through all OTAs again to update with DVO IDs
+ self.skipBatch = False
+ self.validChips = []
+ self.tablesToExport = []
+ self.tablesToExport.append("DiffMeta")
+ tables = []
+ otaCount = 0
+ results = {}
+ self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0
+ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+ self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL obj ID | Remainder |")
+ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+ for x in range(self.startX, self.endX):
+ for y in range(self.startY, self.endY):
+
+ # dodge the corners
+ if x==0 and y==0: continue
+ if x==0 and y==7: continue
+ if x==7 and y==0: continue
+ if x==7 and y==7: continue
+
+ ota = "XY%d%d" % (x, y)
+ if ota not in self.imageIDs: continue
+
+ if False and self.config.test and not ((x == 0) and (y == 1)):
+ print "skipping ota " + ota
+ continue
+
+ if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
+ if self.skipBatch:
+ self.logger.error("fatal problem for exposure, skipping")
+ return False
+
+ if "Chip" in self.imageIDs:
+ if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+ if self.skipBatch:
+ self.logger.error("fatal problem for exposure, skipping")
+ return False
+
+ # print totals
+ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+ self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |",
+ self.totalOriginal,
+ self.totalSatDet,
+ self.totalNulIInstFlux,
+ self.totalNullPeakFlux,
+ self.totalNullObjID,
+ self.totalDetections)
+ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+
+ # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)
+ if len(self.tablesToExport) == 1:
+ self.skipBatch = True
+ self.logger.error("No tables to export")
+ return False
+
+ # XXX EAM : turn this on (and fix up the code) if we need it
+ # self.removeDuplicateObjects()
+
+ self.setMinMaxObjID(tables)
+
+ # update FrameMeta with count OTAs in this file and total number of photometric reference sources
+ sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
+ self.scratchDb.execute(sql)
+ #to make it stop
+ #test = self.scratchDb.getRowCount("Object")
+ #return False
+
+ return True
+
+ def removeDuplicateObjects(self):
+
+ sql = "DROP TABLE SkinnyObject_All"
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ print "SkinnyObject_All table not yet defined"
+
+ sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))"
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ print "failed to create table SkinnyObject_All"
+ raise
+
+ for chipname in self.validChips:
+
+ sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \
+ SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ print "failed to insert entry SkinnyObject_All: ", chipname
+ raise
+
+
+ Nduplicates = 0
+ sql = "SELECT n, objID from (select count(objID) as n, objID from SkinnyObject_All group by objID) as tmp where n > 1"
+ try:
+ rs = self.scratchDb.executeQuery(sql)
+ while (rs.next()):
+ objID = rs.getInt(1)
+ chipID = rs.getString(3)
+ print "delete: ", str(objID), chipID
+ Nduplicates += 1
+
+ rs.close()
+ except:
+ print "failed to get duplicates"
+ raise
+
+ print "duplicates found: ", str(Nduplicates)
+
+ '''
+ Updates imageID {EXP_ID}{OTA} in the provided table
+ '''
+ def updateImageID(self, tableName, x, y):
+
+ sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y)
+ self.scratchDb.execute(sql)
+
+ '''
+ Overriding this method
+ '''
+ def reportNullsInAllPspsTables(self, showPartials):
+
+ # loops round all imported tables, but we want to check one OTA's worth
+ for table in self.pspsTables:
+ if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials)
+ else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials)
+
+
+ '''
+ This function reads the cmf/smf file and loads it into the database as its own table
+ Overriding this method. Filter to only import *.psf extensions
+ '''
+ def importIppTables(self, filter=""):
+
+ regex = ".*.psf"
+ if False and self.config.test and self.config.camera == "gpc1":
+ regex = "XY01.psf"
+
+ print "my ID: " + str(self.id)
+
+ if (self.id < 982483):
+ # XXX EAM NOTE : this is fragile : requires PS1_V4
+ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_MAJOR PSF_MINOR PSF_THETA PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
+ else:
+ # XXX EAM NOTE : this is fragile : requires PS1_V5
+ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
+
+ return super(DetectionBatch, self).importIppTables(columns, regex)
+
+ '''
+ Overriding this method. Use regex to trim off, eg _XY33 extension
+ '''
+ def exportPspsTablesToFits(self, regex="(.*)"):
+ return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
Index: /trunk/ippToPsps/jython/gpc1db.py
===================================================================
--- /trunk/ippToPsps/jython/gpc1db.py (revision 37747)
+++ /trunk/ippToPsps/jython/gpc1db.py (revision 37748)
@@ -108,8 +108,38 @@
AND mjd_obs <= (to_days('" + tend + "') - 678941) "
# self.logger.infoPair("sql",sql)
- elif batchType == "DF":
+ elif batchType == "DF": # this is specifically only for WARPSTACK types of diffs -
+ # it only makes sense for the date to come from the warp1
+ # may need to be more clever if we do other types of diffs
stage = "diff"
- sql = "SELECT stuff from stufftable"
+ sql = "SELECT DISTINCT stage_extra1, radeg,decdeg \
+ FROM addRun \
+ JOIN addProcessedExp using (add_id) \
+ JOIN minidvodbRun using (minidvodb_name) \
+ JOIN minidvodbProcessed using (minidvodb_id) \
+ JOIN mergedvodbRun using (minidvodb_id) \
+ JOIN mergedvodbProcessed using (merge_id) \
+ JOIN diffInputSkyfile \
+ ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
+ AND stage = 'diff') \
+ JOIN skycell using (skycell_id, tess_id) \
+ JOIN warpRun on (warp1 = warp_id \
+ AND diffInputSkyfile.skycell_id = skycell_id) \
+ JOIN fakeRun using (fake_id) \
+ JOIN camRun using (cam_id) \
+ JOIN chipRun using (chip_id) \
+ JOIN rawExp using (exp_id) \
+ WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+ AND minidvodbRun.state = 'merged' \
+ AND minidvodbProcessed.fault = 0 \
+ AND mergedvodbRun.state = 'full' \
+ AND mergedvodbProcessed.fault = 0 \
+ AND addRun.stage = '" + stage + "' \
+ AND addRun.state = 'full' \
+ AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
+ AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+ AND dateobs >= '" + tstart + "'\
+ AND dateobs <= '" + tend + "'"
+
elif batchType == "FW":
@@ -123,5 +153,5 @@
JOIN mergedvodbProcessed USING (merge_id) \
JOIN fullForceRun USING(ff_id) \
- JOIN skycalRun using (skycal_id)
+ JOIN skycalRun using (skycal_id) \
JOIN stackRun USING(stack_id) \
JOIN stackSumSkyfile using (stack_id) \
@@ -222,4 +252,94 @@
return meta
+
+ '''
+ Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved
+ '''
+ def getDiffStageMeta(self, diffSkyfileID):
+
+ self.logger.debug("Querying GPC1 for stack meta data")
+
+ meta = []
+ sql = "SELECT \
+ filter,\
+ diff_mode, \
+ warp1, \
+ stack2, \
+ skycell_id, \
+ diffRun.software_ver, \
+ exp_time \
+ FROM diffRun join diffInputSkyfile using (diff_id) \
+ JOIN warpRun on (warp1=warp_id) \
+ JOIN fakeRun using (fake_id) \
+ JOIN camRun using (cam_id) \
+ JOIN chipRun using (chip_id) \
+ JOIN rawExp using (exp_id) \
+ WHERE diff_skyfile_id = %d" % diffSkyfileID
+
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ except:
+ self.logger.infoPair ("this sql is broken:",sql)
+ self.logger.errorPair("Can't query for", "stack meta data")
+
+ try:
+ meta.append(rs.getString(1))
+ meta.append(rs.getString(2))
+ meta.append(rs.getString(3))
+ meta.append(rs.getString(4))
+ meta.append(rs.getString(5))
+ meta.append(rs.getString(6))
+ meta.append(rs.getString(7))
+
+ except:
+ self.logger.errorPair("getDiffStageMeta()", "empty meta data")
+
+ return meta
+
+ '''
+ get diff stage cmf name
+ '''
+ def getDiffStageCmf(self, dvoDb, diffSkyfileID):
+ print "got here, diffstagecmf"
+ sql = "SELECT DISTINCT diffSkyfile.path_base \
+ FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \
+ JOIN addRun using (minidvodb_name) \
+ JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \
+ AND diff_id = stage_id AND stage = 'diff') \
+ JOIN diffSkyfile using (diff_id, skycell_id) \
+ WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \
+ AND minidvodbRun.state = 'merged' \
+ AND mergedvodbRun.state = 'full' \
+ AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+ AND stage = 'diff'"
+
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ except:
+ self.logger.infoPair("failed sql", sql)
+ self.logger.errorPair("can't query for diff cmf", "dfif_skyfile_id = %d" %diff_skyfile_id)
+ return None
+ print "query worked"
+ try:
+ pathBase = rs.getString(1)
+ except:
+ self.logger.errorPair("no diff cms found for diff_skyfile_id = %d" %diff_skyfile_id)
+ return None
+ print pathBase
+ files = []
+ if pathBase.startswith("neb"):
+ print "open"
+ f=os.popen("neb-ls -p " + pathBase + ".cmf")
+ for i in f.readlines():
+ print "i",i
+ files.append(i.rstrip())
+ print "append"
+ print i
+ if len(files) < 1: return None
+ fits = Fits(self.logger, self.config, files[0])
+ return fits
+
'''
Gets some camera-stage meta data for this cam_id
@@ -394,19 +514,4 @@
def getStackStageCmf(self, dvoDb, stackID):
- '''
- sql = "SELECT DISTINCT staticskyResult.path_base \
- FROM staticskyRun \
- JOIN staticskyInput USING(sky_id) \
- JOIN staticskyResult USING(sky_id) \
- JOIN stackRun USING(stack_id) \
- JOIN stackSumSkyfile USING(stack_id) \
- JOIN addRun ON sky_id = stage_id \
- JOIN minidvodbRun USING (minidvodb_name) \
- WHERE stack_id = " + str(stackID) + " \
- AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
- AND minidvodbRun.state = 'merged' \
- AND stage = 'staticsky'"
- '''
-
sql = "SELECT DISTINCT skycalResult.path_base \
FROM skycalRun \
@@ -458,41 +563,4 @@
fits = Fits(self.logger, self.config, files[0])
return fits
-
- # XXX validate the file?
- # if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-
- # now find and loop through all cmf files at this path_base
-
- ## XXX the code below was needed when we ingested from staticsky (N files per run)
- ## XXX ingesting from skycal solves this problem (1 file per run)
- ## files=os.popen("neb-ls " + pathBase + "%cmf")
- ## for i in files.readlines():
- ## nebPath = i.rstrip()
- ##
- ## # now attempt to run 'neb-ls -p' to actually read this cmf file
- ## try:
- ## self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
- ## paths=os.popen("neb-ls -p " + nebPath)
- ## path = paths.readline().rstrip()
- ## if len(path) < 1:
- ## self.logger.debug("zero length path - skipping")
- ## raise
- ##
- ## print "starint stack stage cmf 3"
- ##
- ## # if we get here, then the cmf is readable, now check the stack_id
- ## fits = Fits(self.logger, self.config, path)
- ## if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
- ## # we have the right file!
- ## self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
- ## return fits
- ## else:
- ## # this is not the correct file
- ## self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
- ##
- ## # an exception here means that nebulous could not read this file, so we just skip it
- ## except:
- ## self.logger.debug("NEB FAILED SKIPPING")
- ## pass
'''
Index: /trunk/ippToPsps/jython/initbatch.py
===================================================================
--- /trunk/ippToPsps/jython/initbatch.py (revision 37747)
+++ /trunk/ippToPsps/jython/initbatch.py (revision 37748)
@@ -16,4 +16,7 @@
from stackbatch import StackBatch
from detectionbatch import DetectionBatch
+#from diffbatch import DiffBatch
+#from forcedwarpbatch import ForcedWarpBatch
+#why no object stuffs
'''
Index: /trunk/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- /trunk/ippToPsps/jython/ipptopspsdb.py (revision 37747)
+++ /trunk/ippToPsps/jython/ipptopspsdb.py (revision 37748)
@@ -799,5 +799,5 @@
def insertDiffMeta(self, batchID, filter, diffType):
- sql = "INSERT INTO stack ( \
+ sql = "INSERT INTO diff ( \
batch_id \
,filter \
Index: /trunk/ippToPsps/jython/loader.py
===================================================================
--- /trunk/ippToPsps/jython/loader.py (revision 37747)
+++ /trunk/ippToPsps/jython/loader.py (revision 37748)
@@ -24,4 +24,9 @@
from objectbatch import ObjectBatch
+from diffbatch import DiffBatch
+#from diffobjectbatch import DiffObjectBatch
+
+#
+#
'''
Loader class
@@ -191,8 +196,8 @@
# loop round IDs of all items to be processed
self.ippToPspsDb.lockBatchTable()
- #self.logger.infoPair("heather:","lockbatchtable")
+ self.logger.infoPair("heather:","lockbatchtable")
unattemptedCount = 0
for id in ids:
- #self.logger.infoPair("heather:","in ids")
+ self.logger.infoPair("heather:","in ids")
#self.logger.infoPair("heather:id",str(id))
batchID = self.ippToPspsDb.createNewBatch(batchType, id)
@@ -202,5 +207,5 @@
unattemptedCount += 1
continue
- #self.logger.infoPair("heather:","passed logic")
+ self.logger.infoPair("heather:","passed logic")
self.ippToPspsDb.unlockTables()
@@ -244,4 +249,5 @@
elif batchType == "DF":
+ self.logger.infoPair("wegothere!","we are awesome")
batch = DiffBatch(self.logger,
self.config,
Index: /trunk/ippToPsps/jython/objectbatch.py
===================================================================
--- /trunk/ippToPsps/jython/objectbatch.py (revision 37747)
+++ /trunk/ippToPsps/jython/objectbatch.py (revision 37748)
@@ -94,5 +94,5 @@
sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \
(select @r:=0) r, ObjectThin t) as foo using (objID) set \
- uniquePspsP2id = ((" +str(self.batchID) "*1000000000 ) + rownum)";
+ uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)";
try: self.scratchDb.execute(sql)
except:
Index: /trunk/ippToPsps/jython/stackbatch.py
===================================================================
--- /trunk/ippToPsps/jython/stackbatch.py (revision 37747)
+++ /trunk/ippToPsps/jython/stackbatch.py (revision 37748)
@@ -955,5 +955,5 @@
(select @r:=0) r, StackObjectThin t) as foo using (objID) set \
stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \
- uniquePspsSTid = ((" +str(self.batchID) "*1000000000 ) + rownum)";
+ uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)";
try: self.scratchDb.execute(sql)
except:
Index: /trunk/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta
===================================================================
--- /trunk/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 37747)
+++ /trunk/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 37748)
@@ -9,5 +9,5 @@
photoCalID dimensionless INT 4 NA photometry code
magSat dimensionless REAL 4 -999 saturation magnitude level
-analVer dimensionless SMALLINT 2 -999 analysis version index
+analVer alphanumeric VARCHAR(100) 100 -999 analysis version index
expTime seconds REAL 4 -999 exposure time of positive image
ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA