Changeset 37748

Timestamp:

Dec 14, 2014, 5:30:49 PM (12 years ago)

Author:

heather

Message:

diff batch progress (sas 37) - metadata is correct, diff detection is in but needs testing

Location:

trunk/ippToPsps

Files:

• config/tables.DF.vot (modified) (1 diff)
• jython/diffbatch.py (modified) (1 diff)
• jython/gpc1db.py (modified) (5 diffs)
• jython/initbatch.py (modified) (1 diff)
• jython/ipptopspsdb.py (modified) (1 diff)
• jython/loader.py (modified) (4 diffs)
• jython/objectbatch.py (modified) (1 diff)
• jython/stackbatch.py (modified) (1 diff)
• pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta (modified) (1 diff)

trunk/ippToPsps/config/tables.DF.vot

@@ -36 +36 @@
         <DESCRIPTION>saturation magnitude level</DESCRIPTION>
       </FIELD>
-      <FIELD name="analVer" arraysize="1" datatype="short" unit="dimensionless" default="-999">
+      <FIELD name="analVer" arraysize="100" datatype="char" unit="alphanumeric" default="-999">
         <DESCRIPTION>analysis version index</DESCRIPTION>
       </FIELD>

trunk/ippToPsps/jython/diffbatch.py

@@ +1 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+import glob
+import time
+import stilts
+
+from java.lang import *
+from java.sql import *
+
+from java.lang import Math
+from org.apache.commons.math.special import Erf
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from sqlUtility import sqlUtility
+
+import logging.config
+
+'''
+DiffBatch class
+
+This class, a sub-class of Batch, processes single exposures in the form of IPP smf files
+
+'''
+class DiffBatch(Batch):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, 
+                 logger,
+                 config,
+                 skychunk,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 diffSkyFileID,
+                 batchID):
+
+       super(DiffBatch, self).__init__(
+               logger,
+               config,
+               skychunk,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               diffSkyFileID,
+               batchID,
+               "DF", 
+               None)
+    #           gpc1Db.getDiffStageCmf(skychunk.dvoLabel,diffSkyFileID))
+
+       # get diff meta data
+       # if error about catID you are usings an old ipptopsps_scratch db
+       self.logger.infoPair("getting diff" ," metadata")
+       meta = self.gpc1Db.getDiffStageMeta(diffSkyFileID)
+       self.logger.infoPair("ok, I guess that worked","now doing this")
+       self.diffSkyFileID = diffSkyFileID
+       if not meta:
+           self.logger.errorPair("Could not get", "diff metadata")
+           raise
+       self.logger.infoPair("so meta", "much amaze")
+       self.filterName = meta[0];
+       self.difftypeID = meta[1];
+       self.posImageID = meta[2];
+       self.negImageID = meta[3];
+       self.skycellName = meta[4];
+       self.analVer =  meta[5];
+       self.expTime = meta[6];
+       self.fits = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, diffSkyFileID)
+       self.header = self.fits.getPrimaryHeader()
+       tessName = self.getKeyValue(self.header, 'TESS_ID')
+       self.tessID = self.scratchDb.getTessID(tessName)
+
+
+       # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL
+       if tessName == 'RINGS.V3':
+           # skycell is, eg "skycell.1133.081"
+           #                 0123456789012345
+           self.projectionID = self.skycellName[8:12]
+           self.skycellID = self.skycellName[13:]
+       else:
+           # skycell is, eg "skycell.081"
+           #                 01234567890
+           self.projectionID = 0
+           self.skycellID = self.skycellName[8:11]
+       if self.skycellID == "":
+           self.skycellID = -1
+
+
+       # drop the existing tables    
+
+       self.dropTableVerbose("DiffMeta")
+       self.dropTableVerbose("DiffToImage")
+       self.dropTableVerbose("DiffDetection")
+
+       
+
+
+       self.logger.infoPair("I can haz",diffSkyFileID)
+
+
+
+       if not self.analVer: self.analVer = -999
+
+       # create an output filename, which is {expID}.FITS
+       self.outputFitsFile = "%08d.FITS" % self.diffSkyFileID
+       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+       print "got here"
+       self.photoCalID = str(self.scratchDb.getPhotoCalID(self.diffSkyFileID))
+       print "got photocalid"
+       self.calibModNum = 0;
+       self.dataRelease = skychunk.dataRelease;
+        
+       self.filterID = self.scratchDb.getFilterID(self.filterName[0]) 
+       
+       print self.filterName
+       print self.filterName[0]
+       print self.filterID
+        # insert what we know about this stack batch into the stack table
+       self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[0], self.difftypeID)
+
+       # dump stuff to log
+       self.logger.infoPair("diff_skyfile_ID", "%d" % self.diffSkyFileID)
+
+
+    '''
+    Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header
+    '''
+    def populateDiffMeta(self):
+        self.logger.infoPair("Proccesing table", "DiffMeta")
+
+        sqlLine = sqlUtility("INSERT INTO DiffMeta (")
+
+        sqlLine.group("diffMetaID",            str(self.diffSkyFileID));
+        sqlLine.group("difftypeID",            self.difftypeID);
+        sqlLine.group("posImageID",            self.posImageID);
+        sqlLine.group("negImageID",            self.negImageID);
+        sqlLine.group("skycellID",             self.skycellID);
+        sqlLine.group("photoCalID",            self.photoCalID);
+        sqlLine.group("magSat",                self.getKeyFloat(self.header, "%.8f", 'FSATUR'));
+        sqlLine.group("analVer",               str(self.analVer));
+        sqlLine.group("expTime",               self.expTime);
+        sqlLine.group("ctype1",                self.getKeyValue(self.header, 'CTYPE1'));
+        sqlLine.group("ctype2",                self.getKeyValue(self.header, 'CTYPE2'));
+        sqlLine.group("crval1",                self.getKeyFloat(self.header, "%.8f", 'CRVAL1'));
+        sqlLine.group("crval2",                self.getKeyFloat(self.header, "%.8f", 'CRVAL2'));
+        sqlLine.group("crpix1",                self.getKeyFloat(self.header, "%.8f", 'CRPIX1'));
+        sqlLine.group("crpix2",                self.getKeyFloat(self.header, "%.8f", 'CRPIX2'));
+        sqlLine.group("cdelt1",                self.getKeyFloat(self.header, "%.8e", 'CDELT1'));
+        sqlLine.group("cdelt2",                self.getKeyFloat(self.header, "%.8e", 'CDELT2'));
+        sqlLine.group("pc001001",              self.getKeyFloat(self.header, "%.8e", 'PC001001'));
+        sqlLine.group("pc001002",              self.getKeyFloat(self.header, "%.8e", 'PC001002'));
+        sqlLine.group("pc002001",              self.getKeyFloat(self.header, "%.8e", 'PC002001'));
+        sqlLine.group("pc002002",              self.getKeyFloat(self.header, "%.8e", 'PC002002'));
+
+        sql = sqlLine.make(") VALUES ( ", ")")
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql: ', sql)
+            raise
+        self.logger.infoPair("DiffMeta","final stuff")
+        self.scratchDb.updateAllRows("DiffMeta", "batchID", str(self.batchID))
+        self.scratchDb.updateAllRows("DiffMeta", "surveyID", str(self.surveyID))
+        self.scratchDb.updateFilterID("DiffMeta", self.filterName[0])
+        self.scratchDb.updateAllRows("DiffMeta", "calibModNum", str(self.calibModNum))
+        self.scratchDb.updateAllRows("DiffMeta", "dataRelease", str(self.skychunk.dataRelease))
+        self.logger.infoPair("DiffMeta","done")
+
+    '''
+    Populates the Diff table for this OTA
+    '''
+    def populateDiffTable(self, ota, results):
+
+        pspsTableName = "DiffDetection"
+        ippTableName =  "diff_psf"
+       
+        results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
+
+        # drop then re-create table
+        self.scratchDb.dropTable(pspsTableName)
+        sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
+        try: self.scratchDb.execute(sql)
+        except: pass
+       
+        # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5)
+        # for examples of code which do this, go to r35097 or earlier
+
+        BEFORE = self.scratchDb.getRowCount(ippTableName)
+        results['SATDET'] = 0 
+
+        extTimeString = str(self.expTime)
+
+        # insert all detections into table : do NOT include the IGNORE unless we have to
+        # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (")
+        sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
+
+        # XXX WARNING: need to apply platescale to convert sizes to arcsec
+        # XXX apFluxF or apFluxNpix + apFluxRadius?
+
+        sqlLine.group("ippDetectID",     "IPP_IDET")                                                
+        sqlLine.group("randomDiffID",     "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")                       
+        sqlLine.group("filterID",        str(self.filterID))                                        
+        sqlLine.group("surveyID",        str(self.surveyID))                                        
+        ##sqlLine.group("obsTime",         str(self.obsTime))     I have expTime
+        sqlLine.group("expTime",         str(self.expTime))                                    
+        sqlLine.group("xPos",            "X_PSF")                                                   
+        sqlLine.group("yPos",            "Y_PSF")                                                   
+        sqlLine.group("xPosErr",         "X_PSF_SIG")                                               
+        sqlLine.group("yPosErr",         "Y_PSF_SIG")                                               
+        if (self.id >= 982483):
+            sqlLine.group("pltScale",        "PLTSCALE")                                      
+            sqlLine.group("posAngle",        "POSANGLE")                                      
+#            sqlLine.group("raErr",           "X_PSF_SIG * abs(PLTSCALE)") 
+#            sqlLine.group("decErr",          "Y_PSF_SIG * abs(PLTSCALE)") 
+        else:
+            sqlLine.group("pltScale",        "0.257")                                      
+            sqlLine.group("posAngle",        "-999")                                      
+#            sqlLine.group("raErr",           "X_PSF_SIG * 0.257") 
+#            sqlLine.group("decErr",          "Y_PSF_SIG * 0.257") 
+        sqlLine.group("psfFlux",         "PSF_INST_FLUX / " + extTimeString)
+        sqlLine.group("psfFluxErr",      "PSF_INST_FLUX_SIG / " + extTimeString)
+        if (self.id >= 982483):
+            sqlLine.group("psfMajorFWHM",    "PSF_FWHM_MAJ")                                               
+            sqlLine.group("psfMinorFWHM",    "PSF_FWHM_MIN")                                               
+            sqlLine.group("psfCore",         "PSF_CORE")                                                    
+        sqlLine.group("psfTheta",        "PSF_THETA")                                               
+        sqlLine.group("psfQf",           "PSF_QF")                                                  
+        sqlLine.group("psfQfPerfect",    "PSF_QF_PERFECT")                                
+        sqlLine.group("psfChiSq",        "PSF_CHISQ")                                     
+        sqlLine.group("psfLikelihood",   "psfLikelihood(EXT_NSIGMA)")                               
+        sqlLine.group("momentXX",        "MOMENTS_XX")                                              
+        sqlLine.group("momentXY",        "MOMENTS_XY")                                              
+        sqlLine.group("momentYY",        "MOMENTS_YY")                                              
+        sqlLine.group("momentR1",        "MOMENTS_R1")                                              
+        sqlLine.group("momentRH",        "MOMENTS_RH")                                              
+#        sqlLine.group("momentM3C",       "MOMENTS_M3C")                                             
+#        sqlLine.group("momentM3S",       "MOMENTS_M3S")                                             
+#        sqlLine.group("momentM4C",       "MOMENTS_M4C")                                             
+#        sqlLine.group("momentM4S",       "MOMENTS_M4S")                                             
+        if (self.id >= 982483):
+            sqlLine.group("apFlux",          "AP_FLUX / " + extTimeString)
+            sqlLine.group("apFluxErr",       "AP_FLUX_SIG / " + extTimeString)
+            sqlLine.group("apFillF",         "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
+ #       sqlLine.group("apRadius",        "AP_MAG_RADIUS")
+        sqlLine.group("kronFlux",        "KRON_FLUX / " + extTimeString)
+        sqlLine.group("kronFluxErr",     "KRON_FLUX_ERR / " + extTimeString)
+        sqlLine.group("kronRad",         "MOMENTS_R1 * 2.5")
+        sqlLine.group("diffNPos",         "DIFF_NPOS")
+        sqlLine.group("diffFRatio",         "DIFF_FRATIO")
+        sqlLine.group("diffNBad",         "DIFF_NRATIO_BAD")
+        sqlLine.group("diffNMask",         "DIFF_NRATIO_MASK")
+        sqlLine.group("diffNAll",         "DIFF_NRATIO_ALL")
+        sqlLine.group("diffPdist",         "DIFF_R_P")
+        sqlLine.group("diffNdist",         "DIFF_R_M")
+        sqlLine.group("diffPSN",         "DIFF_SN_P")
+        sqlLine.group("diffNSN",         "DIFF_SN_M")
+        sqlLine.group("sky",             "SKY  / " + extTimeString)
+        sqlLine.group("skyErr",          "SKY_SIGMA  / " + extTimeString)
+        sqlLine.group("infoFlag",        "FLAGS")                          
+        sqlLine.group("infoFlag2",       "FLAGS2")                          
+        sqlLine.group("dataRelease",     str(self.skychunk.dataRelease))                  
+
+        sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("failed sql: ", sql)
+            raise
+
+        # NOTE : Flux limits : in the current PSPS schema, negative fluxes 
+        # cause problems for sql queries which work in mags as SQL cannot do
+        # something like (f < 0.0) ? -999 : -2.5*log10(f)
+        # as a result, the negative fluxes here result in floating point errors
+        # XXX EAM 2014072 : Is this still a problem?
+ #       sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0"
+ #       self.scratchDb.execute(sql)
+        
+ #       sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0"
+ #       self.scratchDb.execute(sql)
+
+ #       sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0"
+ #       self.scratchDb.execute(sql)
+
+        # we don't delete these anymore
+        results['NULLINSTFLUX'] = 0;
+
+
+# insert stuff from dvo
+
+
+
+    '''
+    Applies indexes and other constraints to the PSPS tables
+    '''
+    def alterPspsTables(self):
+
+        self.logger.debug("Altering PSPS tables")
+        self.logger.debug("Creating indexes on PSPS tables")
+        self.scratchDb.makeColumnUnique("DiffDetection", "objID")
+        self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
+
+        self.populateDiffMeta()
+     
+        # dictionary objects to hold imageIDs for later
+        self.imageIDs = {}
+        self.sourceIDs = {}
+
+        # loop through all OTAs and populate ImageMeta extensions
+        # the column in PSPS
+        self.scratchDb.execute("ALTER TABLE DiffDetection CHANGE dec_ `dec` double")
+
+        return True
+
+
+    '''
+    Applies indexes to the IPP tables
+    '''
+    def indexIppTables(self):
+
+        self.logger.infoPair("Creating indexes on", "IPP tables")
+
+        self.scratchDb.createIndex(extension, "IPP_IDET")
+        # try the test Chip
+        self.scratchDb.createIndex("Chip_psf", "IPP_IDET")
+
+        self.logger.infoPair("created indexes on", "IPP tables")      
+
+    '''
+    Updates provided table with DVO IDs from DVO table
+    '''
+    def updateDvoIDs(self, table, externID):
+
+        imageID = self.scratchDb.getImageIDFromExternID(externID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+
+        sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \
+               a.objID        = b.objID, \
+               a.detectID     = b.detectID, \
+               a.ippObjID     = b.ippObjID, \
+               a.dvoRegionID  = b.catID, \
+               a.ra           = b.ra, \
+               a.dec          = b.dec_, \
+               a.zp           = b.zp, \
+               a.telluricExt  = b.telluricExt, \
+               a.airmass      = b.airmass, \
+               a.expTime      = b.expTime, \
+               a.infoFlag2    = (b.flags << 13) | a.infoFlag2 \
+               WHERE a.ippDetectID = b.ippDetectID \
+               AND b.imageID = " + str(imageID)
+
+        self.scratchDb.execute(sql)
+
+    '''
+    Updates table and generates pspsuniqueids
+    '''
+
+    def updatePspsUniqueIDs(self,table):
+        sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
+        (select @r:=0) r, "+table+" t) as foo using (objID) set \
+        uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql',sql)
+            return
+        
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTablesChip(self, chipname, x, y, results, tables):
+        # XXX EAM NOTE: drop tables Detection_* here so
+        # they do not polute the db?
+        # XXX or put an explicit drop at the end of the loop?
+
+        # self.logger.infoTitle("Processing " + chipname)
+        # this is a bit crude: if the chip is not present, this test will fail and the chip 
+        # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
+
+        # XXX keep this in or not?
+        # if not self.scratchDb.astrometricSolutionOK(chipname):
+        #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+        #     return False
+
+        # does this chip exist in the DVO image table (if NOT, then skip it)
+        if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
+            self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
+            return False
+
+        #self.logger.info("populate stuff ");
+        # populate remainder of tables
+        self.populateDetectionTable(chipname, results)
+        #self.logger.info("successful populate ");
+        # now add DVO IDs
+        self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
+        #self.logger.info("updated dvoids")
+        results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
+        #self.logger.info("deleted nulls")
+        self.logger.info("add psps unique p2 ids")
+        self.updatePspsUniqueIDs("Detection_" + chipname, self.imageIDs[chipname])
+        self.updateImageID("Detection_" + chipname, x, y)
+        #self.logger.info("updateImageId")
+        rowCount = self.scratchDb.getRowCount("Detection_" + chipname) 
+        #self.logger.info("got row count")
+        self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", 
+                chipname, 
+                results['ORIGINALTOTAL'], 
+                results['SATDET'], 
+                results['NULLINSTFLUX'], 
+                results['NULLOBJID'],
+                rowCount)
+        self.totalOriginal     = self.totalOriginal + results['ORIGINALTOTAL']
+        self.totalSatDet       = self.totalSatDet + results['SATDET']
+        self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX']
+        self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
+        self.totalDetections   = self.totalDetections + rowCount
+
+        fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
+        # print "kept ", str(fractionKept), " detections from ", chipname
+
+        if fractionKept < 0.8:
+            # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
+            self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
+            self.skipBatch = True
+            return False;
+
+        #self.logger.info("updated totals")
+        # check we have something in this Detection table TODO add this to table above
+        if rowCount < 1:
+            self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
+            self.skipBatch = True
+            return False;
+
+        # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
+        sql = "UPDATE ImageMeta_" + chipname + " \
+               SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
+        self.scratchDb.execute(sql)
+        
+        # add these to list of tables to export later
+        self.tablesToExport.append("ImageMeta_" + chipname)
+        self.logger.info("export ImageMeta")
+
+        self.tablesToExport.append("Detection_" + chipname)
+        self.logger.info("export Detection")
+
+        tables.append("Detection_" + chipname)
+        self.logger.info("updated detection")
+
+        self.validChips.append(chipname)
+
+        return True
+
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+
+        # loop through all OTAs again to update with DVO IDs
+        self.skipBatch = False
+        self.validChips = []    
+        self.tablesToExport = []    
+        self.tablesToExport.append("DiffMeta")
+        tables = []    
+        otaCount = 0
+        results = {}
+        self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+        self.logger.info("|  OTA  | Initial total |   Sat Det     | NULL instFlux | NULL obj ID   |  Remainder    |")
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+        for x in range(self.startX, self.endX):
+            for y in range(self.startY, self.endY):
+               
+                # dodge the corners
+                if x==0 and y==0: continue
+                if x==0 and y==7: continue
+                if x==7 and y==0: continue
+                if x==7 and y==7: continue
+
+                ota = "XY%d%d" % (x, y)
+                if ota not in self.imageIDs: continue
+
+                if False and self.config.test and not ((x == 0) and (y == 1)):
+                    print "skipping ota " + ota
+                    continue
+
+                if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
+                if self.skipBatch: 
+                    self.logger.error("fatal problem for exposure, skipping")
+                    return False
+
+        if "Chip" in self.imageIDs:
+            if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+            if self.skipBatch: 
+                self.logger.error("fatal problem for exposure, skipping")
+                return False
+
+        # print totals
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+        self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", 
+                self.totalOriginal, 
+                self.totalSatDet, 
+                self.totalNulIInstFlux, 
+                self.totalNullPeakFlux, 
+                self.totalNullObjID,
+                self.totalDetections)
+        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+")
+
+        # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)
+        if len(self.tablesToExport) == 1:
+            self.skipBatch = True
+            self.logger.error("No tables to export")
+            return False
+
+        # XXX EAM : turn this on (and fix up the code) if we need it 
+        # self.removeDuplicateObjects()
+
+        self.setMinMaxObjID(tables)
+
+        # update FrameMeta with count OTAs in this file and total number of photometric reference sources
+        sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
+        self.scratchDb.execute(sql)
+        #to make it stop
+        #test = self.scratchDb.getRowCount("Object")
+        #return False
+        
+        return True
+
+    def removeDuplicateObjects(self):
+
+        sql = "DROP TABLE SkinnyObject_All"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            print "SkinnyObject_All table not yet defined"
+
+        sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            print "failed to create table SkinnyObject_All"
+            raise
+
+        for chipname in self.validChips:
+            
+            sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \
+                   SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname
+            try:
+                self.scratchDb.execute(sql)
+            except:
+                print "failed to insert entry SkinnyObject_All: ", chipname
+                raise
+            
+
+        Nduplicates = 0
+        sql = "SELECT n, objID from (select count(objID) as n, objID from SkinnyObject_All group by objID) as tmp where n > 1"
+        try:
+            rs = self.scratchDb.executeQuery(sql)
+            while (rs.next()):
+                objID = rs.getInt(1)
+                chipID = rs.getString(3)
+                print "delete: ", str(objID), chipID
+                Nduplicates += 1
+
+            rs.close()
+        except:
+            print "failed to get duplicates"
+            raise
+
+        print "duplicates found: ", str(Nduplicates)
+
+    '''
+    Updates imageID {EXP_ID}{OTA} in the provided table
+    '''
+    def updateImageID(self, tableName, x, y):
+
+        sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
+        self.scratchDb.execute(sql)
+
+    '''
+    Overriding this method
+    '''
+    def reportNullsInAllPspsTables(self, showPartials):
+
+        # loops round all imported tables, but we want to check one OTA's worth
+        for table in self.pspsTables:
+            if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials)
+            else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials)
+
+
+    '''
+    This function reads the cmf/smf file and loads it into the database as its own table
+    Overriding this method. Filter to only import *.psf extensions
+    '''
+    def importIppTables(self, filter=""):
+        
+        regex = ".*.psf"
+        if False and self.config.test and self.config.camera == "gpc1":
+            regex = "XY01.psf"
+            
+        print "my ID: " + str(self.id)
+
+        if (self.id < 982483):
+            # XXX EAM NOTE : this is fragile : requires PS1_V4
+            columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG                   PSF_INST_FLUX PSF_INST_FLUX_SIG       PSF_MAJOR PSF_MINOR PSF_THETA          PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S                             AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
+        else:
+            # XXX EAM NOTE : this is fragile : requires PS1_V5
+            columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
+
+        return super(DetectionBatch, self).importIppTables(columns, regex)
+
+    '''
+    Overriding this method. Use regex to trim off, eg _XY33 extension
+    '''
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")

trunk/ippToPsps/jython/gpc1db.py

@@ -108 +108 @@
                    AND mjd_obs <= (to_days('" + tend + "') - 678941) "
            # self.logger.infoPair("sql",sql)      
-        elif batchType == "DF":
+        elif batchType == "DF":            # this is specifically only for WARPSTACK types of diffs -
+            # it only makes sense for the date to come from the warp1
+            # may need to be more clever if we do other types of diffs
 
             stage = "diff"
-            sql = "SELECT stuff from stufftable"
+            sql = "SELECT DISTINCT stage_extra1, radeg,decdeg \
+                   FROM addRun \
+                   JOIN addProcessedExp using (add_id) \
+                   JOIN minidvodbRun using (minidvodb_name) \
+                   JOIN minidvodbProcessed using (minidvodb_id)  \
+                   JOIN mergedvodbRun using (minidvodb_id) \
+                   JOIN mergedvodbProcessed using (merge_id) \
+                   JOIN diffInputSkyfile \
+                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
+                   AND stage = 'diff') \
+                   JOIN skycell using (skycell_id, tess_id) \
+                   JOIN warpRun on (warp1 = warp_id \
+                   AND diffInputSkyfile.skycell_id = skycell_id) \
+                   JOIN fakeRun using (fake_id) \
+                   JOIN camRun using (cam_id) \
+                   JOIN chipRun using (chip_id)  \
+                   JOIN rawExp using (exp_id) \
+                   WHERE  mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+                   AND minidvodbRun.state = 'merged' \
+                   AND minidvodbProcessed.fault = 0 \
+                   AND mergedvodbRun.state = 'full' \
+                   AND mergedvodbProcessed.fault = 0 \
+                   AND addRun.stage = '" + stage + "' \
+                   AND addRun.state = 'full' \
+                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
+                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND dateobs >= '" + tstart + "'\
+                   AND dateobs <= '" + tend  + "'"
+
 
         elif batchType == "FW":
@@ -123 +153 @@
                    JOIN mergedvodbProcessed USING (merge_id) \
                    JOIN fullForceRun USING(ff_id) \
-                   JOIN skycalRun using (skycal_id)
+                   JOIN skycalRun using (skycal_id) \
                    JOIN stackRun USING(stack_id) \
                    JOIN stackSumSkyfile using (stack_id) \
@@ -222 +252 @@
         return meta
 
+
+    '''
+    Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved
+    '''
+    def getDiffStageMeta(self, diffSkyfileID):
+
+        self.logger.debug("Querying GPC1 for stack meta data")
+
+        meta = []
+        sql = "SELECT \
+               filter,\
+               diff_mode, \
+               warp1, \
+               stack2, \
+               skycell_id, \
+               diffRun.software_ver, \
+               exp_time \
+               FROM diffRun join diffInputSkyfile using (diff_id) \
+               JOIN warpRun on (warp1=warp_id) \
+               JOIN fakeRun using (fake_id) \
+               JOIN camRun using (cam_id) \
+               JOIN chipRun using (chip_id) \
+               JOIN rawExp using (exp_id) \
+               WHERE diff_skyfile_id = %d" % diffSkyfileID
+               
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+        except:
+            self.logger.infoPair ("this sql is broken:",sql)
+            self.logger.errorPair("Can't query for", "stack meta data")
+           
+        try:
+            meta.append(rs.getString(1))
+            meta.append(rs.getString(2))
+            meta.append(rs.getString(3))
+            meta.append(rs.getString(4))
+            meta.append(rs.getString(5))
+            meta.append(rs.getString(6))
+            meta.append(rs.getString(7))
+
+        except:
+            self.logger.errorPair("getDiffStageMeta()", "empty meta data")
+     
+        return meta
+
+    '''
+    get diff stage cmf name
+    '''
+    def getDiffStageCmf(self, dvoDb, diffSkyfileID):
+        print "got here, diffstagecmf"
+        sql = "SELECT DISTINCT diffSkyfile.path_base \
+               FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \
+               JOIN addRun using (minidvodb_name) \
+               JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \
+               AND diff_id = stage_id AND stage = 'diff') \
+               JOIN diffSkyfile using (diff_id, skycell_id) \
+               WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \
+               AND minidvodbRun.state = 'merged' \
+               AND mergedvodbRun.state = 'full' \
+               AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+               AND stage = 'diff'"
+
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+        except:
+            self.logger.infoPair("failed sql", sql)
+            self.logger.errorPair("can't query for diff cmf", "dfif_skyfile_id = %d" %diff_skyfile_id)
+            return None
+        print "query worked"
+        try: 
+            pathBase = rs.getString(1)
+        except:    
+            self.logger.errorPair("no diff cms found for diff_skyfile_id = %d" %diff_skyfile_id)
+            return None
+        print pathBase
+        files = []
+        if pathBase.startswith("neb"):
+            print "open"
+            f=os.popen("neb-ls -p " + pathBase + ".cmf")
+            for i in f.readlines():
+                print "i",i
+                files.append(i.rstrip())
+                print "append"
+        print i
+        if len(files) < 1: return None
+        fits = Fits(self.logger, self.config, files[0])
+        return fits
+
     '''
     Gets some camera-stage meta data for this cam_id
@@ -394 +514 @@
     def getStackStageCmf(self, dvoDb, stackID):
      
-        '''
-        sql = "SELECT DISTINCT staticskyResult.path_base \
-               FROM staticskyRun  \
-               JOIN staticskyInput USING(sky_id)  \
-               JOIN staticskyResult USING(sky_id) \
-               JOIN stackRun USING(stack_id)  \
-               JOIN stackSumSkyfile USING(stack_id) \
-               JOIN addRun ON sky_id = stage_id \
-               JOIN minidvodbRun USING (minidvodb_name) \
-               WHERE stack_id = " + str(stackID) + " \
-               AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
-               AND minidvodbRun.state = 'merged' \
-               AND stage = 'staticsky'"
-        '''
-
         sql = "SELECT DISTINCT skycalResult.path_base \
                FROM skycalRun  \
@@ -458 +563 @@
         fits = Fits(self.logger, self.config, files[0])
         return fits
-
-        # XXX validate the file?
-        # if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-
-            # now find and loop through all cmf files at this path_base
-
-            ## XXX the code below was needed when we ingested from staticsky (N files per run)
-            ## XXX ingesting from skycal solves this problem (1 file per run)
-            ## files=os.popen("neb-ls " + pathBase + "%cmf")
-            ## for i in files.readlines():
-            ##     nebPath = i.rstrip()
-            ## 
-            ##     # now attempt to run 'neb-ls -p' to actually read this cmf file
-            ##     try:
-            ##         self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
-            ##         paths=os.popen("neb-ls -p " + nebPath)
-            ##         path = paths.readline().rstrip()
-            ##         if len(path) < 1:
-            ##            self.logger.debug("zero length path - skipping")
-            ##            raise
-            ## 
-            ##         print "starint stack stage cmf 3"
-            ## 
-            ##         # if we get here, then the cmf is readable, now check the stack_id
-            ##         fits = Fits(self.logger, self.config, path)
-            ##         if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-            ##             # we have the right file!
-            ##             self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
-            ##             return fits
-            ##         else:
-            ##             # this is not the correct file
-            ##             self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
-            ## 
-            ##     # an exception here means that nebulous could not read this file, so we just skip it
-            ##     except:
-            ##         self.logger.debug("NEB FAILED SKIPPING")
-            ##         pass
 
     '''

trunk/ippToPsps/jython/initbatch.py

@@ -16 +16 @@
 from stackbatch import StackBatch
 from detectionbatch import DetectionBatch
+#from diffbatch import DiffBatch
+#from forcedwarpbatch import ForcedWarpBatch
+#why no object stuffs
 
 '''

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -799 +799 @@
     def insertDiffMeta(self, batchID, filter, diffType):
 
-        sql = "INSERT INTO stack ( \
+        sql = "INSERT INTO diff ( \
                batch_id \
                ,filter \

trunk/ippToPsps/jython/loader.py

@@ -24 +24 @@
 from objectbatch import ObjectBatch
 
+from diffbatch import DiffBatch 
+#from diffobjectbatch import DiffObjectBatch
+
+#
+#
 '''
 Loader class
@@ -191 +196 @@
         # loop round IDs of all items to be processed
         self.ippToPspsDb.lockBatchTable()
-        #self.logger.infoPair("heather:","lockbatchtable")
+        self.logger.infoPair("heather:","lockbatchtable")
         unattemptedCount = 0
         for id in ids:
-            #self.logger.infoPair("heather:","in ids")
+            self.logger.infoPair("heather:","in ids")
             #self.logger.infoPair("heather:id",str(id))
             batchID = self.ippToPspsDb.createNewBatch(batchType, id)
@@ -202 +207 @@
                 unattemptedCount += 1
                 continue
-            #self.logger.infoPair("heather:","passed logic")
+            self.logger.infoPair("heather:","passed logic")
 
             self.ippToPspsDb.unlockTables()
@@ -244 +249 @@
     
                 elif batchType == "DF":
+                    self.logger.infoPair("wegothere!","we are awesome")
                     batch = DiffBatch(self.logger,
                             self.config,

trunk/ippToPsps/jython/objectbatch.py

@@ -94 +94 @@
         sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \
         (select @r:=0) r, ObjectThin t) as foo using (objID) set \
-        uniquePspsP2id = ((" +str(self.batchID) "*1000000000 ) + rownum)";
+        uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)";
         try: self.scratchDb.execute(sql)
         except:

trunk/ippToPsps/jython/stackbatch.py

@@ -955 +955 @@
         (select @r:=0) r, StackObjectThin t) as foo using (objID) set \
         stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \
-        uniquePspsSTid = ((" +str(self.batchID) "*1000000000 ) + rownum)";
+        uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)";
         try: self.scratchDb.execute(sql)
         except:

trunk/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta

@@ -9 +9 @@
 photoCalID   dimensionless   INT 4 NA photometry code
 magSat       dimensionless   REAL 4 -999 saturation magnitude level
-analVer      dimensionless   SMALLINT 2 -999 analysis version index
+analVer      alphanumeric   VARCHAR(100) 100 -999 analysis version index
 expTime      seconds         REAL 4 -999 exposure time of positive image
 ctype1       alphanumeric   VARCHAR(MAX) 8000 name of astrometric projection in RA