Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py	(revision 31587)
@@ -32,5 +32,17 @@
     "../config/2/tables.vot"
     '''
-    def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 id,
+                 batchType, 
+                 inputFitsPath="", 
+                 survey="", 
+                 useFullTables=False):
+
+        self.everythingOK = False
+        self.readHeader = False
 
         # set up logging
@@ -40,4 +52,8 @@
 
         # set up class variables
+        self.id = id
+        self.gpc1Db = gpc1Db
+        self.ippToPspsDb = ippToPspsDb
+        self.scratchDb = scratchDb
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
@@ -46,4 +62,11 @@
         self.useFullTables = useFullTables
 
+        if self.alreadyProcessed(): return
+
+        # do we have an input file?
+        if self.inputFitsPath != "":
+
+            if not self.readPrimaryHeader(): return
+
         # TODO
         self.tablesToExport = []
@@ -51,9 +74,4 @@
         # open config
         doc = ElementTree(file="config.xml")
-
-        # create Gpc1Db object
-        self.gpc1Db = Gpc1Db(self.logger)
-        self.ippToPspsDb = IppToPspsDb(logger)
-        self.scratchDb = ScratchDb(logger, self.useFullTables)
 
         if self.survey != "":
@@ -88,13 +106,9 @@
         self.dateStr = now.strftime("%Y-%m-%d")
 
-        if self.inputFitsPath != "": 
-            file = open(self.inputFitsPath)
-            self.header = self.parseFitsHeader(file)
-            self.logger.info("Read primary and found " + str(len(self.header)) + " header cards") 
-            # TODO close file?
-
         # create DVO tables if accessing DVO directly
         if not self.useFullTables: self.scratchDb.createDvoTables()
 
+        self.everythingOK = True
+
     '''
     Destructor
@@ -103,4 +117,27 @@
 
         self.logger.debug("Batch destructor")
+
+
+    '''
+    Reads the primary header of the FITS file
+    '''
+    def readPrimaryHeader(self):
+
+        if self.readHeader: return True
+
+        # does it exist?
+        if not os.path.isfile(self.inputFitsPath):
+
+            self.logger.error("Cannot read file at '" + self.inputFitsPath + "'")
+            return False
+
+        file = open(self.inputFitsPath)
+        self.header = self.parseFitsHeader(file)
+        self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards")
+        # TODO close file?
+
+        self.readHeader = True
+
+        return True
 
 
@@ -138,6 +175,8 @@
             file.seek(index + 2880, 0)
             
-        if found != True: self.logger.error("...could not find extension '" + name + "'")
-        else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
+        if found != True: 
+            self.logger.error("...could not read header in extension '" + name + "'")
+            return
+        #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
 
         return header
@@ -262,9 +301,13 @@
 
         first = True
+
+        self.totalDetections = 0
         for table in tables:
 
-            sql = "SELECT MIN(objID), MAX(objID) FROM " + table
-            rs = self.scratchDb.stmt.executeQuery(sql)
+            sql = "SELECT MIN(objID), MAX(objID), COUNT(objID) FROM " + table
+            rs = self.scratchDb.executeQuery(sql)
             rs.first()
+
+            self.totalDetections = self.totalDetections + rs.getLong(3)
 
             if first:
@@ -276,6 +319,9 @@
 
             first = False
+            rs.close()
 
         self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
+        self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID))
+
 
     '''
@@ -313,5 +359,5 @@
          self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
          for table in self.pspsTables:
-             self.logger.info("Creating PSPS table: " + table.name)
+             self.logger.debug("Creating PSPS table: " + table.name)
              table.write(self.scratchDb.url + '#' + table.name)
              self.tablesToExport.append(table.name)
@@ -337,5 +383,5 @@
     Accepts a regular expression filter so not all tables need to be imported
     '''
-    def importIppTables(self, filter):
+    def importIppTables(self, filter=""):
 
       self.logger.info("Attempting to import tables from input FITS file")
@@ -347,12 +393,12 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.logger.info("   Reading IPP table " + table.name + " from FITS file")
+          self.logger.info("Reading IPP table " + table.name + " from FITS file")
           table = stilts.tpipe(table, cmd='explodeall')
 
           # drop any previous tables before import
-          self.scratchDb.dropTable(table.name)
+          #self.scratchDb.dropTable(table.name)
 
           # IPP FITS files are littered with infinities, so remove these
-          self.logger.info("   Removing Infinity values from all columns")
+          self.logger.debug("Removing Infinity values from all columns")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
@@ -360,10 +406,11 @@
           try:
               table.write(self.scratchDb.url + '#' + table.name)
+              self.scratchDb.killLastConnectionID()
+              count = count + 1
           except:
-              self.logger.exception("   Problem writing table '" + table.name + "' to the database")
-          count = count + 1
+              self.logger.exception("Problem writing table '" + table.name + "' to the database")
+
 
       self.logger.info("Done. Imported %d tables" % count)
-
       self.indexIppTables()
 
@@ -373,8 +420,8 @@
     def exportPspsTablesToFits(self, regex="(.*)"):
 
-        self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS")
+        self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex)
         _tables = []
 
-        self.logger.info("    Selecting database tables")
+        self.logger.info("Selecting database tables")
         for table in self.tablesToExport:
 
@@ -383,6 +430,11 @@
 
            # get everything from table
-           _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
-
+           try:
+               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
+               self.scratchDb.killLastConnectionID()
+           except:
+               self.logger.exception("Could not read from DB table: " + table)
+               return False
+               
            # replace nulls and empty fields with weird PSPS -999 pseudo-null
            _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
@@ -395,8 +447,13 @@
            _tables.append(_table)
 
-        self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
-        stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
-        self.logger.info("    ...done")
-        self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...")
+        try:
+            stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
+            self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        except:
+            self.logger.exception("Could not write to FITS")
+            return False
+
+        return True
 
     '''
@@ -447,6 +504,24 @@
     '''
     def alreadyProcessed(self):
-           self.logger.info("Not implemented")
-
-
-
+        self.logger.info("Not implemented")
+
+
+    '''
+    Creates and publishes a batch
+    '''
+    def run(self):
+
+        if not self.everythingOK: return
+
+        self.createEmptyPspsTables()
+        self.importIppTables()
+        if self.populatePspsTables():
+            if self.exportPspsTablesToFits():
+                self.writeBatchManifest()
+                self.createTarball()
+                self.publishToDatastore()
+                #self.reportNullsInAllPspsTables(False)
+                #sys.exit()
+        self.logger.info("Finished.")
+
+
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py	(revision 31587)
@@ -6,6 +6,9 @@
 from java.lang import *
 from java.sql import *
+
 from batch import Batch
 from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
 
 import logging.config
@@ -19,22 +22,35 @@
     Constructor
     '''
-    def __init__(self, logger, camID, inputFile, test=False, useFullTables=False):
+    def __init__(self, 
+                 logger,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 camID, 
+                 inputFile, 
+                 test=False, 
+                 useFullTables=False):
+
        super(DetectionBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               camID,
                "detection", 
                inputFile, 
-               "MD04",
-               useFullTables) # TODO
+               "MD04", # TODO
                #"3PI") # TODO
-
-       self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       meta = self.gpc1Db.getCameraStageMeta(camID)
-      
-       self.expID = meta[0];
-       self.expName = meta[1];
-       self.distGroup = meta[2];
-
-       self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup))
+               useFullTables)
+
+       if not self.everythingOK: return
+
+       # meta data to the log
+       self.logger.info("New Detection Batch:")
+       self.logger.info("Cam ID:             %d" % self.id)
+       self.logger.info("file:               %s" % inputFile)
+       self.logger.info("Exp ID:             %d" % self.expID)
+       self.logger.info("Exp name:           %s" % self.expName)
+       self.logger.info("Distribution group: %s" % self.distGroup)
 
        # create an output filename, which is {expID}.FITS
@@ -167,5 +183,5 @@
         ," + self.header['PCA2X0Y2'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
@@ -184,5 +200,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ImageMeta"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -310,9 +326,9 @@
                )"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateFilterID(tableName, self.filter)
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
-        self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
+        if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
 
@@ -327,5 +343,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE Detection"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -375,10 +391,9 @@
                ,EXT_NSIGMA \
                FROM " + ota + "_psf"
-
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # set obsTime
         sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
         self.scratchDb.updateAllRows(tableName, "historyModNum", "0")
@@ -387,5 +402,5 @@
         self.scratchDb.updateFilterID(tableName, self.filter)
 
-        # now delete bad flux
+        # now delete bad flux and bad chip positions
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU")
@@ -401,5 +416,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -415,5 +430,5 @@
                ,surveyID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
@@ -429,5 +444,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -443,5 +458,5 @@
                ,filterID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
@@ -484,5 +499,5 @@
 
         imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
-        self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -494,5 +509,5 @@
                AND b.imageID = " + str(imageID)
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -511,4 +526,5 @@
 
         # loop through all OTAs and populate ImageMeta extensions
+        self.logger.info("Reading all fits headers and populating ImageMeta tables")
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
@@ -524,4 +540,8 @@
                 # load corresponding header into memory
                 header = self.findAndReadFITSHeader(ota + ".hdr", file)
+                if not header:
+                    self.logger.error("No header found for OTA " + ota)
+                    continue
+
 
                 # store sourceID/imageID combo in Db so DVO can look up later
@@ -578,6 +598,7 @@
 
                 # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
-                self.scratchDb.stmt.execute(sql)
+                sql = "UPDATE ImageMeta_" + ota + " \
+                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
+                self.scratchDb.execute(sql)
 
                 self.populateSkinnyObjectTable(ota)
@@ -603,5 +624,5 @@
         # update FrameMeta with count OTAs in this file and total number of photometric reference sources
         sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         
         return True
@@ -613,5 +634,5 @@
 
         sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -619,4 +640,9 @@
     '''
     def alreadyProcessed(self):
+
+        meta = self.gpc1Db.getCameraStageMeta(self.id)
+        self.expID = meta[0];
+        self.expName = meta[1];
+        self.distGroup = meta[2];
 
         return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID)
@@ -634,9 +660,31 @@
 
 
+    '''
+    Overriding this method. Filter to only import *.psf extensions
+    '''
+    def importIppTables(self, filter=""):
+       return super(DetectionBatch, self).importIppTables(".*.psf")
+
+
+    '''
+    Overriding this method. Use regex to trim off, eg _XY33 extension
+    '''
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+
+
+# TODO put in config
+useFullTables=True
+testMode=False
+
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("detectionbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
 
 gpc1Db = Gpc1Db(logger)
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
 camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam")
 logger.info("Found %d exposures" % len(camIDs))
@@ -645,25 +693,16 @@
 for camID in camIDs:
 
-    logger.info("-------------------------------------------------- cam ID: %d" % camID)
+    #if camID < 43764: continue # TODO 
 
     file = gpc1Db.getCameraStageSmf(camID)
-    if not os.path.isfile(file):
-        logger.error("Cannot read file at '" + file)
-        continue
-
-    detectionBatch = DetectionBatch(logger, camID, file, False, True)
-
-    if not detectionBatch.alreadyProcessed():
-
-        detectionBatch.createEmptyPspsTables()
-        detectionBatch.importIppTables(".*.psf")
-        if detectionBatch.populatePspsTables():
-            detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)")
-            detectionBatch.writeBatchManifest()
-            #detectionBatch.reportNullsInAllPspsTables(False)
-            #detectionBatch.createTarball()
-            #detectionBatch.publishToDatastore()
-    
-            i = i+1
-           # if i > 0: sys.exit()
-
+
+    detectionBatch = DetectionBatch(logger,
+                                    gpc1Db,
+                                    ippToPspsDb,
+                                    scratchDb,
+                                    camID, 
+                                    file, 
+                                    testMode, 
+                                    useFullTables)
+    detectionBatch.run()
+
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py	(revision 31587)
@@ -29,4 +29,33 @@
         self.logger.debug("Gpc1Db destructor")
 
+
+    '''
+    TODO
+    '''
+    def getIDsInThisDVODbForThisStageFudge(self):
+
+        sql = "SELECT staticskyRun.sky_id \
+               FROM staticskyInput, staticskyRun, stackRun, staticskyResult \
+               WHERE staticskyRun.sky_id = staticskyInput.sky_id \
+               AND staticskyInput.stack_id = stackRun.stack_id \
+               AND staticskyInput.sky_id = staticskyResult.sky_id \
+               AND staticskyRun.label like 'MD04.staticsky' \
+               AND stackRun.filter like 'i%'"
+
+        try:
+            rs = self.executeQuery(sql)
+        except:
+            self.logger.exception("Can't query for ids in DVO")
+
+        ids = []
+        while (rs.next()):
+            ids.append(rs.getInt(1))
+
+        rs.close()
+
+        self.logger.info("Found %d items in DVO database '" % (len(ids)))
+
+        return ids
+
     '''
     Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far)
@@ -40,5 +69,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for ids in DVO")
@@ -50,5 +79,5 @@
         rs.close()
 
-        self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
+        self.logger.info("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
 
         return ids
@@ -73,5 +102,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for imageIDs")
@@ -105,5 +134,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -113,4 +142,5 @@
 
         return meta
+
     '''
     Gets some camera-stage meta data for this cam_id
@@ -127,5 +157,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -150,5 +180,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -171,4 +201,6 @@
             files = glob.glob(path + "/*.cmf")
 
+        if len(files) < 1: return "NULL"
+
         return files[0] # TODO just returning first file - check
 
@@ -186,5 +218,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -200,8 +232,6 @@
 
             f=os.popen("neb-ls -p "+path+"%cmf")
-            print "neb-ls -p "+path+"%cmf"
             for i in f.readlines():
                 files.append(i.rstrip())
-                print i.rstrip()
 
         # or not a neb path
@@ -211,2 +241,32 @@
         return files
 
+
+    '''
+    TODO hack to get exposure time for a stack
+    '''
+    def getStackExpTime(self, stackID):
+
+        self.logger.debug("Querying GPC1 for stack exposure time")
+
+        sql = "SELECT SUM(exp_time) * (COUNT(warp_id) - reject_images) / COUNT(warp_id) as EXPTIME \
+               FROM staticskyRun JOIN staticskyInput using(sky_id) \
+               JOIN stackRun using(stack_id) \
+               JOIN stackSumSkyfile using(stack_id) \
+               JOIN stackInputSkyfile using(stack_id) \
+               JOIN warpRun using(warp_id) \
+               JOIN fakeRun using(fake_id) \
+               JOIN camRun using(cam_id) \
+               JOIN chipRun using(chip_id) \
+               JOIN rawExp using(exp_id) \
+               WHERE stack_id = %d" % stackID
+
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            return rs.getInt(1)
+        except:
+            self.logger.exception("Can't query for exposure time")
+
+        return 0.0
+
+
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py	(revision 31587)
@@ -37,5 +37,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
         sql = "SELECT MAX(batch_id) FROM batch"
@@ -44,5 +44,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             batchID = rs.getInt(1)
@@ -64,5 +64,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -75,5 +75,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -86,5 +86,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -101,5 +101,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0: 
@@ -128,5 +128,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -149,5 +149,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
 
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py	(revision 31587)
@@ -37,5 +37,15 @@
         self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass
         self.con = DriverManager.getConnection(self.url)
-        self.stmt = self.con.createStatement()
+        self.connectionID = self.getLastConnectionID()
+        self.logger.info("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
+
+        #self.stmt = self.con.createStatement()
+
+
+    '''
+    Disconnect from database
+    '''
+    def disconnect(self):
+        self.con.close()
 
     '''
@@ -45,6 +55,31 @@
 
         self.logger.debug("MySql destructor")
-        self.stmt.close()
-        self.con.close()
+        self.disconnect()
+
+    '''
+    Kills the last connection ID, so long as it's not THIS connection ID
+    '''
+    def killLastConnectionID(self):
+   
+        connectionID = self.getLastConnectionID()
+        if connectionID == self.connectionID:
+            self.logger.error("NOT going to kill THIS connection ID")
+            return
+
+        sql = "KILL %d" % connectionID
+        self.execute(sql)
+
+    '''
+    Gets the last connection ID
+    '''
+    def getLastConnectionID(self):
+
+        sql = "SELECT ID \
+               FROM INFORMATION_SCHEMA.PROCESSLIST \
+               WHERE DB='" + self.dbName + "' \
+               ORDER BY ID"
+        rs = self.executeQuery(sql)
+        rs.last()
+        return rs.getInt(1)
 
     '''
@@ -54,5 +89,5 @@
 
         sql = "UPDATE " + table + " SET " + column + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -62,5 +97,5 @@
 
         sql = "DROP TABLE " + table
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: return
 
@@ -74,5 +109,5 @@
         sql = "ALTER TABLE " + table + " ADD UNIQUE (" + column + ")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
@@ -82,11 +117,29 @@
     def createIndex(self, table, column):
 
-        self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
+        #self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
 
         sql = "CREATE INDEX "+table+"_"+column+"_index ON "+table+" ("+column+")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
+    '''
+    TODO
+    '''
+    def execute(self, sql):
+
+        stmt = self.con.createStatement()
+        stmt.execute(sql)
+        stmt.close()
+
+    '''
+    TODO
+    '''
+    def executeQuery(self, sql):
+
+        stmt = self.con.createStatement()
+        rs = stmt.executeQuery(sql)
+        #stmt.close()
+        return rs
 
     '''
@@ -96,5 +149,5 @@
 
        sql = "SHOW COLUMNS FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        columns = []
        while (rs.next()): columns.append(rs.getString(1))
@@ -109,5 +162,5 @@
 
       sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-      self.stmt.execute(sql)
+      self.execute(sql)
 
     '''
@@ -123,5 +176,5 @@
           
           sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-          self.stmt.execute(sql)
+          self.execute(sql)
 
     '''
@@ -131,11 +184,11 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " = " + value
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -145,11 +198,11 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " IS NULL"
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -160,5 +213,5 @@
        # first, count rows
        sql = "SELECT COUNT(*) FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        rs.first()
        numRows = rs.getInt(1)
@@ -175,5 +228,5 @@
           
           sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL"
-          rs = self.stmt.executeQuery(sql)
+          rs = self.executeQuery(sql)
           rs.first()
           if rs.getInt(1) == numRows:
@@ -191,5 +244,5 @@
         sql = "SELECT COUNT(*) FROM " + table
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             return rs.getInt(1)
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py	(revision 31587)
@@ -45,5 +45,5 @@
         sql = "SELECT surveyID FROM Survey WHERE name = '" + name + "'"
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             return rs.getInt(1)
@@ -61,5 +61,5 @@
         sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             flags = rs.getInt(1)
@@ -78,5 +78,5 @@
         sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             imageID = rs.getInt(1)
@@ -95,5 +95,5 @@
         sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             photcode = rs.getInt(1)
@@ -109,5 +109,5 @@
 
         sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + filter + "'"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -123,5 +123,5 @@
                " + str(imageID) + "    \
                )"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -131,5 +131,5 @@
 
         sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')"
-        self.stmt.execute(sql)
+        self.execute(sql)
         
     '''
@@ -141,5 +141,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0:
@@ -160,9 +160,9 @@
 
         sql = "DROP TABLE dvoMeta"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
         
         sql = "DROP TABLE dvoDetection"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
 
@@ -175,5 +175,5 @@
                )"
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO meta-data database table")
@@ -193,5 +193,5 @@
                #INDEX (ippDetectID) \
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO detection database table")
Index: branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py
===================================================================
--- branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py	(revision 31458)
+++ branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py	(revision 31587)
@@ -8,6 +8,9 @@
 from java.sql import *
 
+from batch import Batch
 from gpc1db import Gpc1Db
-from batch import Batch
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+
 import logging.config
 
@@ -20,7 +23,20 @@
     Constructor
     '''
-    def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 skyID, 
+                 inputFile, 
+                 stackType, 
+                 useFullTables=False):
+
        super(StackBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               skyID,
                "stack", 
                inputFile, 
@@ -28,24 +44,18 @@
                useFullTables) # TODO
 
-       self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       self.skyID = skyID
-
-       # get filterID using init table
-       self.filter = self.header['FPA.FILTER']
-       self.filter = self.filter[0:1]
-
-       self.stackType = stackType
-       meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER'])
-       if len(meta) < 1: return
-       self.stackID = meta[0];
-       self.skycell = meta[1];
-
-       # determine skycell from header value
-       #self.skycell = "skycell.34" #= self.header['SKYCELL']
-       self.skycell = self.skycell[8:]
-
-       self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter))
-
+       if not self.everythingOK: return
+
+       self.expTime = gpc1Db.getStackExpTime(self.stackID)
+
+       self.logger.info("got exp time of %d" % self.expTime)
+
+       # meta data to the log
+       self.logger.info("New Stack Batch:")
+       self.logger.info("Sky ID:     %d" % self.id)
+       self.logger.info("File:       %s" % inputFile)
+       self.logger.info("Stack ID:   %d" % self.stackID)
+       self.logger.info("Stack type: %s" % self.stackType)
+       self.logger.info("Skycell:    %s" % self.skycell)
+       self.logger.info("Filter:     %s" % self.filter)
 
        # delete PSPS tables
@@ -58,16 +68,6 @@
        self.scratchDb.dropTable("ObjectCalColor")
 
-       # delete IPP tables
-       #self.scratchDb.dropTable("SkyChip_psf")
-       #self.scratchDb.dropTable("SkyChip_xsrc")
-       #self.scratchDb.dropTable("SkyChip_xfit")
-       #self.scratchDb.dropTable("SkyChip_xrad")
-
-       self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType))
-       # obs time makes no sense except for nightly stacks
-       #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999"
-
        # create an output filename, which is {filterID}{skycellID}.FITS
-       self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell))
+       self.outputFitsFile = "%08d.FITS" % self.stackID
        self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
 
@@ -77,5 +77,5 @@
 
        # insert what we know about this stack batch into the stack table
-       self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType)
+       self.ippToPspsDb.insertStackMeta(self.batchID, self.id, self.stackID, self.filter, self.stackType)
 
        # insert sourceID/imageID combo so DVO can look it up
@@ -89,5 +89,5 @@
 
         sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.stackID)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -97,5 +97,5 @@
 
         sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -148,5 +148,5 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -196,5 +196,5 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # sersic fit has an extra parameter
@@ -213,5 +213,5 @@
             WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
 
@@ -220,4 +220,5 @@
     '''
     def populateStackMeta(self):
+
         self.logger.info("Procesing StackMeta table")
 
@@ -246,5 +247,5 @@
         ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \
         ," + self.header['FPA.ZP'] + " \
-        ," + self.header['EXPTIME'] + " \
+        ," + str(self.expTime) + " \
         ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \
         ,'" + self.header['CTYPE1'] + "' \
@@ -261,5 +262,5 @@
         ," + self.header['PC002002'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
@@ -272,4 +273,5 @@
     '''
     def populateStackDetection(self):
+
         self.logger.info("Procesing StackDetection table")
 
@@ -321,7 +323,7 @@
                ,X_PSF_SIG \
                ,Y_PSF_SIG \
-               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \
-               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \
-               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \
+               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+" \
+               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+")) / 1.085736) \
+               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+str(self.expTime)+" \
                ,SKY \
                ,SKY_SIGMA \
@@ -352,5 +354,5 @@
                FROM SkyChip_psf"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
@@ -362,6 +364,17 @@
         self.updateStackTypeID("StackDetection")
         self.updateDvoIDs("StackDetection")
-
-        # now delete bad flux
+        sql = "ALTER TABLE StackDetection ADD PRIMARY KEY (objID, stackDetectID)"
+        self.scratchDb.execute(sql)
+
+        if self.stackType == "DEEP_STACK": 
+
+            #if deep stack and instFlux = null and err not null
+            sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \
+                   SET instFlux = 2*b.PSF_INST_FLUX_SIG \
+                   WHERE instFlux IS NULL \
+                   AND b.PSF_INST_FLUX_SIG IS NOT NULL"
+            self.scratchDb.execute(sql)
+            #    instFlux = 2*PSF_INST_FLUX_SIG
+            
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
@@ -372,4 +385,5 @@
     '''
     def populateStackApFlx(self):
+
         self.logger.info("Procesing StackApFlx table")
  
@@ -381,16 +395,16 @@
 
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except: return
 
         # TODO temporarily loading 1st convolved fluxes into unconvolved fields
-        self.logger.info("    Adding un-convolved fluxes")
+        self.logger.info("Adding un-convolved fluxes")
         self.updateApFlxs("", "< 7.0")
-        self.logger.info("    Adding 1st convolved fluxes")
+        self.logger.info("Adding 1st convolved fluxes")
         self.updateApFlxs("c1", "< 7.0")
-        self.logger.info("    Adding 2nd convolved fluxes")
+        self.logger.info("Adding 2nd convolved fluxes")
         self.updateApFlxs("c2", "> 7.0")
 
-        self.logger.info("    Adding petrosians for extended sources")
+        self.logger.info("Adding petrosians for extended sources")
         sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \
         petRadius=b.PETRO_RADIUS \
@@ -403,5 +417,5 @@
         ,petR90Err=b.PETRO_RADIUS_90_ERR \
         WHERE a.ippDetectID=b.IPP_IDET"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID))
@@ -413,4 +427,7 @@
         self.updateStackTypeID("StackApFlx")
         self.updateDvoIDs("StackApFlx")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackApFlx", "objID")
+        self.deleteDetectionsNotInStackDetection("StackApFlx")
+
 
     '''
@@ -418,4 +435,5 @@
     '''
     def populateStackModelFit(self):
+
         self.logger.info("Procesing StackModelFit table")
 
@@ -423,15 +441,14 @@
         sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit"
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except:
             return
 
-
         # populate model parameters
-        self.logger.info("    Adding deVaucouleurs fit")
+        self.logger.info("Adding deVaucouleurs fit")
         self.updateModelFit("deV", "PS_MODEL_DEV")
-        self.logger.info("    Adding exponential fit")
+        self.logger.info("Adding exponential fit")
         self.updateModelFit("exp", "PS_MODEL_EXP")
-        self.logger.info("    Adding sersic fit")
+        self.logger.info("Adding sersic fit")
         self.updateModelFit("ser", "PS_MODEL_SERSIC")
 
@@ -444,4 +461,23 @@
         self.updateStackTypeID("StackModelFit")
         self.updateDvoIDs("StackModelFit")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackModelFit", "objID")
+        self.deleteDetectionsNotInStackDetection("StackModelFit")
+
+    '''
+    Reports and deletes detections in this table that are not in StackDetection
+    '''
+    def deleteDetectionsNotInStackDetection(self, table):
+
+        sql = "SELECT COUNT(*) FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        rs = self.scratchDb.executeQuery(sql)
+        rs.first()
+        nMissing = rs.getInt(1)
+        self.logger.info("%5d detections in %s table that are not in StackDetection. Deleting" % (nMissing, table))
+  
+        if nMissing < 1: return
+        
+        sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        self.scratchDb.execute(sql)
+       
 
     '''
@@ -449,4 +485,5 @@
     '''
     def populateStackToImage(self):
+
         self.logger.info("Procesing StackToImage table")
 
@@ -457,9 +494,9 @@
                    VALUES (\
                    " + str(self.stackID) + ", " + imageID + ")"
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
         # now update StackMeta with correct number of inputs
         sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -467,4 +504,5 @@
     '''
     def populateSkinnyObject(self):
+
         self.logger.info("Procesing SkinnyObject table")
 
@@ -474,8 +512,8 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID))
@@ -486,4 +524,5 @@
     '''
     def populateObjectCalColor(self):
+
         self.logger.info("Procesing ObjectCalColor table")
 
@@ -493,8 +532,8 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateFilterID("ObjectCalColor", self.filter)
@@ -508,5 +547,5 @@
 
         self.logger.info("Altering PSPS tables")
-        self.scratchDb.makeColumnUnique("StackDetection", "objID")
+        #self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
         self.scratchDb.createIndex("StackApFlx", "ippDetectID")
@@ -531,5 +570,5 @@
         imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID'])
 
-        self.logger.info("Updating table '" + table + "' with DVO IDs...")
+        self.logger.debug("Updating table '" + table + "' with DVO IDs...")
         sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -539,6 +578,5 @@
                AND b.sourceID = " + self.header['SOURCEID'] + "\
                AND b.imageID = " + str(imageID)
-        self.scratchDb.stmt.execute(sql)
-
+        self.scratchDb.execute(sql)
 
     '''
@@ -553,6 +591,9 @@
         self.populateStackMeta()
         self.populateStackDetection()
-        self.populateStackModelFit()
-        self.populateStackApFlx()
+
+        if self.stackType != "NIGHTLY_STACK": 
+            self.populateStackModelFit()
+            self.populateStackApFlx()
+   
         self.populateStackToImage()
         self.populateSkinnyObject()
@@ -560,5 +601,10 @@
 
         self.setMinMaxObjID(["StackDetection"])
-        
+       
+        if self.totalDetections < 1: 
+
+            self.logger.error("No detections to publish")
+            return False
+
         return True
 
@@ -568,22 +614,49 @@
     def alreadyProcessed(self):
 
-        return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        # sadly, we have to read the FITS primary header first
+        if not self.readPrimaryHeader(): return False
+
+        # get filterID using init table
+        self.filter = self.header['FPA.FILTER']
+        self.filter = self.filter[0:1]
+
+        self.stackType = stackType
+        meta = self.gpc1Db.getStackStageMeta(self.id, self.header['FPA.FILTER'])
+        if len(meta) < 1: return False
+        self.stackID = meta[0];
+        self.skycell = meta[1];
+        self.skycell = self.skycell[8:]
+
+        #return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        return False # TODOI
+
+
+useFullTables=True
 
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("stackbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
+
 gpc1Db = Gpc1Db(logger)
-stackType = "NIGHTLY_STACK"
-skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
-#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
-#stackType = "DEEP_STACK"
-#skyIDs = [689]
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
+#stackType = "NIGHTLY_STACK"
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStageFudge()
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
+
+stackType = "DEEP_STACK"
+skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
+
+#skyIDs = [942]
 #skyIDs = [299]
 #skyIDs = [302]
 #skyIDs = [8508]
-i = 0
+#i = 0
 for skyID in skyIDs:
-
-    logger.info("-------------------------------------------------- sky ID: %d" % skyID)
+    
+    #if skyID < 1340: continue # nightly
+    #if skyID < 238: continue # deep
 
     cmfFiles = gpc1Db.getStackStageCmfs(skyID)
@@ -591,24 +664,13 @@
     for file in cmfFiles:
 
-        if not os.path.isfile(file):
-            logger.error("Cannot read file at '" + file)
-            continue
-
-        stackBatch = StackBatch(logger, skyID, file, stackType, True)
-
-        if not stackBatch.alreadyProcessed():
-
-            stackBatch.createEmptyPspsTables()
-            stackBatch.importIppTables("")
-            if stackBatch.populatePspsTables():
- 
-                #stackBatch.reportNullsInAllPspsTables(False)
-                stackBatch.exportPspsTablesToFits()
-                stackBatch.writeBatchManifest()
-                #stackBatch.createTarball()
-                #stackBatch.publishToDatastore()
-
-                i = i + 1
-                #if i > 0: sys.exit()
-
-logger.info("Finished")
+        stackBatch = StackBatch(logger,
+                                gpc1Db,
+                                ippToPspsDb,
+                                scratchDb,
+                                skyID, 
+                                file, 
+                                stackType, 
+                                useFullTables)
+
+        stackBatch.run()
+