Index: trunk/ippToPsps/jython/detectionbatch.py
===================================================================
--- trunk/ippToPsps/jython/detectionbatch.py	(revision 34879)
+++ trunk/ippToPsps/jython/detectionbatch.py	(revision 35097)
@@ -3,6 +3,8 @@
 import os.path
 import sys
+import glob
 import time
 import stilts
+
 from java.lang import *
 from java.sql import *
@@ -34,4 +36,5 @@
                  logger,
                  config,
+                 skychunk,
                  gpc1Db,
                  ippToPspsDb,
@@ -44,4 +47,5 @@
                logger,
                config,
+               skychunk,
                gpc1Db,
                ippToPspsDb,
@@ -72,15 +76,8 @@
        self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
 
-       # if test mode
-       if self.config.test:
-           self.startX = 3
-           self.endX = 4
-           self.startY = 3
-           self.endY = 4
-       else:
-           self.startX = 0
-           self.endX = 8
-           self.startY = 0
-           self.endY = 8
+       self.startX = 0
+       self.endX = 8
+       self.startY = 0
+       self.endY = 8
 
        #self.startX = 1
@@ -89,5 +86,5 @@
        #self.endY = 8
 
-       # get a fre primary header values. if in test mode, then use defaults
+       # get a few primary header values. if in test mode, then use defaults
        if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
            self.logger.errorPair("Could not get", "MJD-OBS")
@@ -214,4 +211,7 @@
         ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \
         )"
+
+        print "frame meta sql: ", sql
+
         self.scratchDb.execute(sql)
 
@@ -219,5 +219,5 @@
         self.scratchDb.updateFilterID("FrameMeta", self.filter)
         self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum))
-        self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.skychunk.dataRelease))
 
     '''
@@ -228,11 +228,14 @@
         tableName = "ImageMeta_" + ota
         
+        # XXX we drop the table above so it is not left behind on failure
         # drop then re-create table
-        self.scratchDb.dropTable(tableName)
+        # self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ImageMeta"
         try: self.scratchDb.execute(sql)
         except: pass
-
-        # insert all detections into table
+        if (ota[0:2] == "XY"): ccdID = ota[2:4]
+        else: ccdID = 0
+
+        # insert image metadata into table
         sql = "INSERT INTO " + tableName + " ( \
                frameID \
@@ -299,5 +302,5 @@
                ) VALUES ( \
                " + str(self.expID) + " \
-               ," + ota[2:4] + " \
+               ," + str(ccdID) + " \
                ," + str(self.bias) + " \
                ," + str(self.biasScat) + " \
@@ -366,5 +369,5 @@
         self.scratchDb.updateFilterID(tableName, self.filter)
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
         if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
@@ -386,5 +389,5 @@
         except: pass
        
-        # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene
+        # delete all detections with PSF_INST_MAG < -17.5, as decreed by Gene
         BEFORE = self.scratchDb.getRowCount(ippTableName)
         #don't do this anymore
@@ -464,7 +467,8 @@
                , '" + self.dateStr + "' \
                , 0 \
-               , " + str(self.config.dataRelease) + "\
+               , " + str(self.skychunk.dataRelease) + "\
                FROM " + ippTableName
         # self.logger.info(sql)
+
         self.scratchDb.execute(sql)
 
@@ -517,10 +521,10 @@
         self.scratchDb.execute(sql)
 
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
 
     '''
     Populates the DetectionCalib table for this OTA
     '''
-    def populateDetectionCalibTable(self, ota):
+    def populateDetectionCalibTableUpdateInsert(self, ota):
 
         tableName = "DetectionCalib_" + ota
@@ -562,5 +566,69 @@
             WHERE a.objID = b.objID AND a.detectID = b.detectID"
         self.scratchDb.execute(sql)
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
+        
+    '''
+    Populates the DetectionCalib table for this OTA
+    '''
+    def populateDetectionCalibTable(self, ota):
+
+        # target table name:
+        tableName = "DetectionCalib_" + ota
+        # drop then re-create table
+        self.scratchDb.dropTable(tableName)
+        sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib"
+        try: self.scratchDb.execute(sql)
+        except: pass
+
+        externID = self.imageIDs[ota]
+
+        imageID = self.scratchDb.getImageIDFromExternID(externID)
+        self.logger.infoPair("obtained","imageID")
+
+        # check for & create output directory first
+        datadumpDir = "/tmp/datadump"
+        try:
+            statinfo = os.stat(datadumpDir)
+            # check on the stat results?
+        except:
+            print "making the data dump directory ", datadumpDir
+            os.mkdir(datadumpDir)
+            os.chmod(datadumpDir, 0777)
+            statinfo = os.stat(datadumpDir)
+
+        dumpFile = datadumpDir + "/genetest.xx.dat"
+        files = glob.glob(dumpFile)
+        if len(files) > 0:
+            os.unlink(dumpFile)
+
+        # insert all detections into table
+        sql = "SELECT \
+          a.objID,    \
+          a.detectID, \
+          a.ippObjID,      \
+          a.ippDetectID,   \
+          a.filterID,      \
+          a.surveyID,      \
+          b.ra,            \
+          b.dec_,          \
+          b.raErr,         \
+          b.decErr,        \
+          b.zp,            \
+          b.zpErr,         \
+          b.expTime,       \
+          b.airMass,       \
+          " + str(self.skychunk.dataRelease) + " \
+         FROM              \
+           Detection_" + ota + " as a \
+         JOIN " + self.scratchDb.dvoDetectionTable + " as b \
+         ON (a.objID = b.objID AND a.detectID = b.detectID) \
+         WHERE b.imageID = " + str(imageID) + \
+         " INTO OUTFILE '" + dumpFile + "'"
+        print "sql: ", sql
+        self.scratchDb.execute(sql)
+
+        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
+        print "sql: ", sql
+        self.scratchDb.execute(sql)
         
     '''
@@ -591,6 +659,39 @@
 
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
-
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
+
+    '''
+    Applies indexes and other constraints to the PSPS tables
+    '''
+    def alterPspsTablesChip(self, chipname, extname, x, y):
+
+        # drop the ImageMeta_ table first, or we can leave an invalid table behind
+        tableName = "ImageMeta_" + chipname
+        
+        # drop then re-create table
+        self.scratchDb.dropTable(tableName)
+
+        # load corresponding header into memory
+        header = self.fits.findAndReadHeader(extname)
+        if not header:
+            self.logger.errorPair("No header found for chip", chipname)
+            return False
+        
+        # check we have valid sourceID/imageID pair from the header
+        if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False
+        if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False
+        
+        # store sourceID/imageID combo in Db so DVO can look up later
+        if not self.useFullTables:
+            self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID'])
+            
+        # store these for later
+        self.imageIDs[chipname] = header['IMAGEID']
+            
+        # populate ImageMeta
+        self.populateImageMetaTable(chipname, header)
+        self.updateImageID("ImageMeta_" + chipname, x, y) 
+            
+        return True
 
     '''
@@ -623,25 +724,9 @@
                 ota = "XY%d%d" % (x, y)
                 
-                # load corresponding header into memory
-                header = self.fits.findAndReadHeader(ota + ".hdr")
-                if not header:
-                    self.logger.errorPair("No header found for OTA", ota)
-                    continue
-
-                # check we have valid sourceID/imageID pair from the header
-                if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue
-                if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue
-
-                # store sourceID/imageID combo in Db so DVO can look up later
-                if not self.useFullTables:
-                    self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID'])
-
-                # store these for later
-                self.imageIDs[ota] = header['IMAGEID']
-
-                # populate ImageMeta
-                self.populateImageMetaTable(ota, header)
-                self.updateImageID("ImageMeta_" + ota, x, y) 
-             
+                self.alterPspsTablesChip(ota, ota + ".hdr", x, y)
+
+        # try the test Chip
+        self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0)
+
         # now run DVO code to get all IDs
         if not self.useFullTables: 
@@ -651,6 +736,4 @@
         # the column in PSPS
         self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double")
-
-
 
         return True
@@ -675,5 +758,9 @@
                 extension = "XY%d%d_psf" % (x, y)
                 self.scratchDb.createIndex(extension, "IPP_IDET")
+        # try the test Chip
+        self.scratchDb.createIndex("Chip_psf", "IPP_IDET")
+
         self.logger.infoPair("created indexes on", "IPP tables")      
+
     '''
     Updates provided table with DVO IDs from DVO table
@@ -693,4 +780,82 @@
         self.scratchDb.execute(sql)
 
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTablesChip(self, chipname, x, y, results, tables):
+        # XXX EAM NOTE: drop tables (Detection_, SkinnyObject_, DetectionCalib_, Detection_) here so
+        # they do not polute the db?
+        # XXX or put an explicit drop at the end of the loop?
+
+        #self.logger.infoTitle("Processing " + chipname)
+        # this is a bit crude: if the chip is not present, this test will fail and the chip 
+        # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
+        if not self.scratchDb.astrometricSolutionOK(chipname):
+            self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+            return False
+        self.logger.info("populate stuff ");
+        # populate remainder of tables
+        self.populateDetectionTable(chipname, results)
+        self.logger.info("successful populate ");
+        # now add DVO IDs
+        self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
+        self.logger.info("updated dvoids")
+        results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
+        #self.logger.info("deleted nulls")
+        self.updateImageID("Detection_" + chipname, x, y)
+        self.logger.info("updateImageId")
+        rowCount = self.scratchDb.getRowCount("Detection_" + chipname) 
+        self.logger.info("got row count")
+        self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 
+                chipname, 
+                results['ORIGINALTOTAL'], 
+                results['SATDET'], 
+                results['NULLINSTFLUX'], 
+                results['NULLPEAKADU'], 
+                results['NULLOBJID'],
+                rowCount)
+        self.totalOriginal     = self.totalOriginal + results['ORIGINALTOTAL']
+        self.totalSatDet       = self.totalSatDet + results['SATDET']
+        self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX']
+        self.totalNullPeakFlux = self.totalNullPeakFlux + results['NULLPEAKADU']
+        self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
+        self.totalDetections   = self.totalDetections + rowCount
+        #self.logger.info("updated totals")
+        # check we have something in this Detection table TODO add this to table above
+        if rowCount < 1:
+            self.logger.infoPair("Skipping empty table for chipname", chipname)
+            return False;
+
+        # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
+        sql = "UPDATE ImageMeta_" + chipname + " \
+               SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
+        self.scratchDb.execute(sql)
+        
+        self.logger.info("updated imagedata")
+        self.populateSkinnyObjectTable(chipname)
+        self.logger.info("updated skinnyobject")
+        #self.populateObjectCalColorTable(chipname)
+        #self.logger.info("updated objectcalcolor")
+        self.populateDetectionCalibTable(chipname)
+        self.logger.info("updated detectioncalibtable")
+        
+        # add these to list of tables to export later
+        self.tablesToExport.append("ImageMeta_" + chipname)
+        self.logger.info("export ImageMeta")
+
+        self.tablesToExport.append("Detection_" + chipname)
+        self.logger.info("export Detection")
+
+        self.tablesToExport.append("SkinnyObject_" + chipname)
+        self.logger.info("export Skinny")
+
+        #self.tablesToExport.append("ObjectCalColor_" + chipname)
+        self.tablesToExport.append("DetectionCalib_" + chipname)
+        self.logger.info("export DetectionCalib")
+
+        tables.append("Detection_" + chipname)
+        self.logger.info("updated detectioncalibtable")
+
+        return True
 
     '''
@@ -705,5 +870,5 @@
         otaCount = 0
         results = {}
-        totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID = totalDetections = 0
+        self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
         self.logger.info("|  OTA  | Initial total |   Sat Det     | NULL instFlux | NULL peak ADU | NULL obj ID   |  Remainder    |")
@@ -721,73 +886,17 @@
                 if ota not in self.imageIDs: continue
 
-                #self.logger.infoTitle("Processing " + ota)
-                if not self.scratchDb.astrometricSolutionOK(ota):
-                    self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
-                    continue
-               # self.logger.info("populate stuff ");
-                # populate remainder of tables
-                self.populateDetectionTable(ota, results)
-               # self.logger.info("successful populate ");
-                # now add DVO IDs
-                self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota])
-               # self.logger.info("updated dvoids")
-                results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
-                #self.logger.info("deleted nulls")
-                self.updateImageID("Detection_" + ota, x, y)
-               # self.logger.info("updateImageId")
-                rowCount = self.scratchDb.getRowCount("Detection_" + ota) 
-              #  self.logger.info("got row count")
-                self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 
-                        ota, 
-                        results['ORIGINALTOTAL'], 
-                        results['SATDET'], 
-                        results['NULLINSTFLUX'], 
-                        results['NULLPEAKADU'], 
-                        results['NULLOBJID'],
-                        rowCount)
-                totalOriginal = totalOriginal + results['ORIGINALTOTAL']
-                totalSatDet = totalSatDet + results['SATDET']
-                totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX']
-                totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU']
-                totalNullObjID = totalNullObjID + results['NULLOBJID']
-                totalDetections = totalDetections + rowCount
-                #self.logger.info("updated totals")
-                # check we have something in this Detection table TODO add this to table above
-                if rowCount < 1:
-                    self.logger.debugPair("Skipping empty table for ota", ota)
-                    continue;
-
-                # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " \
-                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota]))
-                self.scratchDb.execute(sql)
-                
-                #self.logger.info("updated imagedata")
-                self.populateSkinnyObjectTable(ota)
-                #self.logger.info("updated skinnyobject")
-                #self.populateObjectCalColorTable(ota)
-                #self.logger.info("updated objectcalcolor")
-                self.populateDetectionCalibTable(ota)
-                #self.logger.info("updated detectioncalibtable")
-                
-                # add these to list of tables to export later
-                self.tablesToExport.append("ImageMeta_" + ota)
-                self.tablesToExport.append("Detection_" + ota)
-                self.tablesToExport.append("SkinnyObject_" + ota)
-                #self.tablesToExport.append("ObjectCalColor_" + ota)
-                self.tablesToExport.append("DetectionCalib_" + ota)
-                tables.append("Detection_" + ota)
-           
-                otaCount = otaCount + 1
-                #self.logger.info("end of ota") 
+                if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
+
+        if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+
         # print totals
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
         self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", 
-                totalOriginal, 
-                totalSatDet, 
-                totalNulIInstFlux, 
-                totalNullPeakFlux, 
-                totalNullObjID,
-                totalDetections)
+                self.totalOriginal, 
+                self.totalSatDet, 
+                self.totalNulIInstFlux, 
+                self.totalNullPeakFlux, 
+                self.totalNullObjID,
+                self.totalDetections)
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
 
@@ -830,7 +939,9 @@
     def importIppTables(self, filter=""):
 
-       if self.config.test: regex = "XY33.psf"
-       else : regex = ".*.psf"
+       ## if self.config.test: regex = "XY33.psf"
+       ## else : regex = ".*.psf"
+       regex = ".*.psf"
   
+       # XXX EAM NOTE : this is fragile : requires PS1_V4
        columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"