Changeset 35417 for trunk/ippToPsps/jython/detectionbatch.py

Timestamp:

Apr 19, 2013, 4:35:54 PM (13 years ago)

Author:

eugene

Message:

use MEMORY engine for dvoDetectionFull; add error handline for dvopsps; check mysql version; remove some test verbosity; check for image existence in dvoImages; skip bad batches but do not exit; fix precision issues; add -realgpc option (part of -test mode); log times in milliseconds

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython/detectionbatch.py (modified) (10 diffs)

trunk/ippToPsps/jython/detectionbatch.py

@@ -212 +212 @@
         )"
 
-        print "frame meta sql: ", sql
+        # print "frame meta sql: ", sql
 
         self.scratchDb.execute(sql)
@@ -607 +607 @@
          WHERE b.imageID = " + str(imageID) + \
          " INTO OUTFILE '" + dumpFile + "'"
-        print "sql: ", sql
-        self.scratchDb.execute(sql)
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.info("failed to select data for detectionCalib")
+            self.logger.infoPair("sql: ", sql)
+            raise
 
         sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
-        print "sql: ", sql
-        self.scratchDb.execute(sql)
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.info("failed to load data from infile for detectionCalib")
+            self.logger.infoPair("sql: ", sql)
+            raise
         
     '''
@@ -655 +662 @@
 
         # load corresponding header into memory
-        header = self.fits.findAndReadHeader(extname)
+        header = self.fits.findAndReadHeader(extname, self.config.test)
         if not header:
-            self.logger.errorPair("No header found for chip", chipname)
+            # XXX There is no way to check if we SHOULD find a given
+            # chip.  We have to accept the absence of the header as
+            # information that the chip is not available.  This
+            # message is too verbose, only print in test mode
+            if self.config.test: self.logger.errorPair("No header found for chip", chipname)
             return False
         
@@ -670 +681 @@
         # store these for later
         self.imageIDs[chipname] = header['IMAGEID']
+        self.sourceIDs[chipname] = header['SOURCEID']
             
         # populate ImageMeta
@@ -691 +703 @@
         # dictionary objects to hold imageIDs for later
         self.imageIDs = {}
+        self.sourceIDs = {}
 
         # loop through all OTAs and populate ImageMeta extensions
@@ -773 +786 @@
         # this is a bit crude: if the chip is not present, this test will fail and the chip 
         # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
-        if not self.scratchDb.astrometricSolutionOK(chipname):
-            self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+
+        # XXX keep this in or not?
+        # if not self.scratchDb.astrometricSolutionOK(chipname):
+        #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+        #     return False
+
+        # does this chip exist in the DVO image table (if NOT, then skip it)
+        if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
+            self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
             return False
+
         self.logger.info("populate stuff ");
         # populate remainder of tables
@@ -803 +824 @@
         self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
         self.totalDetections   = self.totalDetections + rowCount
+
+        fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
+        # print "kept ", str(fractionKept), " detections from ", chipname
+
+        if fractionKept < 0.8:
+            # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
+            self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
+            self.skipBatch = True
+            return False;
+
         #self.logger.info("updated totals")
         # check we have something in this Detection table TODO add this to table above
         if rowCount < 1:
-            self.logger.infoPair("Skipping empty table for chipname", chipname)
+            self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
+            self.skipBatch = True
             return False;
 
@@ -849 +881 @@
 
         # loop through all OTAs again to update with DVO IDs
+        self.skipBatch = False
         self.validChips = []    
         self.tablesToExport = []    
@@ -872 +905 @@
 
                 if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
-
-        if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+                if self.skipBatch: 
+                    self.logger.error("fatal problem for exposure, skipping")
+                    return False
+
+        if "Chip" in self.imageIDs:
+            if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+            if self.skipBatch: 
+                self.logger.error("fatal problem for exposure, skipping")
+                return False
 
         # print totals
@@ -886 +926 @@
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
 
-        # if we only have one table export, i.e. FrameMeta, then get out of here
+        # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)
         if len(self.tablesToExport) == 1:
-            
+            self.skipBatch = True
             self.logger.error("No tables to export")
             return False