Changeset 35352 for branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py

Timestamp:

Apr 3, 2013, 12:52:33 PM (13 years ago)

Author:

eugene

Message:

changed dvoDetectionFull mysql engine to 'MEMORY' to improve ingest speed

handle errrors in calls to dvopsps (only in detectionbatch.py)

removed a lot of excess verbosity used for debugging / testing

determine the mysql major version number via mysql query

added test option -realgpc to use test ipptopsps databases with a real gpc1 dataset

add version-dependent call to 'set session old_alter_table' to allow
the ALTER IGNORE statements to work on mysql v5.5 machines (like our
desktops).

raise an exception in case the load infile operations fail

remove unneeded 'killLastConnectionID'

extend logging time to milliseconds

allow an exposure to be skipped without aborting loader

check that a P2 image exists in the dvoImage table (otherwise skip the chip)

change the mysql floating point types for the dvoDetection table to use FLOAT/DOUBLE instead of REAL/FLOAT

remove the old test for valid astrometry of a chip (NASTRO >= 50)

this test was crude and not really the right test. it was also used
to verify that an image should / should not be included. now we check
against dvoImages to see if the chip should be loaded or not (DVO
rejects input images for bad astrometry)

skip an exposure if too many detections have been rejected. this is a sign of a problem and should require more attention

File:

• branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py (modified) (7 diffs)

branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py

@@ -212 +212 @@
         )"
 
-        print "frame meta sql: ", sql
+        # print "frame meta sql: ", sql
 
         self.scratchDb.execute(sql)
@@ -607 +607 @@
          WHERE b.imageID = " + str(imageID) + \
          " INTO OUTFILE '" + dumpFile + "'"
-        print "sql: ", sql
-        self.scratchDb.execute(sql)
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.info("failed to select data for detectionCalib")
+            self.logger.infoPair("sql: ", sql)
+            raise
 
         sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
-        print "sql: ", sql
-        self.scratchDb.execute(sql)
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.info("failed to load data from infile for detectionCalib")
+            self.logger.infoPair("sql: ", sql)
+            raise
         
     '''
@@ -655 +662 @@
 
         # load corresponding header into memory
-        header = self.fits.findAndReadHeader(extname)
+        header = self.fits.findAndReadHeader(extname, self.config.test)
         if not header:
-            self.logger.errorPair("No header found for chip", chipname)
+            # XXX There is no way to check if we SHOULD find a given
+            # chip.  We have to accept the absence of the header as
+            # information that the chip is not available.  This
+            # message is too verbose, only print in test mode
+            if self.config.test: self.logger.errorPair("No header found for chip", chipname)
             return False
         
@@ -773 +784 @@
         # this is a bit crude: if the chip is not present, this test will fail and the chip 
         # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
-        if not self.scratchDb.astrometricSolutionOK(chipname):
-            self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+
+        # XXX keep this in or not?
+        # if not self.scratchDb.astrometricSolutionOK(chipname):
+        #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
+        #     return False
+
+        # does this chip exist in the DVO image table (if NOT, then skip it)
+        if not self.scratchDb.haveThisChip(self.imageIDs[chipname]):
+            self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
             return False
+
         self.logger.info("populate stuff ");
         # populate remainder of tables
@@ -803 +822 @@
         self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
         self.totalDetections   = self.totalDetections + rowCount
+
+        fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
+        # print "kept ", str(fractionKept), " detections from ", chipname
+
+        if fractionKept < 0.8:
+            # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
+            self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
+            self.skipBatch = True
+            return False;
+
         #self.logger.info("updated totals")
         # check we have something in this Detection table TODO add this to table above
         if rowCount < 1:
-            self.logger.infoPair("Skipping empty table for chipname", chipname)
+            self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
+            self.skipBatch = True
             return False;
 
@@ -849 +879 @@
 
         # loop through all OTAs again to update with DVO IDs
+        self.skipBatch = False
         self.validChips = []    
         self.tablesToExport = []    
@@ -872 +903 @@
 
                 if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
+                if self.skipBatch: 
+                    self.logger.error("fatal problem for exposure, skipping")
+                    return False
 
         if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+
+        if self.skipBatch: 
+            self.logger.error("fatal problem for exposure, skipping")
+            return False
 
         # print totals