Changeset 35417 for trunk/ippToPsps/jython/dvo.py

Timestamp:

Apr 19, 2013, 4:35:54 PM (13 years ago)

Author:

eugene

Message:

use MEMORY engine for dvoDetectionFull; add error handline for dvopsps; check mysql version; remove some test verbosity; check for image existence in dvoImages; skip bad batches but do not exit; fix precision issues; add -realgpc option (part of -test mode); log times in milliseconds

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython/dvo.py (modified) (5 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -193 +193 @@
             p = Popen(cmd, shell=True, stdout=PIPE)
             p.wait()
+            if p.returncode != 0:
+                self.logger.errorPair("dvopsps failed on SkyTable.fits for ", self.skychunk.dvoLocation)
+                raise
 
         self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable)
@@ -591 +594 @@
     def nativeIngestDetections(self, boxId, boxDim):
 
-        # XXX put the chunk below in a separate method 
-        # blow away existing dvoDetection table
-
         # clear the 'ingested' field for all boxes owned by this host
         self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
@@ -607 +607 @@
         ##     raise
 
-        # drop detections table
-        self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
-        self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable)
-        
-        # create detections table
-        self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable)
-        sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection"
-        try: self.scratchDb.execute(sql)
-        except:
-            self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable)
+        # blow away existing dvoDetection table & re-crate
+        if not self.scratchDb.resetDvoDetectionTable():
+            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
             return False
-        self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable)
-        
-        # add fileID column
-        sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL"
-        try: self.scratchDb.execute(sql)
-        except: 
-            self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable)
-            return False
-
-        # add an index to the fileID column
-        self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID")
 
         # make sure we have an up-to-date Images table
@@ -665 +647 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
+        if p.returncode != 0:
+            self.logger.errorPair("dvopsps failed on ", self.scratchDb.dvoDetectionTable)
+            raise
 
         ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
@@ -670 +655 @@
         self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
 
-        # add fileID column
+        # run 'ANALYZE' to get indexes up-to-snuff
         sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable
         try: self.scratchDb.execute(sql)