Changeset 35402

Timestamp:

Apr 16, 2013, 6:27:02 AM (13 years ago)

Author:

eugene

Message:

add error handling for dvopsps skytable; add config entry for memory usage for memory db tables; add / fix version parsing; error handling for xmllint; error handline for dvograbber (unused); set max_heap_table_size before creating memory tables; add error handling for dvopsps objects; break on error from dvopsps detections; do not try to populate Chip table unless it exists; skip batch on failure, but continue to load

Location:

branches/eam_branches/ipp-20130307/ippToPsps/jython

Files:

• batch.py (modified) (4 diffs)
• config.py (modified) (2 diffs)
• datastore.py (modified) (1 diff)
• detectionbatch.py (modified) (2 diffs)
• dvo.py (modified) (1 diff)
• dvoobjects.py (modified) (2 diffs)
• loader.py (modified) (1 diff)
• mysql.py (modified) (2 diffs)
• scratchdb.py (modified) (2 diffs)

branches/eam_branches/ipp-20130307/ippToPsps/jython/batch.py

@@ -276 +276 @@
             p = Popen(cmd, shell=True, stdout=PIPE)
             p.wait()
+            if p.returncode != 0:
+                self.logger.errorPair("error running xmllint for ", tmpPath)
+                raise
         except:
             self.logger.errorPair("Popen failed", cmd)
@@ -301 +304 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
-
         if p.returncode != 0:
             self.logger.errorPair("tar command", "failed")
@@ -311 +313 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
-
         if p.returncode != 0:
             self.logger.errorPair("gzip command", "failed")
@@ -519 +520 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
+        if p.returncode != 0:
+            self.logger.errorPair("dvograbber command", "failed")
+            return False
 
         rowCount = self.scratchDb.getRowCount("dvoDetection")

branches/eam_branches/ipp-20130307/ippToPsps/jython/config.py

@@ -66 +66 @@
         self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text
 
-
-
         # print "config.programName: ", self.programName
         # print "config.configDir: ", self.configDir
@@ -125 +123 @@
             return self.settingsDoc.find(dbType +"/password").text
 
+    def getDbMemory(self, dbType): 
+        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
+            return self.scratchDbLeaf.find('memory').text
+        else:
+            print "memory is not a valid option for non-scratch databases..."
+            raise
+
     def setScratchDbLeaf(self, myhost):
         root = self.settingsDoc.getroot()

branches/eam_branches/ipp-20130307/ippToPsps/jython/datastore.py

@@ -47 +47 @@
                     --list " + tempFile.name
 
+        print "dsreg cmd: ", command
         p = Popen(command, shell=True, stdout=PIPE)
         p.wait()

branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py

@@ -909 +909 @@
                     return False
 
-        if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
-
-        if self.skipBatch: 
-            self.logger.error("fatal problem for exposure, skipping")
-            return False
+        if "Chip" in self.imageIDs:
+            if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
+            if self.skipBatch: 
+                self.logger.error("fatal problem for exposure, skipping")
+                return False
 
         # print totals
@@ -926 +926 @@
         self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
 
-        # if we only have one table export, i.e. FrameMeta, then get out of here
+        # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)
         if len(self.tablesToExport) == 1:
-            
+            self.skipBatch = True
             self.logger.error("No tables to export")
             return False

branches/eam_branches/ipp-20130307/ippToPsps/jython/dvo.py

@@ -193 +193 @@
             p = Popen(cmd, shell=True, stdout=PIPE)
             p.wait()
+            if p.returncode != 0:
+                self.logger.errorPair("dvopsps failed on SkyTable.fits for ", self.skychunk.dvoLocation)
+                raise
 
         self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable)

branches/eam_branches/ipp-20130307/ippToPsps/jython/dvoobjects.py

@@ -67 +67 @@
 
         if self.skychunk.parallel:
+            if self.scratchDb.dbHost == "localhost":
+                self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb")
+                raise
+                
             cmd += " -parallel"
 
@@ -72 +76 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
+        if p.returncode != 0:
+            self.logger.errorPair("dvopsps failed on ", region)
+            raise
 
         # update lists after attempted sync

branches/eam_branches/ipp-20130307/ippToPsps/jython/loader.py

@@ -198 +198 @@
                             # insertion
                             if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
-                                self.dvoDetections.nativeIngestDetections(boxId, boxDim)
+                                if not self.dvoDetections.nativeIngestDetections(boxId, boxDim):
+                                    self.logger.error("Unable to ingest detections with nativeIngest")
+                                    return False
 
                             useFullTables = 1

branches/eam_branches/ipp-20130307/ippToPsps/jython/mysql.py

@@ -55 +55 @@
             rs.first()
             fullVersion = rs.getString(1)
-            self.version = fullVersion[0:3]
+            majVersion = fullVersion[0:3]
+            self.version = float(majVersion)
             
         except:
@@ -61 +62 @@
             raise
 
-        # print "version number: ", self.version
         self.connected = True
 

branches/eam_branches/ipp-20130307/ippToPsps/jython/scratchdb.py

@@ -38 +38 @@
         self.dvoSkyTable = "dvoSkyTable"
         self.dvoPhotcodesTable = "dvoPhotcodes"
+        self.memoryTableSize = config.getDbMemory(dbType)
 
     '''
@@ -509 +510 @@
        self.dropTable(self.dvoDetectionTable)
 
+       # we need a big enough memory table for the incoming detections
+       print "requested memory in gigs: ", self.memoryTableSize
+       tableSize = int(self.memoryTableSize) * 1024 * 1024 * 1024
+       sql = "set @@max_heap_table_size = " + str(tableSize)
+       print "setting max heap size: ", sql
+       try: self.execute(sql)
+       except: 
+           self.logger.errorPair("Unable to set max MEMORY table size for ", self.dvoDetectionTable)
+           return False
+
        # create detections table
        self.logger.infoPair("Creating table", self.dvoDetectionTable)