Changeset 33638 for branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py

Timestamp:

Mar 30, 2012, 2:49:37 PM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py (modified) (12 diffs)

branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py

@@ -21 +21 @@
 
 '''
-Base class of all batch types.
+Abstract base class of all batch types.
 '''
 class Batch(object):
@@ -27 +27 @@
     '''
     Constructor
-
-    >>> batch = Batch(1,2,3,4,5,6,7)
-    >>> print batch.pspsVoTableFilePath
-    "../config/2/tables.vot"
     '''
     def __init__(self, 
                  logger, 
-                 configPath,
-                 doc,
+                 config,
                  gpc1Db,
                  ippToPspsDb,
+                 scratchDb,
                  id,
                  batchID,
                  batchType, 
-                 fits): 
+                 fits,
+                 useFullTables): 
 
         self.everythingOK = False
         self.readHeader = False
-        self.configPath = configPath
-        self.doc = doc
+        self.config = config
         self.fits = fits
+        self.useFullTables = useFullTables
 
         # set up logging
@@ -59 +56 @@
         self.gpc1Db = gpc1Db
         self.ippToPspsDb = ippToPspsDb
+        self.scratchDb = scratchDb
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
@@ -77 +75 @@
                return
 
-        # get info from config
-        self.survey = self.doc.find("options/survey").text
-        self.pspsSurvey = self.doc.find("options/pspsSurvey").text
-        self.dvoGpc1Label = self.doc.find("dvo/gpc1Label").text
-        self.dvoLocation = self.doc.find("dvo/location").text
-        self.useFullTables = int(self.doc.find("dvo/useFullTables").text)
-        self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables)
-
-        if not self.scratchDb.everythingOK: return
+        self.scratchDb.setUseFullTables(self.useFullTables)
 
         # TODO
         self.tablesToExport = []
 
-        if self.survey != "":
-            self.surveyID = self.scratchDb.getSurveyID(self.survey)
+        if self.config.survey != "":
+            self.surveyID = self.scratchDb.getSurveyID(self.config.survey)
         else:
             self.surveyID = -1;
        
-        # get some options from the config
-        self.testMode = int(self.doc.find("options/testMode").text)
-        self.dataRelease = int(self.doc.find("metadata/dataRelease").text)
-
         # create datastore object
-        self.datastore = Datastore(self.logger, self.doc, self.ippToPspsDb)
+        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
 
         # get local storage location from config
         self.batchName = Batch.getNameFromID(self.batchID)
-        self.basePath = self.doc.find("localOutPath").text
         self.subDir = Batch.getSubDir(
-                self.basePath,
+                self.config.basePath,
                 self.batchType, 
-                self.dvoGpc1Label)
+                self.config.dvoLabel)
 
         self.localOutPath = Batch.getOutputPath(
-                self.basePath,
+                self.config.basePath,
                 self.batchType, 
-                self.dvoGpc1Label,
+                self.config.dvoLabel,
                 self.batchID)
 
@@ -128 +113 @@
         self.logger.infoTitle("New " + self.batchType + " batch")
         self.logger.infoPair("Batch name", self.batchName)
-        self.logger.infoPair("Survey", self.survey)
         self.logger.infoPair("Survey ID", "%d" % self.surveyID)
-        self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey)
-        self.logger.infoPair("DVO location", self.dvoLocation)
         self.logger.infoBool("Use full DVO tables?", self.useFullTables)
 
@@ -230 +212 @@
          if value != "NULL": return value
          else:
-             if not self.testMode: return "NULL"
+             if not self.config.test: return "NULL"
              header[key] = default
              self.logger.infoPair("Hardcoding " + key + " to", header[key])
@@ -259 +241 @@
         root.attrib['type'] = self.batchType
         root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 
-        if self.batchType != "IN": root.attrib['survey'] = self.pspsSurvey
+        root.attrib['survey'] = self.config.pspsSurvey
+
+        # min/max object IDs
         try: self.minObjID
         except: pass
@@ -278 +262 @@
 
         # clunky way to prettify XML
-        p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE)
-        p.wait()
-        os.remove(tmpPath)
+        try:
+            cmd = "xmllint --format " + tmpPath + " > " + outPath
+            p = Popen(cmd, shell=True, stdout=PIPE)
+            p.wait()
+        except:
+            self.logger.errorPair("Popen failed", cmd)
+            return False
+        finally: 
+            os.remove(tmpPath)
+
+        return True
 
 
@@ -290 +282 @@
         # set up filenams and paths
         tarFile = Batch.getTarFile(self.batchID)
-        tarPath = Batch.getTarPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
+        tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
         tarballFile = Batch.getTarballFile(self.batchID)
-        tarballPath = Batch.getTarballPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
+        tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
 
         # tar directory
@@ -395 +387 @@
 
       self.logger.infoPair("Importing tables with filter", filter)
-      tables = stilts.treads(self.fits.getPath())
+
+      try:
+          tables = stilts.treads(self.fits.getPath())
+      except:
+          self.logger.errorPair("STILTS could not import from", self.fits.getPath())
+          return False
 
       count = 0
@@ -496 +493 @@
     def getIDsFromDVO(self):
 
-        if self.scratchDb.getRowCount("dvoMeta") < 1:
-            self.logger.error("No DVO IDs found in dvoMeta")
+        if self.scratchDb.getRowCount(self.scratchDb.dvoImagesTable) < 1:
+            self.logger.errorInfo("No DVO images found in", self.scratchDb.dvoImagesTable)
             return False
 
         # TODO path to DVO prog hardcoded temporarily
-        cmd = "../src/dvograbber " + self.configPath + " " + self.dvoLocation
+        cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.config.dvoLocation
         self.logger.infoPair("Running DVO", cmd)
         p = Popen(cmd, shell=True, stdout=PIPE)
@@ -539 +536 @@
                 self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file")
             else:
-                self.writeBatchManifest()
-                if int(self.doc.find("options/publishToDatastore").text): 
-                    # tar and zip ready for publication to datastore
-                    if self.tarAndZip():
-                        # now publish to the datastore
-                        tarballFile = Batch.getTarballFile(self.batchID)
-                        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
-
-                if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
+                if self.writeBatchManifest():
+                    if self.config.datastorePublishing: 
+                        # tar and zip ready for publication to datastore
+                        if self.tarAndZip():
+                            # now publish to the datastore
+                            tarballFile = Batch.getTarballFile(self.batchID)
+                            Batch.publishToDatastore(
+                                    self.datastore, 
+                                    self.batchID, 
+                                    self.batchName, 
+                                    self.subDir, 
+                                    tarballFile)
 
 from datastore import Datastore