Changeset 37403 for branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py

Timestamp:

Sep 19, 2014, 4:05:27 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• ippToPsps/jython/batch.py (modified) (16 diffs)

branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py

@@ -37 +37 @@
                  batchID,
                  batchType, 
-                 fits,
-                 useFullTables): 
-
-        # print "starting the batch"
-
-        # self.printline = 0
-        # self.testprint()
+                 fits): 
 
         self.readHeader = False
@@ -49 +43 @@
         self.skychunk = skychunk
         self.fits = fits
-        self.useFullTables = useFullTables
 
         # set up logging
@@ -55 +48 @@
         self.logger.infoSeparator()
         self.logger.debug("Batch class constructor")
-
-        # self.testprint()
 
         # set up class variables
@@ -68 +59 @@
         self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot"
 
-        # self.testprint()
-
         # check FITS file is ok
         if self.fits: 
@@ -79 +68 @@
                 raise
 
-        self.scratchDb.setUseFullTables(self.useFullTables)
+        # define the dvo table names
+        self.scratchDb.setDvoTableNames()
 
         # TODO
@@ -107 +97 @@
         if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
 
-        # self.testprint()
-
         # store today's date
         now = datetime.datetime.now();
         self.dateStr = now.strftime("%Y-%m-%d")
 
-        # create DVO tables if accessing DVO directly
-        if not self.useFullTables: self.scratchDb.createDvoTables()
+        # create DVO tables (scratch database)
+        self.scratchDb.createDvoTables()
 
         # dump stuff to the log
@@ -120 +108 @@
         self.logger.infoPair("Batch name", self.batchName)
         self.logger.infoPair("Survey ID", "%d" % self.surveyID)
-        self.logger.infoBool("Use full DVO tables?", self.useFullTables)
 
         if self.fits:
             self.logger.infoPair("Input FITS file", self.fits.getOriginalPath())
             self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount())
-
-        # self.testprint()
 
         self.logger.infoPair("Output path", self.localOutPath)
@@ -237 +222 @@
              self.logger.errorPair("Missing header field", key)
              return "NULL"
+
+    '''
+    Returns the string keyword value from this header or else "NULL"
+    '''
+    def getKeyValue(self, header, key):
+
+         if key in header: return header[key]
+         else:
+             self.logger.errorPair("Missing header field", key)
+             return "NULL"
+
+    '''
+    Returns the string keyword value from this header or else "NULL"
+    '''
+    def getKeyFloat(self, header, format, key):
+
+         if key in header: 
+             value = format % float(header[key])
+         else:
+             self.logger.errorPair("Missing header field", key)
+             value = format % -999.9
+
+         return value
+
+    '''
+    Returns the string keyword value from this header or else "NULL"
+    '''
+    def getKeyInt(self, header, defValue, key):
+
+         if key in header: 
+             value = str(int(header[key]))
+         else:
+             self.logger.errorPair("Missing header field", key)
+             value = str(defValue)
+
+         return value
 
     '''
@@ -373 +394 @@
              self.tablesToExport.append(table.name)
 
-         return self.alterPspsTables();
+         return True
 
     '''
@@ -383 +404 @@
 
     '''
-    Alter PSPS tables
+    Alter PSPS tables (Subclass implements this)
     '''   
     def alterPspsTables(self):
@@ -417 +438 @@
 
           # print "read smf table ", table
+          self.logger.infoPair("read smf table", table)
 
           # drop any previous tables before import
@@ -458 +480 @@
            # get everything from table
            try:
-               print "reading table from mysql ", table
                _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
                self.scratchDb.killLastConnectionID()
@@ -467 +488 @@
            # replace nulls and empty fields with weird PSPS -999 pseudo-null
            _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
-
-           match = re.match(regex, table)
+           self.logger.infoPair('regex', regex)
+           self.logger.infoPair('table', table)
+           match = re.search(regex, table)
            newTableName = match.group(1)
 
@@ -542 +564 @@
     def run(self):
 
-        # this is badly named : it creates the tables and then fills in
-        # the ImageMeta tables for each ota (alterPspsTables)
+        # creates the empty PSPS tables
         if not self.createEmptyPspsTables():
             self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables")
+            self.ippToPspsDb.updateProcessed(self.batchID, -1)
+            raise
+
+        # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch)
+        if not self.alterPspsTables():
+            self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)
             raise
@@ -556 +583 @@
                 self.ippToPspsDb.updateProcessed(self.batchID, 1)
                 return True
-
             self.logger.errorPair("Aborting this batch", "could not import IPP tables")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)
             raise
 
+        self.logger.infoPair("populatePspsTables","ok")
         if not self.populatePspsTables():
             if self.skipBatch:
@@ -593 +620 @@
                             tarballFile)
 
+        if self.config.onebatch:
+            self.logger.infoPair("onebatch requested", "exiting")
+            sys.exit(0)
+
+# XXX why is this down here??
 from datastore import Datastore
-