Changeset 37138 for branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py

Timestamp:

Jul 29, 2014, 12:16:23 PM (12 years ago)

Author:

eugene

Message:

move importIppSTTables from batch.py to stackbatch.py and rename importIppTables; make batch:run call importIppTables consistently for all batch types

File:

• branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (modified) (6 diffs)

branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py

@@ -381 +381 @@
              self.tablesToExport.append(table.name)
 
-         return self.alterPspsTables();
+         return True
 
     '''
@@ -391 +391 @@
 
     '''
-    Alter PSPS tables
+    Alter PSPS tables (Subclass implements this)
     '''   
     def alterPspsTables(self):
@@ -447 +447 @@
       self.indexIppTables()
 
-      return True
-    '''
-    Imports IPP tables from FITS file
-
-    Accepts a regular expression filter so not all tables need to be imported
-    '''
-    def importIppSTTables(self, columns="*", filter=""):
-
-      self.logger.infoPair("Importing ST tables with filter", filter)
-
-      # print "trying to read ", self.fits.getPath()
-
-      # ST has 5 cmf files - all with the same table names. To counter this we do it like this.
-
-      filters = ["g","r","i","z","y"]
-      count = 0
-      for f in filters:
-          skip = 0
-          if (f == "g"):
-              if self.gstackID > 0:  
-                  fileName = self.gfits.getPath()
-              else:
-                  skip = 1
-          if (f == "r"):
-              if self.rstackID > 0:
-                  fileName = self.rfits.getPath()
-              else:
-                  skip = 1
-          if (f == "i"):
-              if self.istackID > 0:
-                  fileName = self.ifits.getPath()
-              else:
-                  skip = 1
-          if (f == "z"):
-              if self.zstackID > 0:
-                  fileName = self.zfits.getPath()
-              else:
-                  skip = 1
-          if (f == "y"):
-              if self.ystackID > 0:
-                  fileName = self.yfits.getPath()
-              else:
-                  skip = 1
-
-          if skip == 1:
-              self.logger.infoPair("no files for filter" , f) 
-          else:
-             self.logger.infoPair("using filename:",fileName)
-          
-             try:
-              tables = stilts.treads(fileName)
-             except:
-              self.logger.errorPair("STILTS could not import from", fileName)
-              return False
-              
-             #count = 0
-             for table in tables:
-              
-              # print "import smf table ", table
-              match = re.match(filter, table.name)
-              
-              if not match: continue
-              self.logger.infoPair("Reading IPP table", f + table.name)
-              table = stilts.tpipe(table, cmd='addcol table_index $0')
-                  
-              table = stilts.tpipe(table, cmd='explodeall')
-                  
-              # print "read smf table ", table
-                  
-              # drop any previous tables before import
-              self.scratchDb.dropTable(f + table.name)
-                  
-              # IPP FITS files are littered with infinities, so remove these
-              self.logger.debug("Removing Infinity values from all columns")
-              table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
-              table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
-              table = stilts.tpipe(table, cmd='replaceval Infinity null *')
-              # print "cleaned up values ", table
-                  
-              try:
-                  table.write(self.scratchDb.url + '#' + f + table.name)
-                  # XXX this one is not needed
-                  # self.scratchDb.killLastConnectionID()
-                  count = count + 1
-              except:
-                  self.logger.exception("Problem writing table '" + f + table.name + "' to the database")
-                      
-      self.logger.infoPair("Done. Imported", "%d tables" % count)
-      self.indexIppTables()
-                      
       return True
 
@@ -641 +551 @@
     def run(self):
 
-        # this is badly named : it creates the tables and then fills in
-        # the ImageMeta tables for each ota (alterPspsTables)
+        # creates the empty PSPS tables
         if not self.createEmptyPspsTables():
             self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables")
@@ -648 +557 @@
             raise
 
+        # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch)
+        if not self.alterPspsTables():
+            self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type")
+            self.ippToPspsDb.updateProcessed(self.batchID, -1)
+            raise
+
         # for P2/ST, this reads the detection tables from the CMF/SMF file(s)
         # for OB, this imports object data from DVO
-        if self.batchType == "ST":
-            self.logger.infoPair("did we get here","?")  
-            if not self.importIppSTTables():
-                self.logger.errorPair("skipping this batch", "could not import ST IPP tables")
-                self.ippToPspsDb.updateProcessed(self.batchID, -1)
-                raise
-            self.logger.infoPair("we did the ST","yes")
-        else:    
-            # if not ST we do this 
-            if not self.importIppTables():
-                if self.batchType == "OB":
-                    self.logger.errorPair("skipping this batch", "no dvo tables for this region")
-                    self.ippToPspsDb.updateProcessed(self.batchID, 1)
-                    return True
-                self.logger.errorPair("Aborting this batch", "could not import IPP tables")
-                self.ippToPspsDb.updateProcessed(self.batchID, -1)
-                raise
+        if not self.importIppTables():
+            if self.batchType == "OB":
+                self.logger.errorPair("skipping this batch", "no dvo tables for this region")
+                self.ippToPspsDb.updateProcessed(self.batchID, 1)
+                return True
+            self.logger.errorPair("Aborting this batch", "could not import IPP tables")
+            self.ippToPspsDb.updateProcessed(self.batchID, -1)
+            raise
+
         self.logger.infoPair("populatePspsTables","ok")
         if not self.populatePspsTables():
@@ -700 +607 @@
                             tarballFile)
 
+# XXX why is this down here??
 from datastore import Datastore