Changeset 36826 for branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py

Timestamp:

Jun 7, 2014, 5:36:39 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py (modified) (3 diffs)

branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py

@@ -440 +440 @@
 
       return True
+    '''
+    Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppSTTables(self, columns="*", filter=""):
+
+      self.logger.infoPair("Importing ST tables with filter", filter)
+
+      # print "trying to read ", self.fits.getPath()
+
+      # ST has 5 cmf files - all with the same table names. To counter this we do it like this.
+
+      filters = ["g","r","i","z","y"]
+      count = 0
+      for f in filters:
+          skip = 0
+          if (f == "g"):
+              if self.gstackID > 0:  
+                  fileName = self.gfits.getPath()
+              else:
+                  skip = 1
+          if (f == "r"):
+              if self.rstackID > 0:
+                  fileName = self.rfits.getPath()
+              else:
+                  skip = 1
+          if (f == "i"):
+              if self.istackID > 0:
+                  fileName = self.ifits.getPath()
+              else:
+                  skip = 1
+          if (f == "z"):
+              if self.zstackID > 0:
+                  fileName = self.zfits.getPath()
+              else:
+                  skip = 1
+          if (f == "y"):
+              if self.ystackID > 0:
+                  fileName = self.yfits.getPath()
+              else:
+                  skip = 1
+
+          if skip == 1:
+              self.logger.infoPair("no files for filter" , f) 
+          else:
+             self.logger.infoPair("using filename:",fileName)
+          
+             try:
+              tables = stilts.treads(fileName)
+             except:
+              self.logger.errorPair("STILTS could not import from", fileName)
+              return False
+              
+             #count = 0
+             for table in tables:
+              
+              # print "import smf table ", table
+              match = re.match(filter, table.name)
+              
+              if not match: continue
+              self.logger.infoPair("Reading IPP table", f + table.name)
+              table = stilts.tpipe(table, cmd='addcol table_index $0')
+                  
+              table = stilts.tpipe(table, cmd='explodeall')
+                  
+              # print "read smf table ", table
+                  
+              # drop any previous tables before import
+              self.scratchDb.dropTable(f + table.name)
+                  
+              # IPP FITS files are littered with infinities, so remove these
+              self.logger.debug("Removing Infinity values from all columns")
+              table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+              table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+              table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+              # print "cleaned up values ", table
+                  
+              try:
+                  table.write(self.scratchDb.url + '#' + f + table.name)
+                  # XXX this one is not needed
+                  # self.scratchDb.killLastConnectionID()
+                  count = count + 1
+              except:
+                  self.logger.exception("Problem writing table '" + f + table.name + "' to the database")
+                      
+      self.logger.infoPair("Done. Imported", "%d tables" % count)
+      self.indexIppTables()
+                      
+      return True
 
     '''
@@ -467 +557 @@
            # replace nulls and empty fields with weird PSPS -999 pseudo-null
            _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
-
-           match = re.match(regex, table)
+           self.logger.infoPair('regex', regex)
+           self.logger.infoPair('table', table)
+           match = re.search(regex, table)
            newTableName = match.group(1)
 
@@ -551 +642 @@
         # for P2/ST, this reads the detection tables from the CMF/SMF file(s)
         # for OB, this imports object data from DVO
-        if not self.importIppTables():
-            if self.batchType == "OB":
-                self.logger.errorPair("skipping this batch", "no dvo tables for this region")
-                self.ippToPspsDb.updateProcessed(self.batchID, 1)
-                return True
-
-            self.logger.errorPair("Aborting this batch", "could not import IPP tables")
-            self.ippToPspsDb.updateProcessed(self.batchID, -1)
-            raise
-
+        if self.batchType == "ST":
+            self.logger.infoPair("did we get here","?")  
+            if not self.importIppSTTables():
+                self.logger.errorPair("skipping this batch", "could not import ST IPP tables")
+                self.ippToPspsDb.updateProcessed(self.batchID, -1)
+                raise
+            self.logger.infoPair("we did the ST","yes")
+        else:    
+            # if not ST we do this 
+            if not self.importIppTables():
+                if self.batchType == "OB":
+                    self.logger.errorPair("skipping this batch", "no dvo tables for this region")
+                    self.ippToPspsDb.updateProcessed(self.batchID, 1)
+                    return True
+                self.logger.errorPair("Aborting this batch", "could not import IPP tables")
+                self.ippToPspsDb.updateProcessed(self.batchID, -1)
+                raise
+        self.logger.infoPair("populatePspsTables","ok")
         if not self.populatePspsTables():
             if self.skipBatch: