Changeset 37138 for branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py

Timestamp:

Jul 29, 2014, 12:16:23 PM (12 years ago)

Author:

eugene

Message:

move importIppSTTables from batch.py to stackbatch.py and rename importIppTables; make batch:run call importIppTables consistently for all batch types

File:

• branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (modified) (6 diffs)

branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py

@@ -75 +75 @@
  #      self.logger.infoPair("y stackID ",ystackID)
  #      raise
-       # from each stackget fits name from getStackStageCmf
+       # from each stack, get fits name from getStackStageCmf
        if gstackID > 0:
            self.logger.infoPair ("finding fits file for: ","g.00000")
@@ -141 +141 @@
 
        metastackid = -999
-       if gstackID > 0:
-           metastackid = gstackID
-       if rstackID > 0:
-           metastackid = rstackID
-       if istackID > 0:
-           metastackid = istackID    
-       if zstackID > 0:
-           metastackid = zstackID
-       if ystackID > 0:
-           metastackid = ystackID
+       if gstackID > 0: metastackid = gstackID
+       if rstackID > 0: metastackid = rstackID
+       if istackID > 0: metastackid = istackID    
+       if zstackID > 0: metastackid = zstackID
+       if ystackID > 0: metastackid = ystackID
+
        #there is probably a better way to do this
        if metastackid < 0:
@@ -159 +155 @@
        ## this should be ok - it takes a stack_id and returns filter/skycell etc.
 
-
        if not meta:
            self.logger.errorPair("Could not get stack", "metadata")
@@ -174 +169 @@
        # we need 1133.081 for skycell, but store as an int, so instead store 1133081
        # projection cell is just the 1133 bit
+
        # This Is how it should be done
        #self.projectioncell = self.skycell[8:12]
@@ -217 +213 @@
 
        self.scratchDb.dropTable("StackToImage")
-             # self.testprint()
+       # self.testprint()
 
        # for PV2 we will have (grizy) of stack meta and skychip      
-
 
        # delete IPP tables
@@ -1335 +1330 @@
     def exportPspsTablesToFits(self, regex="(.*)"):
         return super(StackBatch, self).exportPspsTablesToFits("(Stack.*)")
+
+    '''
+    Imports IPP tables from FITS file
+    Accepts a regular expression to match the tables so not all tables need to be imported
+    (This regex feature is not currently used...)
+    '''
+    def importIppTables(self, columns="*", tableRE=""):
+
+      self.logger.infoPair("Importing ST tables with table match expression: ", tableRE)
+
+      # print "trying to read ", self.fits.getPath()
+
+      # ST has 5 cmf files - all with the same table names. To counter this we do it like this.
+
+      filters = ["g","r","i","z","y"]
+      count = 0
+      for f in filters:
+          skip = 0
+          if (f == "g"):
+              if self.gstackID > 0:  
+                  fileName = self.gfits.getPath()
+              else:
+                  skip = 1
+          if (f == "r"):
+              if self.rstackID > 0:
+                  fileName = self.rfits.getPath()
+              else:
+                  skip = 1
+          if (f == "i"):
+              if self.istackID > 0:
+                  fileName = self.ifits.getPath()
+              else:
+                  skip = 1
+          if (f == "z"):
+              if self.zstackID > 0:
+                  fileName = self.zfits.getPath()
+              else:
+                  skip = 1
+          if (f == "y"):
+              if self.ystackID > 0:
+                  fileName = self.yfits.getPath()
+              else:
+                  skip = 1
+
+          if skip == 1:
+              self.logger.infoPair("no files for filter" , f) 
+          else:
+             self.logger.infoPair("using filename:",fileName)
+          
+             try:
+              tables = stilts.treads(fileName)
+             except:
+              self.logger.errorPair("STILTS could not import from", fileName)
+              return False
+              
+             #count = 0
+             for table in tables:
+              
+              # print "import smf table ", table
+              match = re.match(tableRE, table.name)
+              
+              if not match: continue
+              self.logger.infoPair("Reading IPP table", f + table.name)
+              table = stilts.tpipe(table, cmd='addcol table_index $0')
+                  
+              table = stilts.tpipe(table, cmd='explodeall')
+                  
+              # print "read smf table ", table
+                  
+              # drop any previous tables before import
+              self.scratchDb.dropTable(f + table.name)
+                  
+              # IPP FITS files are littered with infinities, so remove these
+              self.logger.debug("Removing Infinity values from all columns")
+              table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+              table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+              table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+              # print "cleaned up values ", table
+                  
+              try:
+                  table.write(self.scratchDb.url + '#' + f + table.name)
+                  count = count + 1
+              except:
+                  self.logger.exception("Problem writing table '" + f + table.name + "' to the database")
+                      
+      self.logger.infoPair("Done. Imported", "%d tables" % count)
+      self.indexIppTables()
+                      
+      return True