Changeset 38999 for trunk/ippToPsps/jython/forcedwarpbatch.py

Timestamp:

Oct 29, 2015, 10:50:19 AM (11 years ago)

Author:

eugene

Message:

generate the table_index columns; update diff and forcedwarp to use fits_to_mysql

File:

• trunk/ippToPsps/jython/forcedwarpbatch.py (modified) (2 diffs)

trunk/ippToPsps/jython/forcedwarpbatch.py

@@ -747 +747 @@
 
         return True
-          
-
+
+    '''
+    Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTables(self, columns="*", filter=""):
+        if self.config.useOhanaMYSQLtoFITS:
+            if not self.importIppTablesOhana():
+                self.logger.errorPair("problem running", "importIppTablesOhana")
+                return False
+        else:
+            if not self.importIppTablesStilts(columns, filter):
+                self.logger.errorPair("problem running", "importIppTablesStilts")
+                return False
+
+        return True
+
+    '''
+    Imports IPP tables from FITS file
+    '''
+    def importIppTablesOhana(self):
+        
+        self.logger.infoPair("Importing DF tables", "using fits_to_mysql")
+
+        count = 0
+        self.tablesLoaded = []
+        self.haveApNpix = {}
+
+        for num in self.number:
+            forcedWarpID = self.number[num]
+            fileName = self.fits[num].getPath()
+            
+            self.logger.infoPair("loading for FW id", forcedWarpID)
+            
+            self.logger.infoPair("using filename:",fileName)
+        
+            ## output table names need to have "_num" appended.  
+            fitsOptions  = "-tablesuffix _" + str(forcedWarpID)
+
+            # this option is equivalent to the stilts "addcol table_index $0" call
+            fitsOptions += " -sequence-column table_index"
+            fitsOptions += " -sequence-column-start 1"
+
+            ## *** actually write the FITS data to mysql
+            self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions)
+
+            ## use stilts to get a list of the tables from the header
+            try:
+                tables = stilts.treads(fileName)
+            except:
+                self.logger.errorPair("STILTS could not import from", fileName)
+                return False
+        
+            # this is an optional field, check for it in the table below
+            self.haveApNpix[num] = 0
+
+            for table in tables:
+                myTable = table.name + "_" + str(forcedWarpID)
+                count = count + 1
+                      
+                self.tablesLoaded.append(myTable)
+
+            self.indexIppTables(forcedWarpID)
+                      
+        # after all tables are read for this forcedWarpID, then index the set
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        return True
 
     '''
     This function reads the cmf/smf file and loads it into the database as its own table
     '''
-    def importIppTables(self, columns="*", tableRE=""):
+    def importIppTablesStilts(self, columns="*", tableRE=""):
 
         self.tablesLoaded = []
@@ -760 +826 @@
         for num in self.number: 
             forcedWarpID = self.number[num]
-            if self.config.retry: return True
+            fileName = self.fits[num].getPath()
+
             self.logger.infoPair("loading for add id", forcedWarpID)
             self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
-            fileName = self.fits[num].getPath()
             
             self.logger.infoPair("using filename:",fileName)