Changeset 38999

Timestamp:

Oct 29, 2015, 10:50:19 AM (11 years ago)

Author:

eugene

Message:

generate the table_index columns; update diff and forcedwarp to use fits_to_mysql

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (2 diffs)
• diffbatch.py (modified) (2 diffs)
• forcedwarpbatch.py (modified) (2 diffs)
• stackbatch.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -466 +466 @@
     '''
     Imports IPP tables from FITS file
-
-    Accepts a regular expression filter so not all tables need to be imported
     '''
     def importIppTablesOhana(self):
@@ -473 +471 @@
         self.logger.infoPair("Importing IPP tables", "using fits_to_mysql")
 
+        count = 0
+        self.tablesLoaded = []
+
+        fileName = self.fits.getPath()
+
+        # this option is equivalent to the stilts "addcol table_index $0" call
+        fitsOptions  = " -sequence-column table_index"
+        fitsOptions += " -sequence-column-start 1"
+
+        ## *** actually write the FITS data to mysql
+        self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions)
+
         ## use stilts to get a list of the tables from the header
         try:
-            tables = stilts.treads(self.fits.getPath())
+            tables = stilts.treads(fileName)
         except:
-            self.logger.errorPair("STILTS could not import from", self.fits.getPath())
-            return False
-
-        self.scratchDb.loadFITStoMYSQL(self.fits.getPath())
-
-        ## may need to do this with direct sql:
-        # table = stilts.tpipe(table, cmd='addcol table_index $0')
-
-        self.tablesLoaded = []
-
-        count = 0
+            self.logger.errorPair("STILTS could not import from", fileName)
+            return False
+
         for table in tables:
-
             # example of listing all columns in a table:
             ## print "-----table: " + table.name

trunk/ippToPsps/jython/diffbatch.py

@@ -564 +564 @@
     '''
     Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTables(self, columns="*", filter=""):
+        if self.config.useOhanaMYSQLtoFITS:
+            if not self.importIppTablesOhana():
+                self.logger.errorPair("problem running", "importIppTablesOhana")
+                return False
+        else:
+            if not self.importIppTablesStilts(columns, filter):
+                self.logger.errorPair("problem running", "importIppTablesStilts")
+                return False
+
+        return True
+
+    '''
+    Imports IPP tables from FITS file
+    '''
+    def importIppTablesOhana(self):
+        
+        self.logger.infoPair("Importing DF tables", "using fits_to_mysql")
+
+        count = 0
+        self.tablesLoaded = []
+        self.haveApNpix = {}
+
+        for num in self.number:
+            diffSkyFileID = self.number[num]
+            fileName = self.fits[num].getPath()
+            
+            self.logger.infoPair("using filename:",fileName)
+        
+            ## output table names need to have "_num" appended.  
+            fitsOptions  = "-tablesuffix _" + str(num)
+
+            # this option is equivalent to the stilts "addcol table_index $0" call
+            fitsOptions += " -sequence-column table_index"
+            fitsOptions += " -sequence-column-start 1"
+
+            ## *** actually write the FITS data to mysql
+            self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions)
+
+            ## use stilts to get a list of the tables from the header
+            try:
+                tables = stilts.treads(fileName)
+            except:
+                self.logger.errorPair("STILTS could not import from", fileName)
+                return False
+        
+            # this is an optional field, check for it in the table below
+            self.haveApNpix[num] = 0
+
+            for table in tables:
+                # check for AP_NPIX columns: missing for SAS39
+                if table.name == "SkyChip_psf":
+                    columns = table.columns()
+                    if 'AP_NPIX' in columns:
+                        self.haveApNpix[num] = 1
+
+                myTable = table.name + "_" + str(num)
+                count = count + 1
+                      
+                self.tablesLoaded.append(myTable)
+
+            # after all tables are read for this 'num', then index the set
+            self.indexIppTables(num)
+
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        return True
+
+    '''
+    Imports IPP tables from FITS file
     Accepts a regular expression to match the tables so not all tables need to be imported
     (This regex feature is not currently used...)
     '''
-    def importIppTables(self, columns="*", tableRE=""):
+    def importIppTablesStilts(self, columns="*", tableRE=""):
         
         if self.config.retry: return True
@@ -632 +704 @@
         return True
 
-
-
-
-
-
     '''
     Overriding this method. Use regex to trim off, eg _XY33 extension

trunk/ippToPsps/jython/forcedwarpbatch.py

@@ -747 +747 @@
 
         return True
-          
-
+
+    '''
+    Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTables(self, columns="*", filter=""):
+        if self.config.useOhanaMYSQLtoFITS:
+            if not self.importIppTablesOhana():
+                self.logger.errorPair("problem running", "importIppTablesOhana")
+                return False
+        else:
+            if not self.importIppTablesStilts(columns, filter):
+                self.logger.errorPair("problem running", "importIppTablesStilts")
+                return False
+
+        return True
+
+    '''
+    Imports IPP tables from FITS file
+    '''
+    def importIppTablesOhana(self):
+        
+        self.logger.infoPair("Importing DF tables", "using fits_to_mysql")
+
+        count = 0
+        self.tablesLoaded = []
+        self.haveApNpix = {}
+
+        for num in self.number:
+            forcedWarpID = self.number[num]
+            fileName = self.fits[num].getPath()
+            
+            self.logger.infoPair("loading for FW id", forcedWarpID)
+            
+            self.logger.infoPair("using filename:",fileName)
+        
+            ## output table names need to have "_num" appended.  
+            fitsOptions  = "-tablesuffix _" + str(forcedWarpID)
+
+            # this option is equivalent to the stilts "addcol table_index $0" call
+            fitsOptions += " -sequence-column table_index"
+            fitsOptions += " -sequence-column-start 1"
+
+            ## *** actually write the FITS data to mysql
+            self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions)
+
+            ## use stilts to get a list of the tables from the header
+            try:
+                tables = stilts.treads(fileName)
+            except:
+                self.logger.errorPair("STILTS could not import from", fileName)
+                return False
+        
+            # this is an optional field, check for it in the table below
+            self.haveApNpix[num] = 0
+
+            for table in tables:
+                myTable = table.name + "_" + str(forcedWarpID)
+                count = count + 1
+                      
+                self.tablesLoaded.append(myTable)
+
+            self.indexIppTables(forcedWarpID)
+                      
+        # after all tables are read for this forcedWarpID, then index the set
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        return True
 
     '''
     This function reads the cmf/smf file and loads it into the database as its own table
     '''
-    def importIppTables(self, columns="*", tableRE=""):
+    def importIppTablesStilts(self, columns="*", tableRE=""):
 
         self.tablesLoaded = []
@@ -760 +826 @@
         for num in self.number: 
             forcedWarpID = self.number[num]
-            if self.config.retry: return True
+            fileName = self.fits[num].getPath()
+
             self.logger.infoPair("loading for add id", forcedWarpID)
             self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
-            fileName = self.fits[num].getPath()
             
             self.logger.infoPair("using filename:",fileName)

trunk/ippToPsps/jython/stackbatch.py

@@ -1423 +1423 @@
             
             ## output table names need to have "filter" prepended.  
-            ## add this as an option
-            self.scratchDb.loadFITStoMYSQL(fileName, "-tableprefix " + filter)
-            ## XXX table = stilts.tpipe(table, cmd='addcol table_index $0')
+            fitsOptions  = "-tableprefix " + filter
+
+            # this pair of options is equivalent to the stilts "addcol table_index $0" call
+            fitsOptions += " -sequence-column table_index"
+            fitsOptions += " -sequence-column-start 1"
+
+            ## *** actually write the FITS data to mysql
+            self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions)
 
             # get the list of tables in this file: