Changeset 38995 for trunk

Timestamp:

Oct 28, 2015, 8:29:33 PM (11 years ago)

Author:

eugene

Message:

add -stilts option; fix stilts and regex for detectionbatch; add tableprefix option for stacks; alter photcode.type to type_as_int

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (5 diffs)
• config.py (modified) (2 diffs)
• detectionbatch.py (modified) (3 diffs)
• dvo.py (modified) (2 diffs)
• scratchdb.py (modified) (2 diffs)
• stackbatch.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -453 +453 @@
     '''
     def importIppTables(self, columns="*", filter=""):
-        if self.useOhanaMYSQLtoFITS:
+        if self.config.useOhanaMYSQLtoFITS:
             if not self.importIppTablesOhana():
                 self.logger.errorPair("problem running", "importIppTablesOhana")
@@ -480 +480 @@
             return False
 
-        self.scratchDb.loadFITStoMYSQL(self.fits.getPath(), "none")
+        self.scratchDb.loadFITStoMYSQL(self.fits.getPath())
 
         ## may need to do this with direct sql:
@@ -515 +515 @@
     def importIppTablesStilts(self, columns="*", filter=""):
         
-        self.logger.infoPair("Importing tables with filter", filter)
+        self.logger.infoPair("Importing tables (stilts) with filter", filter)
         
-        # print "trying to read ", self.fits.getPath()
+        print "trying to read ", self.fits.getPath()
 
         try:
@@ -527 +527 @@
         self.tablesLoaded = []
 
+        print "read tables from ", self.fits.getPath()
+
         count = 0
         for table in tables:
 
             # example of listing all columns in a table:
-            ## print "-----table: " + table.name
-            ## columns = table.columns()
-            ## for column in columns:
-            ##     print column
-            ## 
-            ## print "-----"
-
-            # print "import smf table ", table
+            print "-----table: " + table.name
+            myColumns = table.columns()
+            for column in myColumns:
+                print column
+            print "-----"
+
+            print "import smf table ", table.name
+
             match = re.match(filter, table.name)
             if not match: continue
@@ -555 +557 @@
             self.logger.debug("Removing Infinity values from all columns")
             table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+            print "-----"
             table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+            print "-----"
             table = stilts.tpipe(table, cmd='replaceval Infinity null *')
-            # print "cleaned up values ", table
+            print "cleaned up values ", table.name
 
             try:

trunk/ippToPsps/jython/config.py

@@ -21 +21 @@
     def __init__(self):
 
-        self.useOhanaMYSQLtoFITS = False
+        self.useOhanaMYSQLtoFITS = True
         self.test = False
         self.resetDvo = False
@@ -41 +41 @@
         # by default, we use gpc1test in a test, and do not if not a test
         self.gpc1test = self.test
+
+        if sys.argv.count("-stilts"):
+            self.useOhanaMYSQLtoFITS = False
+            sys.argv.remove("-stilts")
 
         if sys.argv.count("-realgpc"):

trunk/ippToPsps/jython/detectionbatch.py

@@ -844 +844 @@
         self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
         self.logger.info("updated dvoids")
+
         results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
         self.logger.info("deleted nulls")
@@ -1060 +1061 @@
     def importIppTables(self, filter=""):
         
-        ## XXX does the regex work?
-        regex = "XY*"
+        ## regex only works for -stilts import
+        regex = ".*"
     
         ## for a quick test, just do the first chip:
@@ -1076 +1077 @@
             columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
 
-            #### crap crap crap
-
-        ## this is apparently over-riding the choices above
-        ## regex = ".*"
-
         ## the list of columns above works fine for *.psf, but breaks the *.deteff tables
         ## just read all columns

trunk/ippToPsps/jython/dvo.py

@@ -590 +590 @@
     '''
     def importPhotcodes(self, path, tableName):
-        if True:
+        if self.config.useOhanaMYSQLtoFITS:
             self.importPhotcodesOhana(path,tableName)
         else:
@@ -607 +607 @@
       # table = stilts.tpipe(table, cmd='addcol -after TYPE TYPE_AS_INT "TYPE + 0"')
 
-      self.scratchDb.loadFITStoMYSQL (path, tableName)
+      self.scratchDb.loadFITStoMYSQL (path, "-tablename " + tableName)
+
+      sql = "ALTER TABLE " + tableName + " ADD COLUMN TYPE_AS_INT INT AFTER TYPE"
+      self.scratchDb.execute(sql)
+
+      sql = "UPDATE " + tableName + " SET TYPE_AS_INT = ascii(TYPE)"
+      self.scratchDb.execute(sql)
 
       self.logger.debugPair("Finished importing", tableName)

trunk/ippToPsps/jython/scratchdb.py

@@ -809 +809 @@
     Use Ohana fits_to_mysql to load FITS tables into scratch db
     '''
-    def loadFITStoMYSQL(self, filename, tablename):
+    def loadFITStoMYSQL(self, filename, tablename=""):
 
         self.logger.infoPair("starting loadFITStoMYSQL: ", filename)
@@ -819 +819 @@
         cmd += " -dbuser " + self.dbUser
         cmd += " -dbpass " + self.dbPass
+        cmd += " " + tablename
 
         self.logger.debugPair("preparing command: ", cmd)
-
-        if tablename != "none":
-            cmd += " -tablename " + tablename
 
         (stdoutFile, stdoutName) = tempfile.mkstemp(prefix="fits_to_mysql.stdout.", dir="/tmp")

trunk/ippToPsps/jython/stackbatch.py

@@ -1387 +1387 @@
     '''
     Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTables(self, columns="*", filter=""):
+        if self.config.useOhanaMYSQLtoFITS:
+            if not self.importIppTablesOhana():
+                self.logger.errorPair("problem running", "importIppTablesOhana")
+                return False
+        else:
+            if not self.importIppTablesStilts(columns, filter):
+                self.logger.errorPair("problem running", "importIppTablesStilts")
+                return False
+
+        return True
+
+    '''
+    Imports IPP tables from FITS file
+    Accepts a regular expression to match the tables so not all tables need to be imported
+    '''
+    def importIppTablesOhana(self, columns="*", tableRE=""):
+        
+        self.logger.infoPair("Importing ST tables using ", "Ohana")
+
+        self.tablesLoaded = []
+
+        count = 0
+        for filter in self.filters:
+            if self.stackIDs[filter] <= 0:  
+                self.logger.infoPair("no files for filter" , filter) 
+                continue
+
+            fileName = self.fits[filter].getPath()
+            
+            self.logger.infoPair("using filename:",fileName)
+            
+            ## output table names need to have "filter" prepended.  
+            ## add this as an option
+            self.scratchDb.loadFITStoMYSQL(fileName, "-tableprefix " + filter)
+            ## XXX table = stilts.tpipe(table, cmd='addcol table_index $0')
+
+            # get the list of tables in this file:
+            try:
+                tables = stilts.treads(fileName)
+            except:
+                self.logger.errorPair("STILTS could not import from", fileName)
+                return False
+            
+            for table in tables:
+                self.logger.debugPair("Read IPP table", table.name)
+                myTable = filter + table.name.replace(".", "_")
+                self.tablesLoaded.append(myTable)
+                count += 1
+
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        self.indexIppTables()
+
+        return True
+
+    '''
+    Imports IPP tables from FITS file
     Accepts a regular expression to match the tables so not all tables need to be imported
     (This regex feature is not currently used...)
     '''
-    def importIppTables(self, columns="*", tableRE=""):
+    def importIppTablesStilts(self, columns="*", tableRE=""):
         
         if self.config.retry: return True