Changeset 38973

Timestamp:

Oct 27, 2015, 2:38:19 PM (11 years ago)

Author:

eugene

Message:

add debug option to ippjython / ipptopsps.py; move some messages to debug; only do XY01 in test mode; test ingest of a FITS file to mysql

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (1 diff)
• config.py (modified) (3 diffs)
• detectionbatch.py (modified) (7 diffs)
• gpc1db.py (modified) (1 diff)
• pslogger.py (modified) (2 diffs)
• testCode.py (modified) (3 diffs)

trunk/ippToPsps/jython/batch.py

@@ -67 +67 @@
                 logger.errorPair("Could not read FITS PHU for id", "%d" % id)
                 raise
+        else:
+            logger.errorPair("missing fits item?", str(id))
 
         # define the dvo table names

trunk/ippToPsps/jython/config.py

@@ -28 +28 @@
         self.onebatch = False
         self.loopBatch = False
+        self.loggerLevel = logging.INFO
 
         if sys.argv.count("-test"):
@@ -66 +67 @@
             self.loopBatch = True
             sys.argv.remove("-loop")
+
+        if sys.argv.count("-quiet"):
+            self.loggerLevel = logging.WARNING
+            sys.argv.remove("-quiet")
+
+        if sys.argv.count("-debug"):
+            self.loggerLevel = logging.DEBUG
+            sys.argv.remove("-debug")
 
         ## name of the top-level jython script
@@ -121 +130 @@
         self.logger = logging.getLogger(self.programName)
         if (self.name is None):
-            self.logger.setup(self.programName, self.logPath, "none", host, pid, stdout, sendToFile)
+            self.logger.setup(self.programName, self.logPath, "none", host, pid, self.loggerLevel, stdout, sendToFile)
         else:
-            self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile)
+            self.logger.setup(self.programName, self.logPath, self.name, host, pid, self.loggerLevel, stdout, sendToFile)
 
         return self.logger

trunk/ippToPsps/jython/detectionbatch.py

@@ -341 +341 @@
         self.scratchDb.execute(sql)
 
-        print "---- inserted ImageMeta into mysql"
+        self.logger.debugPair("inserted into mysql", "ImageMeta")
 
         self.scratchDb.updateRecalStats(tableName, header['IMAGEID'], str(pltscale));
@@ -347 +347 @@
         self.scratchDb.updateAllRows(tableName, "processingVersion", str(self.skychunk.processingVersion))
 
-        print "---- updated things"
-
         if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
 
-        print "---- completed ImageMeta"
+        self.logger.debugPair("completed", "ImageMeta")
         return True
 
@@ -691 +689 @@
                 for y in range(self.startY, self.endY):
                     
+                    ## for a quick test, just do the first chip:
+                    if self.config.test and self.config.camera == "gpc1":
+                        if x != 0 or y != 1: continue
+
                     # dodge the corners
                     if x==0 and y==0: continue
@@ -717 +719 @@
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
+
+                ## for a quick test, just do the first chip:
+                if self.config.test and self.config.camera == "gpc1":
+                    if x != 0 or y != 1: continue
 
                 # dodge the corners
@@ -927 +933 @@
             for y in range(self.startY, self.endY):
                
+                ## for a quick test, just do the first chip:
+                if self.config.test and self.config.camera == "gpc1":
+                    if x != 0 or y != 1: continue
+
                 # dodge the corners
                 if x==0 and y==0: continue
@@ -1050 +1060 @@
     def importIppTables(self, filter=""):
         
-        regex = ".*.psf"
+        ## XXX does the regex work?
+        regex = "XY*"
     
-        if False and self.config.test and self.config.camera == "gpc1":
-            regex = "XY01.psf"
+        ## for a quick test, just do the first chip:
+        if self.config.test and self.config.camera == "gpc1":
+            regex = "XY01*"
             
         print "my ID: " + str(self.id)
@@ -1066 +1078 @@
             #### crap crap crap
 
-        regex = ".*"
+        ## this is apparently over-riding the choices above
+        ## regex = ".*"
+
+        ## the list of columns above works fine for *.psf, but breaks the *.deteff tables
+        ## just read all columns
         columns = "*"
 

trunk/ippToPsps/jython/gpc1db.py

@@ -170 +170 @@
                    GROUP BY stage_id"
 
-        try:
-            rs = self.executeQuery(sql)
-
-        except:
-            self.logger.exception("Can't query for ids in DVO")
-            self.logger.infoPair("failed sql:", sql)
-            sys.exit(1)
+        self.logger.debug("SQL: " + sql)
+        rs = self.executeQuery(sql)
 
         while (rs.next()): 

trunk/ippToPsps/jython/pslogger.py

@@ -11 +11 @@
    This will default to stout and no file output
    '''
-   def setup(self, programName, basePath, skychunkName, host, pid, stdout=1, sendToFile=0):
+   def setup(self, programName, basePath, skychunkName, host, pid, loggerLevel, stdout=1, sendToFile=0):
 
        formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
-       self.setLevel(logging.INFO)
+       self.setLevel(loggerLevel)
 
        if sendToFile:
@@ -34 +34 @@
        if sendToFile: print "Writing log to: " + FULLPATH
 
+   def debugPair(self, first, second):
+       self.debug("%-40s%s" % (first, second))
+
    def infoPair(self, first, second):
        self.info("%-40s%s" % (first, second))
 
+   def warningPair(self, first, second):
+       self.warning("%-40s%s" % (first, second))
+
    def errorPair(self, first, second):
        self.error("%-40s%s" % (first, second))
-
-   def debugPair(self, first, second):
-       self.debug("%-40s%s" % (first, second))
 
    def infoTitle(self, str):

trunk/ippToPsps/jython/testCode.py

@@ -5 +5 @@
 # import time
 import sys
-# import os
+import os
 import math
-# import logging.config
 
-# from sqlUtility import sqlUtility
+import stilts
 
 # testCode class
@@ -18 +17 @@
         super(testCode, self).__init__(argv)
 
-    def run(self):
+    def connectMysql(self, argv):
 
-        # tables = stilts.treads(fileName)
+        if len(sys.argv) != 6:
+            print "USAGE: ippjython testCode.py (dbHost) (dbUser) (dbPass) (dbName) (filename)"
+            os._exit(2)
 
-        for x in range(0, 8):
-            for y in range(0, 8):
+        # open config and grab database parameters
+        self.dbHost = sys.argv[1]
+        self.dbUser = sys.argv[2]
+        self.dbPass = sys.argv[3]
+        self.dbName = sys.argv[4]
 
-                if not ((x == 0) and (y == 1)):
-                    print "case 1"
-                    print "x : " + str(x)
-                    print "y : " + str(y)
-                else:
-                    print "case 2"
-                    print "x : " + str(x)
-                    print "y : " + str(y)
+        self.filename = sys.argv[5]
+
+        # url for mysql / stilts
+        self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?autoReconnect=true&user="+self.dbUser+"&password="+self.dbPass
+
+        ## no need to set up the connection for this test
+        ## try:
+        ##     self.con = DriverManager.getConnection(self.url)
+        ## except:
+        ##     raise
+
+    '''
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTables(self):
+
+        try:
+            tables = stilts.treads(self.filename)
+        except:
+            print "STILTS could not import from " + filename
+            os._exit(3)
+
+        for table in tables:
+
+            print "import smf table " + table.name
+
+            # need to generate an index on the IPP_IDET column
+            # table = stilts.tpipe(table, cmd='addcol table_index $0')
+            table = stilts.tpipe(table, cmd='explodeall')
+
+            print "read smf table " + table.name
+
+            # drop any previous tables before import
+            # self.scratchDb.dropTable(table.name)
+
+            # limit to specific columns
+            # table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+
+            # IPP FITS files are littered with infinities, so remove these
+            print "Removing Infinity values from all columns"
+            table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+            table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+
+            try:
+                table.write(self.url + '#' + table.name)
+            except:
+                print "Problem writing table '" + table.name + "' to the database"
+                os._exit(4)
+
+        return True
+
+    def run(self, argv):
+
+        self.connectMysql(argv)
+        self.importIppTables()
+
+        ## for x in range(0, 8):
+        ##     for y in range(0, 8):
+        ## 
+        ##         if not ((x == 0) and (y == 1)):
+        ##             print "case 1"
+        ##             print "x : " + str(x)
+        ##             print "y : " + str(y)
+        ##         else:
+        ##             print "case 2"
+        ##             print "x : " + str(x)
+        ##             print "y : " + str(y)
 
 '''
@@ -39 +102 @@
 try:
     testcode = testCode(sys.argv)
-    testcode.run()
-    testcode.exitProgram("completed")
+    testcode.run(sys.argv)
 except: pass