Changeset 31994 for trunk/ippToPsps

Timestamp:

Aug 3, 2011, 11:57:31 AM (15 years ago)

Author:

rhenders

Message:

lots of improvements to logging; improved check that user really wants to delete DVO tables; constructor now takes config doc as arg; first arg to program is path to config; dangerous reset option is now a second option

File:

• trunk/ippToPsps/jython/dvoToMySQL.py (modified) (11 diffs)

trunk/ippToPsps/jython/dvoToMySQL.py

@@ -27 +27 @@
 
     '''
-    def __init__(self, logger, RESETTABLES):
+    def __init__(self, logger, doc, RESETTABLES):
 
         # set up logging
         self.logger = logger
+        self.doc = doc
         self.logger.infoSeparator()
-        self.logger.info("Started dvoToMySql")
-        self.dvoMetaTable = "dvoMetaFull"
-        self.dvoDetectionTable = "dvoDetectionFull"
-
-        # open config
-        self.doc = ElementTree(file="config.xml")
         self.dvoLocation = self.doc.find("dvo/location").text
 
+        # create database object
+        self.scratchDb = ScratchDb(logger, self.doc, 1)
+
         # some log stuff
-        self.logger.infoPair("Using DVO meta table", self.dvoMetaTable)
-        self.logger.infoPair("Using DVO detection table", self.dvoDetectionTable)
+        self.logger.infoPair("Started", "dvoToMySql")
         self.logger.infoPair("Importing DVO database at", self.dvoLocation)
-
-        # create database objects
-        self.scratchDb = ScratchDb(logger, self.doc, 1)
 
         # create DVO tables
@@ -52 +46 @@
 
         # import Images.dat table
-        self.logger.infoPair("Deleting from table", self.dvoMetaTable)
-        sql = "DELETE FROM dvoMetaFull"
+        self.logger.infoPair("Deleting from table", self.scratchDb.dvoMetaTable)
+        sql = "DELETE FROM " + self.scratchDb.dvoMetaTable
         self.scratchDb.execute(sql)
 
@@ -65 +59 @@
         # NB what we and smf files call IMAGE_ID, DVO calls EXTERN_ID. We are sticking
         # with the smf name
-        self.logger.info("Inserting all image meta data into database")
-        sql = "INSERT INTO dvoMetaFull ( \
+        self.logger.infoPair("Populating", "all image meta data")
+        sql = "INSERT INTO " + self.scratchDb.dvoMetaTable + " ( \
                sourceID, \
                imageID, \
@@ -93 +87 @@
                 # get just filename, without extension
                 file = os.path.basename(os.path.splitext(file)[0])
-                self.logger.info("---------------------------------------------: " + file)
+                self.logger.infoSeparator()
 
                 if self.scratchDb.alreadyImportedThisDvoTable(file): continue
@@ -116 +110 @@
       
                 # shove SOURCE_IDs into measurement table
-                self.logger.info("Adding SOURCE_IDs into measurements table")
+                self.logger.infoPair("Adding", "SOURCE_IDs")
                 sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (SOURCE_ID SMALLINT)"
                 self.scratchDb.execute(sql)
@@ -125 +119 @@
 
                 # shove PSPS objID in measurement table
-                self.logger.info("Adding PSPS objID into measurements table")
+                self.logger.infoPair("Adding","PSPS objIDs")
                 sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (PSPS_OBJ_ID BIGINT)"
                 self.scratchDb.execute(sql)
@@ -138 +132 @@
                 # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary 
                 # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest
-                self.logger.infoPair("Populating", self.dvoDetectionTable)
-                sql = "INSERT IGNORE INTO dvoDetectionFull (\
+                self.logger.infoPair("Populating", self.scratchDb.dvoDetectionTable)
+                sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\
                        sourceID \
                        ,imageID \
@@ -163 +157 @@
 
                 # now drop what we don't need
-                self.logger.info("Dropping tables")
+                self.logger.infoPair("Dropping table", cpmTableName)
                 self.scratchDb.dropTable(cpmTableName)
+                self.logger.infoPair("Dropping table", cptTableName)
                 self.scratchDb.dropTable(cptTableName)
        
@@ -199 +194 @@
       for table in tables:
 
-          self.logger.info("Reading IPP table " + table.name + " from FITS file")
+          self.logger.infoPair("Reading IPP table", table.name)
           table = stilts.tpipe(table, cmd='explodeall')
      
           # IPP FITS files are littered with infinities, so remove these
-          self.logger.info("Removing Infinity values from all columns")
+          self.logger.infoPair("Removing", "infinity values")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
 
           #try:
-          self.logger.info("Writing FITS table to database")
+          self.logger.infoPair("Writing FITS table to", "database")
           table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
           #except:
@@ -218 +213 @@
       return tableName
 
+
+'''
+Start of program.
+'''
+
+if len(sys.argv) > 1: CONFIG  = sys.argv[1]
+else:
+    print "** Usage: " + sys.argv[0] + " <configPath> [reset]"
+    sys.exit(1)
+
+# open config file
+configDoc = ElementTree(file=CONFIG)
+
+
 logging.config.fileConfig("logging.conf")
 logging.setLoggerClass(PSLogger)
@@ -225 +234 @@
 RESETTABLES = 0
 
-if len(sys.argv) > 1 and sys.argv[1] == "reset": 
+if len(sys.argv) > 2 and sys.argv[2] == "reset": 
     response = raw_input("* Are you ABSOLUTELY sure you want to recreate the DVO tables (y/n)? ")
     if response == "y": RESETTABLES = 1
-
-dvoToMySql = DvoToMySql(logger, RESETTABLES)
+    else: sys.exit(1)
+
+dvoToMySql = DvoToMySql(logger, configDoc, RESETTABLES)
 
 logger.infoPair("Program...", "complete")