Changeset 38981

Timestamp:

Oct 27, 2015, 4:00:17 PM (11 years ago)

Author:

eugene

Message:

add options to import FITS tables using Ohana fits_to_mysql

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (1 diff)
• dvo.py (modified) (1 diff)
• scratchdb.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -446 +446 @@
         self.logger.warn("alterPspsTables not implemented")
 
+
+    '''
+    Imports IPP tables from FITS file
+
+    Accepts a regular expression filter so not all tables need to be imported
+    '''
+    def importIppTablesOhana(self):
+        
+        self.logger.infoPair("Importing IPP tables", "using fits_to_mysql")
+
+        ## use stilts to get a list of the tables from the header
+        try:
+            tables = stilts.treads(self.fits.getPath())
+        except:
+            self.logger.errorPair("STILTS could not import from", self.fits.getPath())
+            return False
+
+        loadFITStoMYSQL(self.fits.getPath(), "none")
+
+        ## may need to do this with direct sql:
+        # table = stilts.tpipe(table, cmd='addcol table_index $0')
+
+        self.tablesLoaded = []
+
+        count = 0
+        for table in tables:
+
+            # example of listing all columns in a table:
+            ## print "-----table: " + table.name
+            ## columns = table.columns()
+            ## for column in columns:
+            ##     print column
+            ## 
+            ## print "-----"
+
+            self.logger.debugPair("Read IPP table", table.name)
+
+            self.tablesLoaded.append(table.name)
+            count ++
+
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        self.indexIppTables()
+
+        return True
+
     '''
     Imports IPP tables from FITS file

trunk/ippToPsps/jython/dvo.py

@@ -581 +581 @@
 
           count = count + 1
+
+      self.logger.debugPair("Finished importing", tableName)
+
+      return tableName
+
+    '''
+    Imports the Photcode file
+    '''
+    def importPhotcodesOhana(self, path, tableName):
+
+      self.logger.infoPair("Importing file", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path))))
+      self.logger.debugPair("Writing to database table", tableName)
+
+      ## XXX may need to do this in mysql after table is read
+      # convert the 1-char field TYPE to an int for querying ease (probably more sophisticated ways to do this...)
+      # table = stilts.tpipe(table, cmd='addcol -after TYPE TYPE_AS_INT "TYPE + 0"')
+
+      loadFITStoMYSQL (path, tableName)
 
       self.logger.debugPair("Finished importing", tableName)

trunk/ippToPsps/jython/scratchdb.py

@@ -803 +803 @@
 
         return res
+
+    '''
+    Use Ohana fits_to_mysql to load FITS tables into scratch db
+    '''
+    def loadFITStoMYSQL(self, filename, tablename):
+
+        # TODO path to DVO prog hardcoded temporarily
+        cmd = "fits_to_mysql"
+        cmd += " -dbhost " + self.scratchDb.dbHost
+        cmd += " -dbname " + self.scratchDb.dbName
+        cmd += " -dbuser " + self.scratchDb.dbUser
+        cmd += " -dbpass " + self.scratchDb.dbPass
+        cmd += filename
+
+        if tablename != "none":
+            cmd += " -tablename " + tablename
+
+        (stdoutFile, stdoutName) = tempfile.mkstemp(prefix="fits_to_mysql.stdout.", dir="/tmp")
+        (stderrFile, stderrName) = tempfile.mkstemp(prefix="fits_to_mysql.stderr.", dir="/tmp")
+
+        self.logger.infoPair("Running fits_to_mysql", cmd)
+        self.logger.infoPair("Stdout to ", stdoutName)
+        self.logger.infoPair("Stderr to ", stderrName)
+        p = Popen(cmd, shell=True, stdout=stdoutFile, stderr=stderrFile)
+
+        p.wait()
+
+        if p.returncode < 0:
+            self.logger.errorPair("fits_to_mysql terminated on signal for: ", cmd)
+            os._exit(1)
+
+        if p.returncode > 0:
+            self.logger.errorPair("fits_to_mysql failed for: ", cmd)
+            os._exit(1)
+
+        os.close(loadFITStoMYSQL)
+        os.close(loadFITStoMYSQL)