Changeset 37129 for branches/eam_branches/ipp-20140717/ippToPsps/jython/scratchdb.py

Timestamp:

Jul 28, 2014, 6:20:27 AM (12 years ago)

Author:

eugene

Message:

dropping options for stilts dvo import methods; fixing a number of previously ignored errors (eg, dropping non-existent tables); allowing queue questions to strip leading and trailing spaces (so we can read from an input file); echo back the response so input from a file is clearer; re-write big sql blocks with new sql utility code; updating names in MeanObject, ObjectThin to be more consistent; add a new test code program; update detection fields to match new schema; drop the code for the old now-unused tables (detecitonCalib, SkinnyObject, etc)

Location:

branches/eam_branches/ipp-20140717/ippToPsps

Files:

• . (modified) (1 prop)
• jython/scratchdb.py (modified) (8 diffs)

branches/eam_branches/ipp-20140717/ippToPsps/jython/scratchdb.py

@@ -51 +51 @@
     Sets whether we are using normal or pre-ingested DVO tables
     '''
-    def setUseFullTables(self, useFull):
-
-        if useFull:
-            self.dvoImagesTable = "dvoImagesFull"
-            self.dvoDetectionTable = "dvoDetectionFull"
-        else:
-            self.dvoImagesTable = "dvoImages"
-            self.dvoDetectionTable = "dvoDetection"
+    def setDvoTableNames(self):
+
+        self.dvoImagesTable = "dvoImagesFull"
+        self.dvoDetectionTable = "dvoDetectionFull"
 
         self.logger.debugPair("Using DVO images table", self.dvoImagesTable)
@@ -233 +229 @@
     def alreadyImportedThisDvoTable(self, path):
 
+        print "***** alreadyImportedThisDvoTable *****"
+        print "path: " + path
+
         fileStat = os.stat(path)
+        print "path: " + path
 
         sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate = " + str(fileStat.st_mtime) 
-
-        try:
-            rs = self.executeQuery(sql)
+        print "sql: " + sql
+
+        try:
+            rs = self.executeQuery(sql)
+            print "ran sql: " + sql
             rs.first()
             if rs.getInt(1) > 0:
+                print "***** DID import previously *****"
                 self.logger.debugPair("Already imported up-to-date version of",  path)
                 return True
             else:
+                print "***** did NOT import previously *****"
                 return False
         except:
+            print "***** exception in alreadyImportedThisDvoTable *****"
             self.logger.exception("Unable to check whether this DVO table has been imported")
+
+        print "***** done with alreadyImportedThisDvoTable *****"
 
 
@@ -322 +329 @@
     '''
     def isCorrectDvo(self, path):
+
+        sql = "show tables"
+        try:
+            rs = self.executeQuery(sql)  
+            haveDone = False
+            while (rs.next()):
+                tableName = rs.getString(1)
+                # print "table: " + tableName
+                if tableName == self.dvoDoneTable:
+                    haveDone = True
+                    
+            rs.close()
+            if not haveDone:
+                print "dvoDone not yet build"
+                return False
+        except:
+            print "failed to run query: " + sql
 
         sql = "SELECT path FROM " + self.dvoDoneTable + " WHERE path LIKE '%SkyTable.fits'"
@@ -415 +439 @@
     
         dvoImagesTable = "dvoImages"
-        dvoDetectionTable = "dvoDetections"
+        dvoDetectionTable = "dvoDetection"
 
         # drop and create Images table
-        self.logger.debugPair("Creating DVO table", dvoImagesTable)
+        self.logger.infoPair("Creating DVO table", dvoImagesTable)
         sql = "DROP TABLE " + dvoImagesTable
 
@@ -438 +462 @@
 
         # now detection table
-        self.logger.debugPair("Creating DVO table", dvoDetectionTable)
+        self.logger.infoPair("Dropping DVO table", dvoDetectionTable)
         sql = "DROP TABLE " + dvoDetectionTable
         try: self.execute(sql)
-        except: pass
-
-        self.logger.debugPair("Creating DVO table", "dvoDetection")
+        except: 
+            print "problem dropping DVO " + dvoDetectionTable
+            pass
+
+        self.logger.infoPair("Creating DVO table", "dvoDetection")
         sql = "CREATE TABLE dvoDetection ( \
-               imageID INT, \
-               ippDetectID BIGINT, \
+               objID BIGINT, \
                detectID BIGINT, \
                ippObjID BIGINT, \
-               objID BIGINT, \
-               flags INT, \
-               zp FLOAT, \
-               zpErr FLOAT, \
-               airMass FLOAT, \
-               expTime FLOAT, \
+               ippDetectID BIGINT, \
+               imageID INT, \
                ra DOUBLE, \
                dec_ DOUBLE, \
                raErr FLOAT, \
                decErr FLOAT, \
+               zp FLOAT, \
+               telluricExt FLOAT, \
+               airmass FLOAT, \
+               expTime FLOAT, \
+               flags INT, \
                PRIMARY KEY (imageID, ippDetectID), \
                KEY (objID, detectID) \
@@ -466 +492 @@
         except: 
             self.logger.error("Unable to create DVO detection database table")
+            self.sys.exit(1)
 
         # XXX EAM : for the parallel mode, dvoDetection requires duplicate objID values!
@@ -550 +577 @@
        self.logger.infoPair("Dropping table", self.dvoSkyTable)
        self.dropTable(self.dvoSkyTable) 
-       self.logger.infoPair("Dropping table", self.dvoDetectionTable)
-       self.dropTable(self.dvoDetectionTable)
        self.logger.infoPair("Dropping table", self.dvoDoneTable)
        self.dropTable(self.dvoDoneTable) 
+
+       # blow away existing dvoDetection table & re-crate
+       if not self.resetDvoDetectionTable():
+           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
+           return False
 
        # create dvoDone table
@@ -564 +594 @@
 
        self.changeEngineToInnoDB(self.dvoDoneTable)
-
-       # blow away existing dvoDetection table & re-crate
-       if not self.resetDvoDetectionTable():
-           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
-           return False
 
        return True