Changeset 37246 for trunk/ippToPsps/jython/scratchdb.py

Timestamp:

Aug 12, 2014, 4:10:18 PM (12 years ago)

Author:

eugene

Message:

merge changes from branches/eam_branches/ipp-20140717

File:

• trunk/ippToPsps/jython/scratchdb.py (modified) (13 diffs)

trunk/ippToPsps/jython/scratchdb.py

@@ -51 +51 @@
     Sets whether we are using normal or pre-ingested DVO tables
     '''
-    def setUseFullTables(self, useFull):
-
-        if useFull:
-            self.dvoImagesTable = "dvoImagesFull"
-            self.dvoDetectionTable = "dvoDetectionFull"
-        else:
-            self.dvoImagesTable = "dvoImages"
-            self.dvoDetectionTable = "dvoDetection"
+    def setDvoTableNames(self):
+
+        self.dvoImagesTable = "dvoImagesFull"
+        self.dvoDetectionTable = "dvoDetectionFull"
 
         self.logger.debugPair("Using DVO images table", self.dvoImagesTable)
@@ -135 +131 @@
         return filterID
 
+    '''
+    Gets tess ID for this tess name
+    '''
+    def getTessID(self, name):
+     
+        sql = "SELECT tessID FROM TessellationType WHERE name = '" + name + "'"
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            tessID = rs.getInt(1)
+        except:
+            self.logger.errorPair("No tess ID found for tess", name)
+            tessID = -1
+
+        return tessID
+
     ''' 
     Gets imageID from extern ID 
@@ -244 +256 @@
                 return True
             else:
+                self.logger.infoPair("Did NOT already import from",  path)
                 return False
         except:
+            print "***** exception in alreadyImportedThisDvoTable *****"
             self.logger.exception("Unable to check whether this DVO table has been imported")
-
 
     '''
@@ -323 +336 @@
     def isCorrectDvo(self, path):
 
+        sql = "show tables"
+        try:
+            rs = self.executeQuery(sql)  
+            haveDone = False
+            while (rs.next()):
+                tableName = rs.getString(1)
+                # print "table: " + tableName
+                if tableName == self.dvoDoneTable:
+                    haveDone = True
+                    
+            rs.close()
+            if not haveDone:
+                self.logger.info("dvoDone not yet built")
+                return False
+
+        except Exception, e:
+            print str(e)
+            self.logger.errorPair("problem examining table", sql)
+            sys.exit(1)
+
         sql = "SELECT path FROM " + self.dvoDoneTable + " WHERE path LIKE '%SkyTable.fits'"
-        try:
-            rs = self.executeQuery(sql)  
-            rs.first()
-
+        try: rs = self.executeQuery(sql)  
+        except Exception, e:
+            print str(e)
+            self.logger.errorPair("problem querying for dvo path", path)
+            sys.exit(1)
+
+        if not rs.next():
+            self.logger.info("dvoDone exists, but is empty")
+            return False
+
+        try:
             head,tail = os.path.split(rs.getString(1))
 
@@ -334 +374 @@
                 return True
             else: 
-                self.logger.errorPair("DVO does not match. We have", head)
+                self.logger.infoPair("DVO does not match. We have", head)
                 return False
 
         except Exception, e:
             print str(e)
-            self.logger.errorPair("No DVO files ingested for this path", path)
-            return False
+            self.logger.errorPair("problem searching for DVO at", path)
+            sys.exit(1)
 
     '''
@@ -387 +427 @@
             # if no returned rows, return an error
             if not rs.next():
-                print "DVO lacks imageID ", imageID
-                print "sql: ", sql
+                self.logger.infoPair("DVO lacks imageID ", imageID)
                 return False
 
@@ -396 +435 @@
 
             else:
-                print "DVO has wrong imageID? ", imageID
-                print "result was ", rs.getInt(1)
-                print "sql: ", sql
+                self.logger.error("DVO has wrong imageID? ")
+                self.logger.error("imageID: ", imageID)
+                self.logger.error("EXTERN_ID: ", rs.getInt(1))
+                self.logger.infoPair("sql: ", sql)
                 raise
 
@@ -415 +455 @@
     
         dvoImagesTable = "dvoImages"
-        dvoDetectionTable = "dvoDetections"
+        dvoDetectionTable = "dvoDetection"
 
         # drop and create Images table
-        self.logger.debugPair("Creating DVO table", dvoImagesTable)
+        self.logger.infoPair("Creating DVO table", dvoImagesTable)
         sql = "DROP TABLE " + dvoImagesTable
 
@@ -438 +478 @@
 
         # now detection table
-        self.logger.debugPair("Creating DVO table", dvoDetectionTable)
+        self.logger.infoPair("Dropping DVO table", dvoDetectionTable)
         sql = "DROP TABLE " + dvoDetectionTable
         try: self.execute(sql)
-        except: pass
-
-        self.logger.debugPair("Creating DVO table", "dvoDetection")
+        except: 
+            self.logger.errorPair("problem dropping DVO table", dvoDetectionTable)
+            pass
+
+        self.logger.infoPair("Creating DVO table", "dvoDetection")
         sql = "CREATE TABLE dvoDetection ( \
-               imageID INT, \
-               ippDetectID BIGINT, \
+               objID BIGINT, \
                detectID BIGINT, \
                ippObjID BIGINT, \
-               objID BIGINT, \
-               flags INT, \
-               zp FLOAT, \
-               zpErr FLOAT, \
-               airMass FLOAT, \
-               expTime FLOAT, \
+               ippDetectID BIGINT, \
+               imageID INT, \
+               catID INT, \
                ra DOUBLE, \
                dec_ DOUBLE, \
                raErr FLOAT, \
                decErr FLOAT, \
+               zp FLOAT, \
+               telluricExt FLOAT, \
+               airmass FLOAT, \
+               expTime FLOAT, \
+               Mpsf FLOAT, \
+               dMpsf FLOAT, \
+               Mkron FLOAT, \
+               dMkron FLOAT, \
+               Map FLOAT, \
+               dMap FLOAT, \
+               flags INT, \
                PRIMARY KEY (imageID, ippDetectID), \
                KEY (objID, detectID) \
@@ -466 +515 @@
         except: 
             self.logger.error("Unable to create DVO detection database table")
+            self.sys.exit(1)
 
         # XXX EAM : for the parallel mode, dvoDetection requires duplicate objID values!
@@ -511 +561 @@
 
        # we need a big enough memory table for the incoming detections
-       print "requested memory in gigs: ", self.memoryTableSize
+       # print "requested memory in gigs: ", self.memoryTableSize
        tableSize = int(self.memoryTableSize) * 1024 * 1024 * 1024
+       self.logger.infoPair("setting max heap size: ", tableSize)
+
        sql = "set @@max_heap_table_size = " + str(tableSize)
-       print "setting max heap size: ", sql
        try: self.execute(sql)
        except: 
@@ -550 +601 @@
        self.logger.infoPair("Dropping table", self.dvoSkyTable)
        self.dropTable(self.dvoSkyTable) 
-       self.logger.infoPair("Dropping table", self.dvoDetectionTable)
-       self.dropTable(self.dvoDetectionTable)
        self.logger.infoPair("Dropping table", self.dvoDoneTable)
        self.dropTable(self.dvoDoneTable) 
+
+       # blow away existing dvoDetection table & re-crate
+       if not self.resetDvoDetectionTable():
+           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
+           return False
 
        # create dvoDone table
@@ -564 +618 @@
 
        self.changeEngineToInnoDB(self.dvoDoneTable)
-
-       # blow away existing dvoDetection table & re-crate
-       if not self.resetDvoDetectionTable():
-           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
-           return False
 
        return True