Changeset 37754 for trunk

Timestamp:

Dec 17, 2014, 1:24:02 PM (12 years ago)

Author:

heather

Message:

final on diff batches - they work, all columns I can fill are properly filled

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (1 diff)
• diffbatch.py (modified) (6 diffs)
• gpc1db.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -382 +382 @@
         self.logger.infoPair("Min objID", "%ld" % self.minObjID)
         self.logger.infoPair("Max objID", "%ld" % self.maxObjID)
+
+
+    '''                                                                                            
+    Sets min and max obj ID using the provided table, or list of tables                            
+    '''
+    def setMinMaxDiffObjID(self, tables):
+        first = True
+        self.totalDetections = 0
+        for table in tables:
+            sql = "SELECT MIN(diffObjID), MAX(diffObjID), COUNT(diffObjID) FROM " + table
+            rs = self.scratchDb.executeQuery(sql)
+            rs.first()
+            self.totalDetections = self.totalDetections + rs.getLong(3)
+            if first:
+                self.minObjID = rs.getLong(1)
+                self.maxObjID = rs.getLong(2)
+            else:
+                if rs.getLong(1) < self.minObjID: self.minObjID = rs.getLong(1)
+                if rs.getLong(2) > self.maxObjID: self.maxObjID = rs.getLong(2)
+            first = False
+            rs.close()
+        self.ippToPspsDb.updateDetectionStats(self.batchID, self.minObjID, self.maxObjID, self.totalDetections)
+        self.logger.infoPair("Total detections", "%ld" % self.totalDetections)
+        self.logger.infoPair("Min objID", "%ld" % self.minObjID)
+        self.logger.infoPair("Max objID", "%ld" % self.maxObjID)
+
+
 
     '''

trunk/ippToPsps/jython/diffbatch.py

@@ -310 +310 @@
         sqlLine.group("a.dvoRegionID",             "b.catID")
         sqlLine.group("a.ra", "b.ra")
-        sqlLine.group("a.dec_", "b.dec_")
+        sqlLine.group("a.dec", "b.dec_")
         sqlLine.group("a.raErr", "b.raErr")
         sqlLine.group("a.decErr", "b.decErr")
@@ -326 +326 @@
             raise
         self.logger.infoPair("need to cull"," nulls objid")
+        results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection", "diffObjID")
         self.logger.infoPair("adding","uniquePSPSDFid")
-        updatePspsUniqueIDs("DiffDetection")
+        self.updatePspsUniqueIDs("DiffDetection")
 
     '''
@@ -349 +350 @@
             #self.scratchDb.execute(sql)
 
-        self.tablesToExport.append("DiffToImage")
+        #self.tablesToExport.append("DiffToImage")
 
 
@@ -424 +425 @@
     def updatePspsUniqueIDs(self,table):
         sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
-        (select @r:=0) r, "+table+" t) as foo using (objID) set \
+        (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
         uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
         try: self.scratchDb.execute(sql)
@@ -456 +457 @@
 
         
-        self.setMinMaxObjID("DiffDetection")
-
-        # update FrameMeta with count OTAs in this file and total number of photometric reference sources
-        sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
-        self.scratchDb.execute(sql)
-        #to make it stop
+        self.setMinMaxDiffObjID(["DiffDetection"])
+
+
         #test = self.scratchDb.getRowCount("Object")
         #return False
@@ -467 +465 @@
         return True
 
-    def removeDuplicateObjects(self):
-
-        sql = "DROP TABLE SkinnyObject_All"
-        try:
-            self.scratchDb.execute(sql)
-        except:
-            print "SkinnyObject_All table not yet defined"
-
-        sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))"
-        try:
-            self.scratchDb.execute(sql)
-        except:
-            print "failed to create table SkinnyObject_All"
-            raise
-
-        for chipname in self.validChips:
-            
-            sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \
-                   SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname
-            try:
-                self.scratchDb.execute(sql)
-            except:
-                print "failed to insert entry SkinnyObject_All: ", chipname
-                raise
-            
-
-        Nduplicates = 0
-        sql = "SELECT n, objID from (select count(objID) as n, objID from SkinnyObject_All group by objID) as tmp where n > 1"
-        try:
-            rs = self.scratchDb.executeQuery(sql)
-            while (rs.next()):
-                objID = rs.getInt(1)
-                chipID = rs.getString(3)
-                print "delete: ", str(objID), chipID
-                Nduplicates += 1
-
-            rs.close()
-        except:
-            print "failed to get duplicates"
-            raise
-
-        print "duplicates found: ", str(Nduplicates)
-
-    '''
-    Updates imageID {EXP_ID}{OTA} in the provided table
-    '''
-    def updateImageID(self, tableName, x, y):
-
-        sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
-        self.scratchDb.execute(sql)
-
-    '''
-    Overriding this method
-    '''
-    def reportNullsInAllPspsTables(self, showPartials):
-
-        # loops round all imported tables, but we want to check one OTA's worth
-        for table in self.pspsTables:
-            if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials)
-            else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials)
 
     '''

trunk/ippToPsps/jython/gpc1db.py

@@ -293 +293 @@
             meta.append(rs.getString(6))
             meta.append(rs.getString(7))
-
+            meta.append(rs.getString(8))
         except:
             self.logger.errorPair("getDiffStageMeta()", "empty meta data")