Changeset 38052 for branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py

Timestamp:

Mar 28, 2015, 8:28:32 PM (11 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py (modified) (14 diffs)

branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py

@@ -14 +14 @@
 from ipptopspsdb import IppToPspsDb
 from scratchdb import ScratchDb
-from dvoobjects import DvoObjects
+#from dvoobjects import DvoObjects
+from dvodiffobjects import DvoDiffObjects
 from sqlUtility import sqlUtility
 
@@ -53 +54 @@
 
        try:
-           self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
+           self.dvoDiffObjects = DvoDiffObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
        except:
-           self.logger.errorPair("Unable to create instance of", "DvoObjects")
+           self.logger.errorPair("Unable to create instance of", "DvoDiffObjects")
            raise
 
@@ -72 +73 @@
         self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
         self.ippToPspsDb.insertObjectMeta(self.batchID, self.region)
-        self.dvoObjects.nativeIngestRegion(self.region)
+        self.dvoDiffObjects.nativeIngestRegion(self.region)
 
         cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
@@ -177 +178 @@
             sql += ")) "
 
-            sql += "SET DiffDetObject.n" + filter[1] + " = NCODE, "
-            sql += "DiffDetObject.nDetections = DiffDetObject.nDetections + cps.NSTACK_DET"
+            sql += "SET DiffDetObject.n" + filter[1] + " = NWARP, "
+            sql += "DiffDetObject.nDetections = DiffDetObject.nDetections + cps.NWARP"
             self.logger.info(sql)
-            self.scratchDb.execute(sql)
+            try:
+                self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed sql", sql)
+                raise
+                return False
+
 
         # XXX this does not seem like a good thing to leave in SQL
-        self.logger.infoPair("Calculating nDetections from", "n[filters]")
-        for filter in filters:
+        #self.logger.infoPair("Calculating nDetections from", "n[filters]")
+        #for filter in filters:
             # now do a sum of n[filters], but do not include the ones with -999
-            sql  = "UPDATE DiffDetObject "
-            sql += "SET nDetections = nDetections + n" + filter[1]
-            sql += " WHERE n" + filter[1] + " != -999"
-            self.scratchDb.execute(sql)
+         #   sql  = "UPDATE DiffDetObject "
+         #   sql += "SET nDetections = nDetections + n" + filter[1]
+         #   sql += " WHERE n" + filter[1] + " != -999"
+         #   try:
+         #       self.scratchDb.execute(sql)
+         #   except:
+         #       self.logger.errorPair("failed sql", sql)
+         #       raise
+                
+
 
     '''
@@ -213 +226 @@
         except:
             self.logger.errorPair("failed sql", sql)
-            return False
+            raise
  
     ## use mean ra and dec (dec < 0)
@@ -231 +244 @@
         except:
             self.logger.errorPair("failed sql", sql)
-            return False
+            raise
 
     '''
@@ -251 +264 @@
             except:
                 self.logger.errorPair("Couldn't squash out of range stuff", sql)
-                return False
+                raise
 
             self.logger.infoPair("setting to null  > 1e-38 and < 1e-38 in", "cps FLUX_KRON")
@@ -260 +273 @@
             except:
                 self.logger.errorPair("Couldn't squash out of range stuff", sql)
-                return False
+                raise
 
         
@@ -278 +291 @@
         sqlLine.group("batchID",         "'" + str(self.batchID) + "'")
         sqlLine.group("dvoRegionID",     "CAT_ID") 
-        sqlLine.group("skycellID",       "SKYCELL_ID") 
+#        sqlLine.group("skycellID",       "SKYCELL_ID") 
         sqlLine.group("dataRelease",     "'" + str(self.skychunk.dataRelease) + "'")
         sqlLine.group("objInfoFlag",     "FLAGS") 
@@ -294 +307 @@
         except:
             self.logger.errorPair("Couldn't populate DiffDetObject table", sql)
-            return False
+            raise
  
         # add row count columns so we can perform joins to get colors
@@ -307 +320 @@
 
         self.logger.infoPair("update DiffDetObject from ","cps table")
-
+        #this fills ng/etc
         self.updateDiffObjectFromCps(cpsTableName)
-
+        #this fills qfperfect
+        self.updateMeanObjectFromCps(cpsTableName)
         # XXX EAM 20140724 : is this necessary??
         #objects can have out of range ra dec in dvo - need to find and kill them at the end
@@ -350 +364 @@
         except:
             self.logger.errorPair("Couldn't cull outsiders from DiffDetObject table", sql)
-            return False
+            raise
 
         self.logger.infoPair("Dropping row column from", "DiffDetObject table")
@@ -368 +382 @@
 
     def updatePspsUniqueIDs(self,table):
-        sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
-        (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
-        uniquePspsDOid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+#        sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
+#        (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
+#        uniquePspsDOid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+        sql = "UPDATE DiffDetObject set uniquePspsDOid = (("+str(self.batchID)+"*1000000000 ) + row)"
         try: self.scratchDb.execute(sql)
         except:
             self.logger.errorPair('failed sql',sql)
-            return
+            raise
 
 
@@ -407 +422 @@
         self.scratchDb.execute("ALTER TABLE DiffDetObject CHANGE dec_ `dec` double")
 
+        self.logger.infoPair("Adding 'row' columns to", "Diff Det Object")
+        self.scratchDb.addRowCountColumn("DiffDetObject", "row")
+        self.logger.infoPair("Adding unique", "Diff Det Object ")
         self.updatePspsUniqueIDs("DiffDetObject")
-
-        self.dvoObjects.purgeRegion(self.region)
+        self.logger.infoPair("Dropping row column from", "DiffDetObject table")
+        self.scratchDb.dropColumn("DiffDetObject", "row")
+
+        self.dvoDiffObjects.purgeRegion(self.region)
 
         #this is abuse of something but this is how I get the object batches to crash to further investigate them