Changeset 37801 for branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py

Timestamp:

Jan 11, 2015, 1:31:50 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py (modified) (31 diffs)

branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py

@@ -73 +73 @@
        # we just need one valid filter to get the generic metadata
        validFilter = 'none'
-
+       validFilterCount = 0
        # get a list of stackIDs, identify the CMFs, read the headers:
        for filter in self.filters:
@@ -81 +81 @@
            stackID = gpc1Db.getStackIDFromSkyIDAndFilter(skychunk.dvoLabel, skyID, filterName)
            self.stackIDs[filter] = stackID
-
+           if (stackID > 0):
+               validFilterCount=validFilterCount + 1
            if (stackID <= 0): 
                continue
@@ -115 +116 @@
            self.logger.errorPair("can't find any valid stacks...?? ","??")
            raise
+       # did we find any valid stacks?
+       if validFilterCount == 0:
+           self.logger.errorPair("can't find any valid stacks...?? ","??")
+           raise
+       else:
+           self.logger.infoPair("number of filters found:",validFilterCount)
     
        validStackID = self.stackIDs[validFilter]
@@ -154 +161 @@
 
        self.dropTableVerbose("StackObjectThin")
-       self.dropTableVerbose("StackObjectRaw")
+       self.dropTableVerbose("StackObjectAttributes")
        
        self.dropTableVerbose("StackModelFitExtra")
@@ -162 +169 @@
 
        self.dropTableVerbose("StackApFlx")
-       self.dropTableVerbose("StackApFlxFull")
-       self.dropTableVerbose("StackApFlxFullC1")
-       self.dropTableVerbose("StackApFlxFullC2")
+       self.dropTableVerbose("StackApFlxEGUnc")
+       self.dropTableVerbose("StackApFlxEGCon6")
+       self.dropTableVerbose("StackApFlxEGCon8")
 
        self.dropTableVerbose("StackPetrosian")
@@ -284 +291 @@
         self.selectDvoObjIDs()
 
+        self.generateRandomIDs()
+
+        self.generateStackDetectRowIDsAndPspsUniqueIDs()
+
         # add indexes StackObjectThin
         self.scratchDb.createIndex("StackObjectThin", "objID")
@@ -325 +336 @@
             sqlLine.group("a."+filter+"ApMag",         "b.Map")
             sqlLine.group("a."+filter+"ApMagErr",      "b.dMap")
-            sqlLine.group("a."+filter+"telluricExt",   "b.telluricExt")
+          #  sqlLine.group("a."+filter+"telluricExt",   "b.telluricExt")
 
             sqlLine.group("a."+filter+"infoFlag2",     "(b.flags << 13)")
@@ -365 +376 @@
 
     '''
-    Populates the StackObjectRaw table
-    '''
-    def populateStackObjectRaw(self):
-
-        tablename = "StackObjectRaw"
+    Populates the StackObjectAttributes table
+    '''
+    def populateStackObjectAttributes(self):
+
+        tablename = "StackObjectAttributes"
 
         self.logger.infoPair("Procesing table", tablename)
         self.insertDvoIDs(tablename, "StackObjectThin")
-
+        
         # if we are going to add a key, do it here so it is useful
         self.scratchDb.createIndex(tablename, "objID") 
@@ -420 +431 @@
             sqlLine.group("a."+filter+"KronFluxErr",   "b.KRON_FLUX_ERR / " + str(exptime))
             sqlLine.group("a."+filter+"ApFillFac",     "b.AP_NPIX / (3.14159265359 * POW(b.AP_MAG_RADIUS - 0.5, 2))")
-            sqlLine.group("a."+filter+"ApRadius",      "b.AP_MAG_RADIUS")
+#            sqlLine.group("a."+filter+"ApRadius",      "b.AP_MAG_RADIUS")
             sqlLine.group("a."+filter+"KronRad",       "b.MOMENTS_R1 * 2.5")
             sqlLine.group("a."+filter+"PlateScale",    "b.PLTSCALE")
@@ -430 +441 @@
             except:
                 self.logger.errorPair('failed sql',sql)
-                return
+                raise
 
             # insert detection information for each filter based on the DVO contents
@@ -446 +457 @@
             except:
                 self.logger.errorPair('failed sql',sql)
-                return
+                raise
 
 
@@ -549 +560 @@
         # insert detection information for each filter based on the
         # DVO contents.  Since I already have extracted
-        # StackObjectRaw, this is probably faster using that table:
+        # StackObjectAttributes, this is probably faster using that table:
         sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET")
         sqlLine.group("a." + filter + model + "Mag",       "a." + filter + model + "Mag + b.zp")
-        sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID")
-
+        sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID" + " AND b.imageID = " + str(self.imageIDs[filter]))
         try:
             self.scratchDb.execute(sql)
         except:
             self.logger.errorPair('failed sql',sql)
-            return
-
+            raise
         # we have (modulo signs):
         # gRaExp  = gra  + (X_EXT - gxPos)*gPlateScale / cos(gdec)
@@ -583 +592 @@
 
         # insert detection information for each filter based on the DVO contents
-        sqlLine = sqlUtility("UPDATE " + tablename + " AS a, StackObjectThin as b, StackObjectRaw as c SET ")
+        sqlLine = sqlUtility("UPDATE " + tablename + " AS a, StackObjectThin as b, StackObjectAttributes as c SET ")
         sqlLine.group(extR, rawR + " - (" + extR + " - " + rawX + ")*" + rawS + " / 3600.0 / cos(radians(" + rawD + "))")
         sqlLine.group(extD, rawD + " + (" + extD + " - " + rawY + ")*" + rawS + " / 3600.0")
@@ -596 +605 @@
         except:
             self.logger.errorPair('failed sql',sql)
-            return
+            raise
 
     '''
@@ -655 +664 @@
         sqlLine.group("a." + filter + "petCf",        "b.PETRO_FILL")
 
-        sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET")
+        sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET ")
 
         try: self.scratchDb.execute(sql)
@@ -670 +679 @@
         except:
             self.logger.errorPair('failed sql',sql)
-            return
+            raise
 
     '''
@@ -690 +699 @@
 
         for filter in self.filters:
-            self.populateStackApFlxFilter(tablename, "", filter)
-            self.populateStackApFlxFilter(tablename, "C1", filter)
-            self.populateStackApFlxFilter(tablename, "C2", filter)
+            self.populateStackApFlxFilter(tablename, "Unc", filter)
+            self.populateStackApFlxFilter(tablename, "Con6", filter)
+            self.populateStackApFlxFilter(tablename, "Con8", filter)
 
         self.deleteRowsWithNoStackData(tablename)
@@ -698 +707 @@
 
     '''
-    Populates the StackApFlxFull table
-    '''
-    def populateStackApFlxFull(self, version):
-
-        tablename = "StackApFlxFull" + version
+    Populates the StackApFlxEG table
+    '''
+    def populateStackApFlxEG(self, version):
+
+        tablename = "StackApFlxEG" + version
 
         if self.config.camera == "simtest":
@@ -721 +730 @@
 
     '''
-    Updates aperture fluxes for StackApFkx table
+    Updates aperture fluxes for StackApFlx table
     '''
     def populateStackApFlxFilter(self, tablename, version, filter):
@@ -728 +737 @@
 
         # order of the 3 convolutions is fixed (raw, C1, C2) = (1, 2, 3)
-        if version == "":   
+        if version == "Unc":   
             psfCondition = "(b.table_index % 3 = 1)"
             prefix = ""
-        if version == "C1": 
+        if version == "Con6": 
             psfCondition = "(b.table_index % 3 = 2)"
-            prefix = "c1"
-        if version == "C2": 
+            prefix = "c6"
+        if version == "Con8": 
             psfCondition = "(b.table_index % 3 = 0)" 
-            prefix = "c2"
+            prefix = "c8"
 
         if prefix == "none":
@@ -769 +778 @@
         except:
             self.logger.errorPair('failed sql',sql)
-            return
+            raise
 
         # set the flux values from the cmf file
@@ -833 +842 @@
             sqlLine.group(field + "Err",       field + "err" + " * POW(10, 3.56 - 0.4* b.zp)")
             sqlLine.group(field + "Std",       field + "Std" + " * POW(10, 3.56 - 0.4* b.zp)")
-        sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID")
+        sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(self.imageIDs[filter]) )
         self.scratchDb.execute(sql)
 
@@ -931 +940 @@
 
         return True
+
+    def generateRandomIDs(self):
+        sql = "UPDATE StackObjectThin set randomStackObjID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)"
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql',sql)
+            raise
+
+
+    def generateStackDetectRowIDsAndPspsUniqueIDs(self):
+        sql = "UPDATE StackObjectThin join (select @r:=@r+1 rownum, objID from \
+        (select @r:=0) r, StackObjectThin t) as foo using (objID) set \
+        stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \
+        uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)";
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql',sql)
+            raise
+        
 
     '''
@@ -951 +979 @@
         sqlLine.group("skyCellID",        "'" + str(self.skycellID) + "'")
         sqlLine.group("dataRelease",      "'" + str(self.skychunk.dataRelease) + "'")
-        sqlLine.group("randomStackObjID", "FLOOR(RAND()*9223372036854775807)")
+   #     sqlLine.group("randomStackObjID", "FLOOR(RAND()*9223372036854775807)")
 
         whereClause = " WHERE ("
@@ -971 +999 @@
         except:
             self.logger.errorPair('failed sql',sql)
-            return
+            raise
 
     '''
@@ -982 +1010 @@
             self.scratchDb.addColumn(table, filter + "haveData", "tinyint")
 
-        fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, "
+        fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, "
 
         for i in range(len(self.filters)):
@@ -988 +1016 @@
             fields += filter + "ippDetectID, "
             fields += filter + "stackDetectID, "
-            fields += filter + "stackMetaID" # careful here: do not add comma to last entry
+            fields += filter + "stackMetaID " # careful here: do not add comma to last entry
             if (i < len(self.filters) - 1):
                 fields += ", "
-        
+
+        if (table == "StackObjectAttributes"):
+            fields += ", stackDetectRowID "
         sql  = "INSERT INTO " + table + " (" + fields + ") "
         sql += "SELECT " + fields + " FROM " + mainTable
@@ -998 +1028 @@
         except:
             self.logger.errorPair('failed sql',sql)
-            return
-    
+            raise
+
+
+        
     '''
     Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
@@ -1036 +1068 @@
         self.populateStackObjectThin()
 
-        self.populateStackObjectRaw()
+        self.populateStackObjectAttributes()
 
         if self.stackType != "NIGHTLY_STACK":
@@ -1053 +1085 @@
             self.populateStackApFlx()
 
-            self.populateStackApFlxFull("")
-            self.populateStackApFlxFull("C1")
-            self.populateStackApFlxFull("C2")
+            self.populateStackApFlxEG("Unc")
+            self.populateStackApFlxEG("Con6")
+            self.populateStackApFlxEG("Con8")
 
         # delete rows with no content. for the other tables, this is done in 
@@ -1130 +1162 @@
         self.logger.infoPair("Done. Imported", "%d tables" % count)
         self.indexIppTables()
-                      
+#        return False
         return True