Changeset 33948 for branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py

Timestamp:

May 29, 2012, 5:07:20 PM (14 years ago)

Author:

eugene

Message:

merging changes from trunk

File:

• branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py (modified) (11 diffs)

branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py

@@ -51 +51 @@
                1)
 
-       if not self.everythingOK: return
-
-       self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName)
+       try:
+           self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName)
+       except:
+           self.logger.errorPair("Unable to create instance of", "DvoObjects")
+           raise
 
        # create an output filename, which is {dvoINDEX}.FITS
@@ -78 +80 @@
     def alterPspsTables(self):
 
-        self.logger.debug("Altering PSPS tables")
+        # dec is reserved in MySQL, so STILTS replaces if with dec_, which PSPS doesn't like. so, force it back again using ``
+        self.scratchDb.execute("ALTER TABLE Object CHANGE dec_ `dec` double")
+
         return True
 
@@ -95 +99 @@
     def insertMags(self, cpsTable):
 
-        # list of all filters we are interested in
-        allFilters = ['g', 'r', 'i', 'z', 'y']
+        # list of all filters PSPS is interested in
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
         
-        # this loop looks into the DVO Photcodes tables and gets the 'CODE' for each the filters above that are listed
-        filters = []
-        for filter in allFilters:
-            code = self.scratchDb.getCodeForThisFilter(filter)
-            if code < 0: continue
-            filters.append([code, filter])
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
     
         # get a count of the available filters
         filterCount = len(filters)
 
-        self.logger.infoPair("Available filters", filters)
+        self.logger.infoPair("Available filters in Photcodes", filters)
 
         # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cps table")
         for filter in filters:
 
             sql = "UPDATE Object \
-                   JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-" + str(filter[0]) + ") \
+                   JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
                    SET \
                    n" + filter[1] + " = NCODE \
@@ -120 +120 @@
                    ," + filter[1] + "MeanFlux = 3631 * POW(10.0, (-0.4*MAG)) \
                    ," + filter[1] + "MeanMagErr = MAG_ERR \
-                   ," + filter[1] + "Min = MAG_20 \
-                   ," + filter[1] + "Max = MAG_80 \
+                   ," + filter[1] + "Min = MAG_20/1000 \
+                   ," + filter[1] + "Max = MAG_80/1000 \
                    "
-
+            
+            self.scratchDb.execute(sql)
+
+        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
+        cut = 0.5
+        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
+        for filter in filters:
+
+            sql = "UPDATE Object \
+                   SET " + filter[1] + "MeanMagErr = null \
+                   WHERE " + filter[1] + "MeanMagErr > " + str(cut)
             self.scratchDb.execute(sql)
 
@@ -131 +141 @@
     def updateColors(self):
 
+        self.logger.infoPair("Calculating", "colors")
         sql = "UPDATE Object \
                SET \
@@ -147 +158 @@
             self.logger.errorPair("Couldn't calculate colors", sql)
 
-
-    '''
-    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
-    '''
-    def populatePspsTables(self):
+    '''
+    Populates the Object table
+    '''
+    def populateObjectTable(self):
 
         cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
         cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
 
-        self.logger.infoPair("Inserting objects from", "cpt")
-        sql = "INSERT INTO Object (\
+        self.logger.infoPair("Populating", "ObjectCalColor")
+        self.logger.infoPair("Inserting objects from", "cpt file")
+
+        # note `` around dec here, as this is a reserved word in MySQL
+        sql = "INSERT IGNORE INTO Object (\
                objID \
                ,ippObjID \
                ,objInfoFlag \
+               ,varFlag \
                ,surveyID \
                ,ra \
-               ,dec_ \
+               ,`dec` \
                ,raErr \
                ,decErr \
                ,nDetections \
+               ,dataRelease \
                ) \
                SELECT \
@@ -172 +187 @@
                ,CAT_ID*1000000000 + OBJ_ID \
                ,FLAGS \
+               ,0 \
                ," + str(self.surveyID) + " \
                ,RA \
@@ -178 +194 @@
                ,DEC_ERR \
                ,NMEASURE \
+               , " + str(self.config.dataRelease) + "\
                FROM " + cptTableName
 
@@ -184 +201 @@
         except:
             self.logger.errorPair("Couldn't populate Object table", sql)
+            return False
 
         # add row count columns so we can perform joins to get colors
@@ -190 +208 @@
         self.scratchDb.addRowCountColumn(cpsTableName, "row")
 
-        self.logger.infoPair("Adding magnitudes from", "cps table")
         self.insertMags(cpsTableName)
 
-        self.logger.infoPair("Calculating", "colors")
         self.updateColors()
 
@@ -205 +221 @@
         return True
 
+    '''
+    Populates the ObjectCalColor table
+    '''
+    def populateObjectCalColorTable(self):
+
+        self.logger.infoPair("Populating", "ObjectCalColor")
+
+        sql = "INSERT INTO ObjectCalColor ( \
+               objID \
+               ,ippObjID \
+               ,dataRelease \
+               ) \
+               SELECT \
+               objID \
+               ,ippObjID \
+               ,dataRelease \
+               FROM Object"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("Couldn't populate ObjectCalColor table", sql)
+            return False
+
+        return True
+
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+
+        if not self.populateObjectTable(): return False
+        #if not self.populateObjectCalColorTable(): return False
+
+        # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number
+        self.logger.infoPair("Forcing uniqueness on", "objID in Object table")
+        rowCountBefore = self.scratchDb.getRowCount("Object")
+        self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)")
+        rowCountAfter = self.scratchDb.getRowCount("Object")
+        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
+
+        return True
+