Changeset 37403 for branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/objectbatch.py

Timestamp:

Sep 19, 2014, 4:05:27 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• ippToPsps/jython/objectbatch.py (modified) (17 diffs)

branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/objectbatch.py

@@ -15 +15 @@
 from scratchdb import ScratchDb
 from dvoobjects import DvoObjects
+from sqlUtility import sqlUtility
 
 import logging.config
@@ -37 +38 @@
                  scratchDb,
                  dvoID,
-                 batchID,
-                 useFullTables):
+                 batchID):
 
        super(ObjectBatch, self).__init__(
@@ -50 +50 @@
                batchID,
                "OB", 
-               None,
-               1)
+               None)
 
        try:
-           ## XX pass in the existing connection to scratchDb
            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
        except:
@@ -74 +72 @@
         self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
         self.ippToPspsDb.insertObjectMeta(self.batchID, self.region)
-        if True:
-            self.dvoObjects.nativeIngestRegion(self.region)
-        else:
-            self.dvoObjects.ingestRegion(self.region)
+        self.dvoObjects.nativeIngestRegion(self.region)
 
         cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
@@ -90 +85 @@
     def alterPspsTables(self):
 
-        # dec is reserved in MySQL, so STILTS replaces if with dec_, which PSPS doesn't like. so, force it back again using ``
-        self.scratchDb.execute("ALTER TABLE Object CHANGE dec_ `dec` double")
-
         return True
 
@@ -100 +92 @@
     def indexIppTables(self):
 
-        self.logger.infoPair("Creating indexes on", "IPP tables")
+        # since dvopsps is now used, no action is needed here
+        # self.logger.infoPair("Creating indexes on", "IPP tables")
 
         return True
@@ -107 +100 @@
     Inserts stuff for all mags
     '''
-    def insertMags(self, cpsTable):
+    def updateMeanObjectFromCps(self, cpsTable):
+
+        # list of all filters PSPS is interested in
+        # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
+        
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
+    
+        # get a count of the available filters
+        filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable()
+
+        self.logger.infoPair("Available filters in Photcodes", filters)
+
+        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cps table")
+        for filter in filters:
+
+            filterID = self.scratchDb.getFilterID(filter[1])
+
+            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
+            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
+            # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!)
+
+            # set the MeanObject fields based largely on dvopsps cps fields:
+
+            # XXX EAM 20140724 : filterCount is meant to match
+            # Nsecfilt, but is potentially not determined correctly.
+            # use a call to a dvo-native command which knows how to
+            # find Nsecfilt (or save in the db with dvopsps)
+
+            # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting?
+            # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 
+            #  " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+
+            sql = "UPDATE MeanObject JOIN \
+                   " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+                   SET \
+                    MeanObject." + filter[1] + "QfPerfect      = PSF_QF_PERF_MAX \
+                   ,MeanObject." + filter[1] + "MeanPSFMag     = MAG \
+                   ,MeanObject." + filter[1] + "MeanPSFMagErr  = MAG_ERR \
+                   ,MeanObject." + filter[1] + "MeanPSFMagStd  = MAG_STDEV \
+                   ,MeanObject." + filter[1] + "MeanPSFMagMin  = MAG_MIN \
+                   ,MeanObject." + filter[1] + "MeanPSFMagMax  = MAG_MAX \
+                   ,MeanObject." + filter[1] + "MeanPSFMagNpt  = NUSED \
+                   ,MeanObject." + filter[1] + "MeanKronMag    = MAG_KRON \
+                   ,MeanObject." + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \
+                   ,MeanObject." + filter[1] + "MeanKronMagStd = MAG_KRON_STDEV \
+                   ,MeanObject." + filter[1] + "MeanKronMagNpt = NUSED_KRON \
+                   ,MeanObject." + filter[1] + "MeanApMag      = MAG_AP \
+                   ,MeanObject." + filter[1] + "MeanApMagErr   = MAG_AP_ERR \
+                   ,MeanObject." + filter[1] + "MeanApMagStd   = MAG_AP_STDEV \
+                   ,MeanObject." + filter[1] + "MeanApMagNpt   = NUSED_AP \
+                   ,MeanObject." + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) "
+
+
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed update MeanObject", sql)
+                raise
+
+        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
+        # XXX EAM 20140724 : keep this cut?
+        cut = 0.5
+        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
+        for filter in filters:
+
+            sql = "UPDATE MeanObject \
+                   SET " + filter[1] + "MeanPSFMagErr = null \
+                   WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut)
+            self.scratchDb.execute(sql)
+
+    '''
+    Inserts stuff for all mags
+    '''
+    def updateObjectThinFromCps(self, cpsTable):
 
         # list of all filters PSPS is interested in
@@ -126 +193 @@
             filterID = self.scratchDb.getFilterID(filter[1])
 
-            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
-            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
-
-            sql = "UPDATE Object JOIN \
-                   " + cpsTable + " AS cps ON (cps.row = (Object.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
-                   SET \
-                   n" + filter[1] + " = NCODE \
-                   ," + filter[1] + "MeanPSFMag = MAG \
-                   ," + filter[1] + "MeanPSFMagErr = MAG_ERR \
-                   ," + filter[1] + "MeanKronMag = MAG_KRON \
-                   ," + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \
-                   ," + filter[1] + "StackPSFMag = STACK_PSF_MAG \
-                   ," + filter[1] + "StackPSFMagErr = STACK_PSF_MAG_ERR \
-                   ," + filter[1] + "StackKronMag = STACK_KRON_MAG \
-                   ," + filter[1] + "StackKronMagErr = STACK_KRON_MAG_ERR \
-                   ," + filter[1] + "20pct = MAG_20/1000 \
-                   ," + filter[1] + "80pct = MAG_80/1000 \
-                   ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \
-                   ," + filter[1] + "StackDetectID = STACK_DETECT_ID"
-
-# old method for setting the Stack (PSF,Kron) Mags:
-#                  ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \
-#                  ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \
-#                  ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \
-#                  ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \
-
-            #self.logger.info(sql)
-            self.scratchDb.execute(sql)
-
+            # XXX EAM 20140724 : this is quite awkward, add a objRow and ncode value to mysql db table?
+            # sqlLine = sqlUtility()
+
+            sql  = "UPDATE ObjectThin JOIN "
+            sql += cpsTable + " AS cps "
+            sql += "ON (cps.row = (ObjectThin.row * "
+            sql += str(filterCount) + ")-(" 
+            sql += str(filterCount) + " - " 
+            sql += str(filter[0])
+            sql += ")) "
+
+            sql += "SET ObjectThin.n" + filter[1] + " = NCODE, "
+            sql += "ObjectThin.nStackDetections = ObjectThin.nStackDetections + cps.NSTACK_DET"
+            self.logger.info(sql)
+            self.scratchDb.execute(sql)
+
+        # XXX this does not seem like a good thing to leave in SQL
         self.logger.infoPair("Calculating nDetections from", "n[filters]")
         for filter in filters:
             # now do a sum of n[filters], but do not include the ones with -999
-            sql = "UPDATE Object \
-                   SET nDetections = nDetections + n" + filter[1] + " \
-                   WHERE n" + filter[1] + " != -999"
-            self.scratchDb.execute(sql)
-            
-
-
-
-        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
-        cut = 0.5
-        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
-        for filter in filters:
-
-            sql = "UPDATE Object \
-                   SET " + filter[1] + "MeanPSFMagErr = null \
-                   WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut)
-            self.scratchDb.execute(sql)
-
-        self.logger.infoPair("Setting to NULL all x20pct =", "32.767" ) 
-        for filter in filters:
-            sql  = "UPDATE Object \
-                   SET " + filter[1] + "20pct = null \
-                   WHERE " + filter[1] + "20pct > 32.766"
-            self.scratchDb.execute(sql)
-        self.logger.infoPair("Setting to NULL all x80pct =", "32.767" )
-        for filter in filters:
-            sql  = "UPDATE Object \
-                    SET " + filter[1] + "80pct = null \
-                    WHERE " + filter[1] + "80pct > 32.766"
-            self.scratchDb.execute(sql)
-
-
-
-
-    '''
-    Inserts colors
-    '''
-    def updateColors(self):
-
-        self.logger.infoPair("Calculating", "colors")
-        sql = "UPDATE Object \
-               SET \
-               grMeanColor = (gMeanMag - rMeanMag) \
-               ,riMeanColor = (rMeanMag - iMeanMag) \
-               ,izMeanColor = (iMeanMag - zMeanMag) \
-               ,zyMeanColor = (zMeanMag - yMeanMag) \
-               ,grMeanColorErr = SQRT(POW(gMeanMagErr,2) + POW(rMeanMagErr,2)) \
-               ,riMeanColorErr = SQRT(POW(rMeanMagErr,2) + POW(iMeanMagErr,2)) \
-               ,izMeanColorErr = SQRT(POW(iMeanMagErr,2) + POW(zMeanMagErr,2)) \
-               ,zyMeanColorErr = SQRT(POW(zMeanMagErr,2) + POW(yMeanMagErr,2)) \
-                   "
-        try:
-            self.scratchDb.execute(sql)
-        except:
-            self.logger.errorPair("Couldn't calculate colors", sql)
+            sql  = "UPDATE ObjectThin "
+            sql += "SET nDetections = nDetections + n" + filter[1]
+            sql += " WHERE n" + filter[1] + " != -999"
+            self.scratchDb.execute(sql)
+
+    '''
+    give objectName to objectThin
+    XXX EAM 20140714 : This seems quite inefficient in SQL, move to dvopsps?
+    '''
+    def updateObjName(self):
+    ##this is a 4 part update:
+        self.logger.infoPair("updating objName", "using stacks")
+    ## if we have a stackra and dec, use stackra and dec , dec >=0
+
+        sql = "update ObjectThin set objName =  concat('PS1.2 J', \
+        lpad(floor(raStack/15.),2,'0'), \
+        lpad((raStack/15.-(floor(raStack/15.)))*60. ,2,'0'), \
+        lpad(format(((raStack/15.-(floor(raStack/15.)))*60.-floor((raStack/15.-(floor(raStack/15.)))*60.))*60.,2),4,'0'), \
+        '+', \
+        lpad(floor(decStack),2,'0'), \
+        lpad((decStack-(floor(decStack)))*60. ,2,'0'), \
+        lpad(format(((decStack-(floor(decStack)))*60.-floor((decStack-(floor(decStack)))*60.))*60.,2),4,'0') \
+        ) where raStack > -999 and decStack > -999 and decStack >= 0"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("failed sql", sql)
+            return False
+
+
+    ## if we have a stackra and dec, use stackra and dec, dec < 0
+
+        sql = "update ObjectThin set objName =  concat('PS1.2 J', \
+        lpad(floor(raStack/15.),2,'0'), \
+        lpad((raStack/15.-(floor(raStack/15.)))*60. ,2,'0'), \
+        lpad(format(((raStack/15.-(floor(raStack/15.)))*60.-floor((raStack/15.-(floor(raStack/15.)))*60.))*60.,2),4,'0'), \
+        '-', \
+        lpad(floor(decStack),2,'0'), \
+        lpad((decStack-(floor(decStack)))*60. ,2,'0'), \
+        lpad(format(((decStack-(floor(decStack)))*60.-floor((decStack-(floor(decStack)))*60.))*60.,2),4,'0') \
+        ) where raStack > -999 and decStack > -999 and decStack <= 0"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("failed sql", sql)
+            return False
+        self.logger.infoPair("updating objName", "using means")
+    ## use mean ra and dec (dec >= 0)
+        sql = "update ObjectThin set objName =  concat('PS1.2 J', \
+        lpad(floor(raMean/15.),2,'0'), \
+        lpad((raMean/15.-(floor(raMean/15.)))*60. ,2,'0'), \
+        lpad(format(((raMean/15.-(floor(raMean/15.)))*60.-floor((raMean/15.-(floor(raMean/15.)))*60.))*60.,2),4,'0'), \
+        '+', \
+        lpad(floor(decMean),2,'0'), \
+        lpad((decMean-(floor(decMean)))*60. ,2,'0'), \
+        lpad(format(((decMean-(floor(decMean)))*60.-floor((decMean-(floor(decMean)))*60.))*60.,2),4,'0') \
+        ) where raMean > -999 and decMean > -999 and decMean >= 0"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("failed sql", sql)
+            return False
+ 
+    ## use mean ra and dec (dec < 0)
+
+        sql = "update ObjectThin set objName =  concat('PS1.2 J', \
+        lpad(floor(raMean/15.),2,'0'), \
+        lpad((raMean/15.-(floor(raMean/15.)))*60. ,2,'0'), \
+        lpad(format(((raMean/15.-(floor(raMean/15.)))*60.-floor((raMean/15.-(floor(raMean/15.)))*60.))*60.,2),4,'0'), \
+        '-', \
+        lpad(floor(decMean),2,'0'), \
+        lpad((decMean-(floor(decMean)))*60. ,2,'0'), \
+        lpad(format(((decMean-(floor(decMean)))*60.-floor((decMean-(floor(decMean)))*60.))*60.,2),4,'0') \
+        ) where raMean > -999 and decMean > -999 and decMean < 0"
+
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("failed sql", sql)
+            return False
 
     '''
     Populates the Object table
     '''
-    def populateObjectTable(self):
+    def populateObjectThinTable(self):
 
         cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
@@ -223 +303 @@
 
         if False:
+            # XXX EAM 20140724 : this is probably wrong : flux measurements can be 0.0 or negative: please review
             self.logger.infoPair("setting to null  > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR")
             sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 "
+            self.exitProgram("review this code" + sql)
       
             try:
@@ -242 +324 @@
 
         
-        self.logger.infoPair("Populating", "Object s")
-        self.logger.infoPair("Inserting objects from", "cpt file")
-
-        # note `` around dec here, as this is a reserved word in MySQL
-        sql = "INSERT IGNORE INTO Object (\
-               objID \
-               ,ippObjID \
-               ,objInfoFlag \
-               ,qualityFlag \
-               ,surveyID \
-               ,ra \
-               ,`dec` \
-               ,raErr \
-               ,decErr \
-               ,nDetections \
-               ,qfPerfect \
-               ,sgSep \
-               ,dataRelease \
-               ,Random \
-               ,batchID \
-               ) \
-               SELECT \
-               EXT_ID \
-               ,CAT_ID*1000000000 + OBJ_ID \
-               ,FLAGS \
-               ,FLAGS >> 24 & 0xFF \
-               ," + str(self.surveyID) + " \
-               ,RA \
-               ,DEC_ \
-               ,RA_ERR \
-               ,DEC_ERR \
-               ,0 \
-               ,PSF_QF_PERF \
-               ,STARGAL_SEP \
-               , " + str(self.skychunk.dataRelease) + "\
-               , RAND() \
-               , " + str(self.batchID) + "\
-               FROM " + cptTableName
+        self.logger.info("Populating ThinObject")
+        self.logger.info("Inserting objects from cpt file")
+
+        # note "dec" is a reserved word in MySQL
+        # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems...
+        # INSERT IGNORE INTO ObjectThin
+
+        sqlLine = sqlUtility("INSERT INTO ObjectThin (")
+
+        sqlLine.group("objID",           "EXT_ID") 
+        sqlLine.group("ippObjID",        "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly
+        sqlLine.group("surveyID",        "'" + str(self.surveyID) + "'")
+        sqlLine.group("randomID",        "FLOOR(RAND()*9223372036854775807)") # XXX where does this number come from??
+        sqlLine.group("batchID",         "'" + str(self.batchID) + "'")
+        sqlLine.group("dvoRegionID",     "CAT_ID") 
+        sqlLine.group("tessID",          "TESS_ID") 
+        sqlLine.group("projectionID",    "PROJECTION_ID") 
+        sqlLine.group("skycellID",       "SKYCELL_ID") 
+        sqlLine.group("dataRelease",     "'" + str(self.skychunk.dataRelease) + "'")
+        sqlLine.group("objInfoFlag",     "FLAGS") 
+        sqlLine.group("qualityFlag",     "FLAGS >> 24 & 0xFF")
+        sqlLine.group("consistencyFlag", "'0'") 
+        sqlLine.group("raStack",         "RA_STK")
+        sqlLine.group("decStack",        "DEC_STK") 
+        sqlLine.group("raStackErr",      "RA_STK_ERR") 
+        sqlLine.group("decStackErr",     "DEC_STK_ERR")
+        sqlLine.group("raMean",          "RA_MEAN")
+        sqlLine.group("decMean",         "DEC_MEAN")
+        sqlLine.group("raMeanErr",       "RA_ERR")
+        sqlLine.group("decMeanErr",      "DEC_ERR")
+        sqlLine.group("posMeanChisq",    "CHISQ_POS")
+        sqlLine.group("nStackObjectRows", "'0'") # XXX I need to add / define this in dvopsps
+        sqlLine.group("nStackDetections", "'0'")
+        sqlLine.group("nDetections",      "'0'")
+        sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName)
 
         try:
@@ -289 +368 @@
         # add row count columns so we can perform joins to get colors
         self.logger.infoPair("Adding 'row' columns to", "Object and cps tables")
-        self.scratchDb.addRowCountColumn("Object", "row")
+        self.scratchDb.addRowCountColumn("ObjectThin", "row")
         self.scratchDb.addRowCountColumn(cpsTableName, "row")
 
-        self.insertMags(cpsTableName)
-
-#don't do this as we removed those columns.. 
-#        self.updateColors()
-
+##        self.insertMeanMags(cpsTableName)
+## this is the new version
+        self.logger.infoPair("update objName for ","objectThin")
+        self.updateObjName()
+
+        self.logger.infoPair("update ObjectThin from ","cps table")
+
+        self.updateObjectThinFromCps(cpsTableName)
+
+        # XXX EAM 20140724 : is this necessary??
         #objects can have out of range ra dec in dvo - need to find and kill them at the end
 
@@ -312 +396 @@
         #count out of range
 
-        sql = "SELECT count(*) FROM Object where Object.dec > " + str(decMax) + " or Object.dec < " + str(decMin) + " or Object.ra > " + str(raMax) + " or Object.ra < " + str(raMin)       
+        sql = "SELECT count(*) FROM ObjectThin where \
+              ObjectThin.decMean > " + str(decMax) + " \
+              or ObjectThin.decMean < " + str(decMin) + " \
+              or ObjectThin.raMean > " + str(raMax) + " \
+              or ObjectThin.raMean < " + str(raMin)       
     
         rs = self.scratchDb.executeQuery(sql)
@@ -321 +409 @@
         
  
-        sql = "DELETE FROM Object where Object.dec > " + str(decMax) + " or Object.dec < " + str(decMin) + " or Object.ra > " + str(raMax) + " or Object.ra < " + str(raMin) 
+        sql = "DELETE FROM ObjectThin where \
+              ObjectThin.decMean > " + str(decMax) + " or \
+              ObjectThin.decMean < " + str(decMin) + " or \
+              ObjectThin.raMean > " + str(raMax) + " or \
+              ObjectThin.raMean < " + str(raMin) 
         self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range")
 
@@ -327 +419 @@
             self.scratchDb.execute(sql)
         except:
-            self.logger.errorPair("Couldn't cull outsiders from Object table", sql)
-            return False
-
-        self.logger.infoPair("Dropping row column from", "Object table")
-        self.scratchDb.dropColumn("Object", "row")
-        self.logger.infoPair("Purging from scratch Db", self.region + " region")
-
-        self.dvoObjects.purgeRegion(self.region)
-
-        self.setMinMaxObjID(["Object"])
+            self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql)
+            return False
+
+        self.logger.infoPair("Dropping row column from", "ObjectThin table")
+        self.scratchDb.dropColumn("ObjectThin", "row")
+        ##self.logger.infoPair("Purging from scratch Db", self.region + " region")
+
+        ##Don't do this till after MeanObject
+        ##self.dvoObjects.purgeRegion(self.region)
+
+        self.setMinMaxObjID(["ObjectThin"])
 
         return True
@@ -343 +436 @@
     Populates the ObjectCalColor table
     '''
-    def populateObjectCalColorTable(self):
-
-        self.logger.infoPair("Populating", "ObjectCalColor")
-
-        sql = "INSERT INTO ObjectCalColor ( \
-               objID \
-               ,ippObjID \
-               ,dataRelease \
-               ) \
-               SELECT \
-               objID \
-               ,ippObjID \
-               ,dataRelease \
-               FROM Object"
-        try:
-            self.scratchDb.execute(sql)
-        except:
-            self.logger.errorPair("Couldn't populate ObjectCalColor table", sql)
-            return False
+    def populateMeanObjectTable(self):
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+        cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
+        self.logger.infoPair("Populating MeanObject from ", "ObjectThin")
+
+        sql = "INSERT INTO MeanObject ( objID ) SELECT objID FROM ObjectThin"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("Couldn't populate MeanObject table", sql)
+            return False
+
+        self.logger.infoPair("Adding 'row' columns to", "MeanObject table")
+        self.scratchDb.addRowCountColumn("MeanObject", "row")
+ 
+
+        ##self.scratchDb.addRowCountColumn(cpsTableName, "row")
+        self.logger.infoPair("update MeanObjects from ","cps table")
+        self.updateMeanObjectFromCps(cpsTableName)
+
+        self.logger.infoPair("Dropping row column from", "MeanObject table")
+        self.scratchDb.dropColumn("MeanObject", "row")
 
         return True
@@ -370 +466 @@
     def populatePspsTables(self):
 
-        if not self.populateObjectTable(): return False
+        if not self.populateObjectThinTable(): return False
         #if not self.populateObjectCalColorTable(): return False
+        if not self.populateMeanObjectTable(): return False
 
         # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number
-        self.logger.infoPair("Forcing uniqueness on", "objID in Object table")
-        rowCountBefore = self.scratchDb.getRowCount("Object")
+        self.logger.infoPair("Forcing uniqueness on", "objID in ObjectThin table")
+        rowCountBefore = self.scratchDb.getRowCount("ObjectThin")
 
         # XXX EAM : note that in mysql versions later than 5.1, this fails
@@ -386 +483 @@
             self.scratchDb.execute("set session old_alter_table=1")
             
-        self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)")
+        self.scratchDb.execute("ALTER IGNORE TABLE ObjectThin ADD UNIQUE INDEX(objID)")
         if self.scratchDb.version > 5.1:
             self.scratchDb.execute("set session old_alter_table=0")
 
-        rowCountAfter = self.scratchDb.getRowCount("Object")
+        rowCountAfter = self.scratchDb.getRowCount("ObjectThin")
         self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
 
+        self.logger.infoPair("Forcing uniqueness on", "objID in MeanObject table")
+        rowCountBefore = self.scratchDb.getRowCount("MeanObject")
+
+        # XXX EAM : note that in mysql versions later than 5.1, this fails
+        # unless the following is called first: 
+        # set session old_alter_table=1
+        # follow the command with 
+        # set session old_alter_table=0
+        # OF COURSE, this fails for mysql version < 5.5...
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=1")
+            
+        self.scratchDb.execute("ALTER IGNORE TABLE MeanObject ADD UNIQUE INDEX(objID)")
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=0")
+
+        rowCountAfter = self.scratchDb.getRowCount("MeanObject")
+        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
+
+        self.dvoObjects.purgeRegion(self.region)
+
+        #this is abuse of something but this is how I get the object batches to crash to further investigate them
+        
+#        rowCountAfter = self.scratchDb.getRowCount("Object")
         return True
-
+#        return False