Index: trunk/ippToPsps/jython/objectbatch.py =================================================================== --- trunk/ippToPsps/jython/objectbatch.py (revision 39255) +++ trunk/ippToPsps/jython/objectbatch.py (revision 39565) @@ -147,7 +147,12 @@ # find Nsecfilt (or save in the db with dvopsps) - # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? - # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter - # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + # the math below depends on filterCount = Nsecfilt and + # both MeanObject.row and cps.row being 1 counting, and + # filter[0] also being 1 counting + + # cps.row - 1 = (cpt.row - 1)*Nsecfilt + (filter[0] - 1) + # cps.row = cpt.row * Nsecfilt - (Nsecfilt - filter[0]) + # cps.row = cpt.row * Nsecfilt - Nsecfilt + filter[0] + # [note that the sql below has parenthesis around (Nsecfilt - filter[0]) sql = "UPDATE MeanObject JOIN \ @@ -169,6 +174,5 @@ ,MeanObject." + filter[1] + "MeanApMagStd = MAG_AP_STDEV \ ,MeanObject." + filter[1] + "MeanApMagNpt = NUSED_AP \ - ,MeanObject." + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) " - + ,MeanObject." + filter[1] + "Flags = FLAGS " try: self.scratchDb.execute(sql) @@ -301,5 +305,5 @@ sqlLine.group("processingVersion", "'" + str(self.skychunk.processingVersion) + "'") sqlLine.group("objInfoFlag", "FLAGS") - sqlLine.group("qualityFlag", "FLAGS >> 24 & 0xFF") + sqlLine.group("qualityFlag", "FLAGS >> 23 & 0xFF") sqlLine.group("raStack", "RA_STK") sqlLine.group("decStack", "DEC_STK") @@ -339,57 +343,64 @@ self.updateObjectThinFromCps(cpsTableName) - # XXX EAM 20140724 : is this necessary?? - #objects can have out of range ra dec in dvo - need to find and kill them at the end - - self.logger.infoPair("Determining", "ra/dec range") - - raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) - raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) - decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) - decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) - - self.logger.infoPair("R_MIN", raMin) - self.logger.infoPair("R_MAX", raMax) - self.logger.infoPair("D_MIN", decMin) - self.logger.infoPair("D_MAX", decMax) - #count out of range - - sql = "SELECT count(*) FROM ObjectThin where \ - ObjectThin.decMean > " + str(decMax) + " \ - or ObjectThin.decMean < " + str(decMin) + " \ - or ObjectThin.raMean > " + str(raMax) + " \ - or ObjectThin.raMean < " + str(raMin) - - rs = self.scratchDb.executeQuery(sql) - - rs.first() - nToDelete = rs.getInt(1) - - #delete out of range - - - sql = "DELETE FROM ObjectThin where \ - ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ - ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ - ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ - ObjectThin.raMean < (" + str(raMin) + " - .0033)" - self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") - - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) - raise - - ##Don't do this till after MeanObject - ##self.dvoObjects.purgeRegion(self.region) + if False: + # this chunk of code deletes objects which are out of ra,dec range for the table. + # this was a problem in an early version of DVO for cases where the astrometry went insane. + # this causes problems for the ra = 0,360 boundary (the test below does not handle that situation) + # and the restrictions below are poorly defined for the regions near the pole. + + # in any case, ObjectThin needs to maintain the same order + # as the cpt table until MeanObjects have been created or + # the join to the cps table will fail + + # XXX EAM 20140724 : is this necessary?? + # objects can have out of range ra dec in dvo - need to find and kill them at the end + + self.logger.infoPair("Determining", "ra/dec range") + + raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) + raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) + decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) + decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) + + self.logger.infoPair("R_MIN", raMin) + self.logger.infoPair("R_MAX", raMax) + self.logger.infoPair("D_MIN", decMin) + self.logger.infoPair("D_MAX", decMax) + #count out of range + + sql = "SELECT count(*) FROM ObjectThin where \ + ObjectThin.decMean > " + str(decMax) + " \ + or ObjectThin.decMean < " + str(decMin) + " \ + or ObjectThin.raMean > " + str(raMax) + " \ + or ObjectThin.raMean < " + str(raMin) + + rs = self.scratchDb.executeQuery(sql) + + rs.first() + nToDelete = rs.getInt(1) + + #delete out of range + + + sql = "DELETE FROM ObjectThin where \ + ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ + ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ + ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ + ObjectThin.raMean < (" + str(raMin) + " - .0033)" + self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) + raise + self.logger.infoPair("updatePspsUniqueIDs","start") self.updatePspsUniqueIDs() self.logger.infoPair("updatePspsUniqueIDs","end") + self.logger.infoPair("Dropping row column from", "ObjectThin table") self.scratchDb.dropColumn("ObjectThin", "row") - ##self.logger.infoPair("Purging from scratch Db", self.region + " region") self.logger.infoPair("Dropped row column", "objectThin") - ##Don't do this till after MeanObject self.setMinMaxObjID(["ObjectThin"]) @@ -415,9 +426,4 @@ self.scratchDb.addRowCountColumn("MeanObject", "row") - - - - - ##self.scratchDb.addRowCountColumn(cpsTableName, "row") self.logger.infoPair("update MeanObjects from ","cps table") self.updateMeanObjectFromCps(cpsTableName) @@ -434,8 +440,14 @@ if not self.populateObjectThinTable(): return False - #if not self.populateObjectCalColorTable(): return False if not self.populateMeanObjectTable(): return False # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + + ### XXX the code below first removes duplicate objID entries + ### from ObjectThin, then does the same for MeanObject. This + ### is a big problem: we have no guarantee that the surviving + ### rows are the correct matched rows. + + ### force objID uniqueness on *** ObjectThin *** self.logger.infoPair("Forcing uniqueness on", "objID in ObjectThin table") rowCountBefore = self.scratchDb.getRowCount("ObjectThin") @@ -457,4 +469,5 @@ self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + ### force objID uniqueness on *** MeanObject *** self.logger.infoPair("Forcing uniqueness on", "objID in MeanObject table") rowCountBefore = self.scratchDb.getRowCount("MeanObject") @@ -476,9 +489,6 @@ self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + # delete the cpt, cps tables from the scratch mysql (or we will run out of space) self.dvoObjects.purgeRegion(self.region) - #this is abuse of something but this is how I get the object batches to crash to further investigate them - -# rowCountAfter = self.scratchDb.getRowCount("Object") - return True -# return False + return True