Index: trunk/ippToPsps/config/tables.IN.vot =================================================================== --- trunk/ippToPsps/config/tables.IN.vot (revision 39564) +++ trunk/ippToPsps/config/tables.IN.vot (revision 39565) @@ -659,5 +659,5 @@ DIFF_WITH_SINGLE 0x000000011 Difference source matched to a single positive detection. DIFF_WITH_DOUBLE 0x000000022 Difference source matched to positive detections in both images. - MATCHED 0x000000044 Difference source matched to positive detections in both images. + MATCHED 0x000000044 source generated based on another image (forced photometry at source location). ON_SPIKE 0x000000088 More than 25% of (PSF-weighted) pixels land on diffraction spike. ON_STARCORE 0x0000001016 More than 25% of (PSF-weighted) pixels land on starcore. @@ -753,12 +753,12 @@ POOR 0x000000022 Used within relphot; skip star. ICRF_QSO 0x000000044 object IDed with known ICRF quasar (may have ICRF position measurement) - OTHEF_QSO 0x000000088 identified as likely QSO (Hernitschek et al 2015), without ICRF reference data - TRANSIENT 0x0000001016 identified as a non-periodic (stationary) transient - VARIABLE 0x0000002032 identified as a periodic variable - RRLYRA 0x0000004064 identified as likely RR Lyra (Hernitschek et al 2015) - HAS_SOLSYS_DET 0x00000080128 identified with a known solar-system object (asteroid or other) - ALL_SOLSYS_DET 0x00000100256 identified with a known solar-system object (asteroid or other) - UNDEF_1 0x00000200512 Unused bit value. - UNDEF_2 0x000004001024 Unused bit value. + HERN_QSO_P60 0x000000088 identified as likely QSO (Hernitschek et al 2015), P_QSO >= 0.60 + HERN_QSO_P05 0x0000001016 identified as possible QSO (Hernitschek et al 2015), P_QSO >= 0.05 + HERN_RRL_P60 0x0000002032 identified as likely RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.60 + HERN_RRL_P05 0x0000004064 identified as possible RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.05 + HERN_VARIABLE 0x00000080128 identified as a variable based on ChiSq (Hernitschek et al 2015) + TRANSIENT 0x00000100256 identified as a non-periodic (stationary) transient + HAS_SOLSYS_DET 0x00000200512 at least one detection identified with a known solar-system object (asteroid or other). + MOST_SOLSYS_DET 0x000004001024 most detections identified with a known solar-system object (asteroid or other). LARGE_PM 0x000008002048 star with large proper motion RAW_AVE 0x000010004096 simple weighted average position was used (no IRLS fitting) @@ -777,8 +777,8 @@ GOOD 0x0200000033554432 good-quality measurement in our data (eg,PS) GOOD_ALT 0x0400000067108864 good-quality measurement in external data (eg, 2MASS) - GOOD_STACK 0x08000000134217728 good-quality object in the stack (> 1 good stack) - BEST_STACK 0x10000000268435456 the primary stack measurement are the best measurements - SUSPECT_STACK 0x20000000536870912 suspect object in the stack (> 1 good or suspect stack, less than 2 good) - BAD_STACK 0x400000001073741824 good-quality object in the stack (> 1 good stack) + GOOD_STACK 0x08000000134217728 good-quality object in the stack (> 1 good stack measurement) + BEST_STACK 0x10000000268435456 the primary stack measurements are the best measurements + SUSPECT_STACK 0x20000000536870912 suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement) + BAD_STACK 0x400000001073741824 poor-quality stack object (no more than 1 good or suspect measurement) @@ -809,16 +809,16 @@ SECF_USE_UBERCAL 0x000000088 Ubercal photometry used in average measurement. SECF_HAS_PS1 0x0000001016 PS1 photometry used in average measurement. - SECF_HAS_STACK 0x0000002032 PS1 stack photometry exists. + SECF_HAS_PS1_STACK0x0000002032 PS1 stack photometry exists. SECF_HAS_TYCHO 0x0000004064 Tycho photometry used for synthetic magnitudes. SECF_FIX_SYNTH 0x00000080128 Synthetic magnitudes repaired with zeropoint map. - PHOTOM_PASS_0 0x00000100256 Average magnitude calculated in 0th pass. - PHOTOM_PASS_1 0x00000200512 Average magnitude calculated in 1th pass. - PHOTOM_PASS_2 0x000004001024 Average magnitude calculated in 2th pass. - PHOTOM_PASS_3 0x000008002048 Average magnitude calculated in 3th pass. - PHOTOM_PASS_4 0x000010004096 Average magnitude calculated in 4th pass. - PSPS_OBJ_EXT 0x000020008192 Extended in this band (PSPS only). + SECF_RANK_0 0x00000100256 Average magnitude uses only rank 0 detections. + SECF_RANK_1 0x00000200512 Average magnitude uses only rank 1 detections. + SECF_RANK_2 0x000004001024 Average magnitude uses only rank 2 detections. + SECF_RANK_3 0x000008002048 Average magnitude uses only rank 3 detections. + SECF_RANK_4 0x000010004096 Average magnitude uses only rank 4 detections. SECF_STACK_PRIMARY0x0000400016384 PS1 stack photometry comes from primary skycell. + SECF_OBJ_EXT 0x0100000016777216 Extended in this band. @@ -847,7 +847,8 @@ QF_OBJ_GOOD 0x000000044 Good-quality measurement in our data (eg,PS). QF_OBJ_GOOD_ALT 0x000000088 Good-quality measurement in external data (eg, 2MASS). - QF_OBJ_GOOD_STACK 0x0000001016 Good-quality object in the stack (> 1 good stack). - QF_OBJ_SUSPECT_STACK0x0000002032 Suspect object in the stack (> 1 good or suspect stack, less tham 2 good). - QF_OBJ_BAD_STACK 0x0000004064 Good-quality object in the stack (> 1 good stack). + QF_OBJ_GOOD_STACK 0x0000001016 good-quality object in the stack (> 1 good stack measurement) + QF_OBJ_BEST_STACK 0x0000002032 the primary stack measurements are the best measurements. + QF_OBJ_SUSPECT_STACK0x0000004064 suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement). + QF_OBJ_BAD_STACK 0x0000008064 poor-quality stack object (no more than 1 good or suspect measurement). Index: trunk/ippToPsps/jython/objectbatch.py =================================================================== --- trunk/ippToPsps/jython/objectbatch.py (revision 39564) +++ trunk/ippToPsps/jython/objectbatch.py (revision 39565) @@ -147,7 +147,12 @@ # find Nsecfilt (or save in the db with dvopsps) - # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? - # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter - # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + # the math below depends on filterCount = Nsecfilt and + # both MeanObject.row and cps.row being 1 counting, and + # filter[0] also being 1 counting + + # cps.row - 1 = (cpt.row - 1)*Nsecfilt + (filter[0] - 1) + # cps.row = cpt.row * Nsecfilt - (Nsecfilt - filter[0]) + # cps.row = cpt.row * Nsecfilt - Nsecfilt + filter[0] + # [note that the sql below has parenthesis around (Nsecfilt - filter[0]) sql = "UPDATE MeanObject JOIN \ @@ -169,6 +174,5 @@ ,MeanObject." + filter[1] + "MeanApMagStd = MAG_AP_STDEV \ ,MeanObject." + filter[1] + "MeanApMagNpt = NUSED_AP \ - ,MeanObject." + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) " - + ,MeanObject." + filter[1] + "Flags = FLAGS " try: self.scratchDb.execute(sql) @@ -301,5 +305,5 @@ sqlLine.group("processingVersion", "'" + str(self.skychunk.processingVersion) + "'") sqlLine.group("objInfoFlag", "FLAGS") - sqlLine.group("qualityFlag", "FLAGS >> 24 & 0xFF") + sqlLine.group("qualityFlag", "FLAGS >> 23 & 0xFF") sqlLine.group("raStack", "RA_STK") sqlLine.group("decStack", "DEC_STK") @@ -339,57 +343,64 @@ self.updateObjectThinFromCps(cpsTableName) - # XXX EAM 20140724 : is this necessary?? - #objects can have out of range ra dec in dvo - need to find and kill them at the end - - self.logger.infoPair("Determining", "ra/dec range") - - raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) - raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) - decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) - decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) - - self.logger.infoPair("R_MIN", raMin) - self.logger.infoPair("R_MAX", raMax) - self.logger.infoPair("D_MIN", decMin) - self.logger.infoPair("D_MAX", decMax) - #count out of range - - sql = "SELECT count(*) FROM ObjectThin where \ - ObjectThin.decMean > " + str(decMax) + " \ - or ObjectThin.decMean < " + str(decMin) + " \ - or ObjectThin.raMean > " + str(raMax) + " \ - or ObjectThin.raMean < " + str(raMin) - - rs = self.scratchDb.executeQuery(sql) - - rs.first() - nToDelete = rs.getInt(1) - - #delete out of range - - - sql = "DELETE FROM ObjectThin where \ - ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ - ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ - ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ - ObjectThin.raMean < (" + str(raMin) + " - .0033)" - self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") - - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) - raise - - ##Don't do this till after MeanObject - ##self.dvoObjects.purgeRegion(self.region) + if False: + # this chunk of code deletes objects which are out of ra,dec range for the table. + # this was a problem in an early version of DVO for cases where the astrometry went insane. + # this causes problems for the ra = 0,360 boundary (the test below does not handle that situation) + # and the restrictions below are poorly defined for the regions near the pole. + + # in any case, ObjectThin needs to maintain the same order + # as the cpt table until MeanObjects have been created or + # the join to the cps table will fail + + # XXX EAM 20140724 : is this necessary?? + # objects can have out of range ra dec in dvo - need to find and kill them at the end + + self.logger.infoPair("Determining", "ra/dec range") + + raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) + raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) + decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) + decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) + + self.logger.infoPair("R_MIN", raMin) + self.logger.infoPair("R_MAX", raMax) + self.logger.infoPair("D_MIN", decMin) + self.logger.infoPair("D_MAX", decMax) + #count out of range + + sql = "SELECT count(*) FROM ObjectThin where \ + ObjectThin.decMean > " + str(decMax) + " \ + or ObjectThin.decMean < " + str(decMin) + " \ + or ObjectThin.raMean > " + str(raMax) + " \ + or ObjectThin.raMean < " + str(raMin) + + rs = self.scratchDb.executeQuery(sql) + + rs.first() + nToDelete = rs.getInt(1) + + #delete out of range + + + sql = "DELETE FROM ObjectThin where \ + ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ + ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ + ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ + ObjectThin.raMean < (" + str(raMin) + " - .0033)" + self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) + raise + self.logger.infoPair("updatePspsUniqueIDs","start") self.updatePspsUniqueIDs() self.logger.infoPair("updatePspsUniqueIDs","end") + self.logger.infoPair("Dropping row column from", "ObjectThin table") self.scratchDb.dropColumn("ObjectThin", "row") - ##self.logger.infoPair("Purging from scratch Db", self.region + " region") self.logger.infoPair("Dropped row column", "objectThin") - ##Don't do this till after MeanObject self.setMinMaxObjID(["ObjectThin"]) @@ -415,9 +426,4 @@ self.scratchDb.addRowCountColumn("MeanObject", "row") - - - - - ##self.scratchDb.addRowCountColumn(cpsTableName, "row") self.logger.infoPair("update MeanObjects from ","cps table") self.updateMeanObjectFromCps(cpsTableName) @@ -434,8 +440,14 @@ if not self.populateObjectThinTable(): return False - #if not self.populateObjectCalColorTable(): return False if not self.populateMeanObjectTable(): return False # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + + ### XXX the code below first removes duplicate objID entries + ### from ObjectThin, then does the same for MeanObject. This + ### is a big problem: we have no guarantee that the surviving + ### rows are the correct matched rows. + + ### force objID uniqueness on *** ObjectThin *** self.logger.infoPair("Forcing uniqueness on", "objID in ObjectThin table") rowCountBefore = self.scratchDb.getRowCount("ObjectThin") @@ -457,4 +469,5 @@ self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + ### force objID uniqueness on *** MeanObject *** self.logger.infoPair("Forcing uniqueness on", "objID in MeanObject table") rowCountBefore = self.scratchDb.getRowCount("MeanObject") @@ -476,9 +489,6 @@ self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + # delete the cpt, cps tables from the scratch mysql (or we will run out of space) self.dvoObjects.purgeRegion(self.region) - #this is abuse of something but this is how I get the object batches to crash to further investigate them - -# rowCountAfter = self.scratchDb.getRowCount("Object") - return True -# return False + return True