Index: trunk/ippToPsps/jython/forcedobjectbatch.py =================================================================== --- trunk/ippToPsps/jython/forcedobjectbatch.py (revision 37918) +++ trunk/ippToPsps/jython/forcedobjectbatch.py (revision 37927) @@ -25,5 +25,5 @@ ''' -class ForcedMeanObjectBatch(Batch): +class ForcedObjectBatch(Batch): ''' @@ -40,5 +40,5 @@ batchID): - super(ForcedMeanObjectBatch, self).__init__( + super(ForcedObjectBatch, self).__init__( logger, config, @@ -71,5 +71,5 @@ self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) - self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region) + self.ippToPspsDb.insertForcedObjectMeta(self.batchID, self.region) self.dvoForcedObjects.nativeIngestRegion(self.region) @@ -86,4 +86,13 @@ return False + self.logger.infoPair("checking if we have anything in"," cpy/cpt") + rowcountcpt = self.scratchDb.getRowCount(cptTableName); + rowcountcpy = self.scratchDb.getRowCount(cpyTableName); + if (rowcountcpt == 0): + self.logger.infoPair("0 rows for ","cpt table") + return False + if (rowcountcpy == 0): + self.logger.infoPair("o rows for ","cpy table") + return False return True @@ -104,4 +113,19 @@ return True + + + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self): + sql = "UPDATE ForcedMeanObject set uniquePspsFOid = (("+str(self.batchID)+"*1000000000 ) + row)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + + ''' @@ -146,16 +170,16 @@ SET \ ForcedMeanObject." + filter[1] + "nTotal = NWARP \ - ,ForcedMeanObject." + filter[1] + "nIncPSFMag = NUSED_WRP \ - ,ForcedMeanObject." + filter[1] + "nIncKronMag = NUSED_KRON_WRP \ - ,ForcedMeanObject." + filter[1] + "nIncApMag = NUSED_AP_WRP \ - ,ForcedMeanObject." + filter[1] + "PSFMag = MAG_PSF_WRP \ - ,ForcedMeanObject." + filter[1] + "PSFMagErr = MAG_PSF_WRP_ERR \ - ,ForcedMeanObject." + filter[1] + "PSFMagStd = MAG_PSF_WRP_STD \ - ,ForcedMeanObject." + filter[1] + "KronMag = MAG_KRON_WRP \ - ,ForcedMeanObject." + filter[1] + "KronMagErr = MAG_KRON_WRP_ERR \ - ,ForcedMeanObject." + filter[1] + "KronMagStd = MAG_KRON_WRP_STD \ - ,ForcedMeanObject." + filter[1] + "ApMag = MAG_AP_WRP \ - ,ForcedMeanObject." + filter[1] + "ApMagErr = MAG_AP_WRP_ERR \ - ,ForcedMeanObject." + filter[1] + "ApMagStd = MAG_AP_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "nIncPSFFlux = NUSED_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncKronFlux = NUSED_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncApFlux = NUSED_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFFlux = FLUX_PSF_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFFluxErr = FLUX_PSF_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "PSFFluxStd = FLUX_PSF_WRP_STDEV \ + ,ForcedMeanObject." + filter[1] + "KronFlux = FLUX_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "KronFluxErr = FLUX_KRON_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "KronFluxStd = FLUX_KRON_WRP_STDEV \ + ,ForcedMeanObject." + filter[1] + "ApFlux = FLUX_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "ApFluxErr = FLUX_AP_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "ApFluxStd = FLUX_AP_WRP_STDEV \ ,ForcedMeanObject." + filter[1] + "Flags = FLAGS " @@ -163,5 +187,5 @@ try: self.scratchDb.execute(sql) except: - self.logger.errorPair("failed update MeanObject", sql) + self.logger.errorPair("failed update ForcedMeanObject", sql) raise @@ -175,13 +199,17 @@ SET " + filter[1] + "PSFMagErr = null \ WHERE " + filter[1] + "PSFMagErr > " + str(cut) - self.scratchDb.execute(sql) - +# taking out the cut since this is for mag, not flux + + # try: self.scratchDb.execute(sql) + # except: + # self.logger.errorPair("failed setting null meanmagerr","ForcedMeanObject") + # raise self.logger.infoPair("Calculating nDetections from", "n[filters]") for filter in filters: # now do a sum of n[filters], but do not include the ones with -999 - sql = "UPDATE ForcedMeanObject" + sql = "UPDATE ForcedMeanObject " sql += "SET nDetections = nDetections + " + filter[1] +"nTotal " - sql += " WHERE n" + filter[1] + " != -999" - self.scratchDb.execute(sql) + sql += " WHERE " + filter[1] + "nTotal != -999" + #self.scratchDb.execute(sql) try: self.scratchDb.execute(sql) except: @@ -192,5 +220,5 @@ Inserts stuff for all mags from the cpy files ''' - def updateForcedMeanObjectFromCpy(self, cpyTable): + def updateForcedMeanTablesFromCpy(self, cpyTable): # list of all filters PSPS is interested in @@ -229,5 +257,5 @@ " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ SET \ - ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \ + ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \ ,ForcedMeanObject." + filter[1] + "nIncR6 = NMEAS \ ,ForcedMeanObject." + filter[1] + "FmeanflxR5 = FLUX_AP_R5 \ @@ -239,32 +267,40 @@ ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std = FLUX_STD_AP_R6 \ ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill = FLUX_FIL_AP_R6 \ - ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \ - ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \ - ,ForcedMeanObject." + filter[1] + "LensPSFShearE2 = E2_SH_PSF \ ,ForcedMeanObject." + filter[1] + "Gamma = GAMMA \ ,ForcedMeanObject." + filter[1] + "E1 = E1 \ ,ForcedMeanObject." + filter[1] + "E2 = E2 " - try: self.scratchDb.execute(sql) except: - self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql) + self.logger.errorPair("failed update on ForcedMeanObject using cpy table",sql) + raise + + sql = "UPDATE ForcedMeanLensing JOIN \ + " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanLensing.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ForcedMeanLensing." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \ + ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \ + ,ForcedMeanLensing." + filter[1] + "LensPSFShearE2 = E2_SH_PSF " + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update on ForcedMeanLensing using cpy table", sql) raise @@ -299,32 +335,65 @@ except: self.logger.errorPair("Couldn't populate Object table from cpt", sql) - return False + raise # add row count columns so we can perform joins to get colors - self.logger.infoPair("Adding 'row' columns to", "Object and cps tables") + self.logger.infoPair("Adding 'row' columns to", "Object ,cps, cpy tables") self.scratchDb.addRowCountColumn("ForcedMeanObject", "row") self.scratchDb.addRowCountColumn(cpsTableName, "row") + self.scratchDb.addRowCountColumn(cpyTableName, "row") + + self.logger.infoPair("generating unique ids for ","ForcedMeanObject") + self.updatePspsUniqueIDs() self.logger.infoPair("update ForcedMeanObject from ","cps table") - self.updateForcedMeanObjectFromCps(cpsTableName) - self.logger.infoPair("update ForcedMeanObject from ","cpy table") - - self.updateForcedMeanObjectFromCpy(cpyTableName) + sqlLine = sqlUtility("INSERT INTO ForcedMeanLensing (") + + sqlLine.group("objID", "objID") + sqlLine.group("uniquePspsFOid","uniquePspsFOid") + sqlLine.group("ippObjID", "ippObjID") + sqlLine.group("batchID", "batchID") + sqlLine.group("nDetections", "nDetections") + sqlLine.group("gStackID", "gStackID") + sqlLine.group("rStackID", "rStackID") + sqlLine.group("iStackID", "iStackID") + sqlLine.group("zStackID", "zStackID") + sqlLine.group("yStackID", "yStackID") + + sql = sqlLine.makeRaw(") SELECT ", " FROM ForcedMeanObject") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate ForcedMeanLensing table from ForcedMeanObject", sql) + raise + + self.logger.infoPair("adding row to ","ForcedMeanLensing") + self.scratchDb.addRowCountColumn("ForcedMeanLensing", "row") + + self.logger.infoPair("updating ForcedMeanObject and ForcedMeanLensing from ","cpy tables") + self.updateForcedMeanTablesFromCpy(cpyTableName) self.logger.infoPair("Dropping row column from", "ForcedMeanObject table") self.scratchDb.dropColumn("ForcedMeanObject", "row") - + self.logger.infoPair("Dropping row column from", "ForcedMeanLensing table") + self.scratchDb.dropColumn("ForcedMeanLensing", "row") ##self.logger.infoPair("Purging from scratch Db", self.region + " region") ##Don't do this till after MeanObject ##self.dvoForcedObjects.purgeRegion(self.region) - + self.logger.infoPair("setting minmaxobjid for ", "ForcedMeanObject table") self.setMinMaxObjID(["ForcedMeanObject"]) - - return True + self.logger.infoPair("checking if we have detctions", "FOrcedMeanObject table") + rowcount = self.scratchDb.getRowCount("ForcedMeanObject") + if rowcount == 0: + self.logger.infoPair("skipping this one","ForcedMeanObject ") + self.skipBatch = False + return True + else: + return True