Index: /trunk/ippToPsps/jython/Makefile.am =================================================================== --- /trunk/ippToPsps/jython/Makefile.am (revision 39006) +++ /trunk/ippToPsps/jython/Makefile.am (revision 39007) @@ -17,4 +17,5 @@ dvoforceddetections.py \ dvoforcedobjects.py \ + dvoforcedgalaxy.py \ dvoobjects.py \ dvo.py \ Index: /trunk/ippToPsps/jython/dvodetections.py =================================================================== --- /trunk/ippToPsps/jython/dvodetections.py (revision 39006) +++ /trunk/ippToPsps/jython/dvodetections.py (revision 39007) @@ -14,4 +14,5 @@ ''' A class for ingesting DVO detections into MySQL +XXX EAM 20151029 : I do not believe this class is used at all. I am poisoning it with an exit ''' class DvoDetections(Dvo): @@ -21,8 +22,8 @@ ''' def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): -# def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): - + super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) -# super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + print "----- DvoDetections is not used, is it?? -----" + os._exit(1) # declare DVO file types of interest Index: /trunk/ippToPsps/jython/dvoforceddetections.py =================================================================== --- /trunk/ippToPsps/jython/dvoforceddetections.py (revision 39006) +++ /trunk/ippToPsps/jython/dvoforceddetections.py (revision 39007) @@ -14,4 +14,5 @@ ''' A class for ingesting DVO detections into MySQL +XXX EAM 20151029 : I do not believe this class is used at all. I am poisoning it with an exit ''' class DvoForcedDetections(Dvo): @@ -21,8 +22,8 @@ ''' def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): -# def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): super(DvoForcedDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) -# super(DvoForcedDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + print "----- DvoDetections is not used, is it?? -----" + os._exit(1) # declare DVO file types of interest Index: /trunk/ippToPsps/jython/dvoforcedgalaxy.py =================================================================== --- /trunk/ippToPsps/jython/dvoforcedgalaxy.py (revision 39007) +++ /trunk/ippToPsps/jython/dvoforcedgalaxy.py (revision 39007) @@ -0,0 +1,58 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoForcedGalaxy(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): + + super(DvoForcedGalaxy, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) + + # declare DVO file types of interest + self.loadPhotcodes() + + ''' + ingest forced object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + cmd = "dvopsps forced_galaxy_shape" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + return True Index: /trunk/ippToPsps/jython/dvoforcedobjects.py =================================================================== --- /trunk/ippToPsps/jython/dvoforcedobjects.py (revision 39006) +++ /trunk/ippToPsps/jython/dvoforcedobjects.py (revision 39007) @@ -21,11 +21,8 @@ ''' def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): -# def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): super(DvoForcedObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) -# super(DvoForcedObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) # declare DVO file types of interest - self.ingestFileTypes = ['cpt', 'cps', 'cpy'] self.loadPhotcodes() Index: /trunk/ippToPsps/jython/forcedgalaxybatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedgalaxybatch.py (revision 39006) +++ /trunk/ippToPsps/jython/forcedgalaxybatch.py (revision 39007) @@ -70,5 +70,4 @@ self.logger.infoPair("DVO INDEX", "%d" % self.id) - ''' Overriden from batch base class to import directly from DVO for forced galaxy shape parameters @@ -109,5 +108,4 @@ return True - ''' Applies indexes to the IPP Tables @@ -120,9 +118,6 @@ ''' def updatePspsUniqueIDs(self): - sql = "UPDATE ForcedGalaxyShape set uniquePspspFGid = (("+str(self.batchID)+"*1000000000 ) + row)" - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair('failed sql', sql) - raise + sql = "UPDATE ForcedGalaxyShape set uniquePspsFGid = (("+str(self.batchID)+"*1000000000 ) + row)" + self.scratchDb.execute(sql) ''' @@ -145,28 +140,6 @@ # This is going to need to join elsewhere to get all the fields? - # XippDetectID, XforcedSummaryID - # Ra, Dec, RaErr, DecErr - - # This won't work, as we need to join on modeltype/obj_id/filter -# + cpqTable + " AS cpq ON (cpq.row = (ForcedGalaxyShape.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) " \ -# + " ForcedGalaxyShape." + filter[1] + "GalModelType = MODEL_TYPE," \ - + # XforcedSummaryID, Ra, Dec, RaErr, DecErr # Should GalChiSq fold in Npix somehow? -# sql = "UPDATE ForcedGalaxyShape JOIN " \ -# + cpqTable + " AS cpq ON (cpq.row = (ForcedGalaxyShape.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) " \ -# + " SET " + \ -# + " ForcedGalaxyShape." + filter[1] + "ippDetectID = DET_ID," \ -# + " ForcedGalaxyShape." + filter[1] + "forcedSummaryID = IMAGE_ID," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMajor = MAJOR_AXIS," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMajorErr = MAJOR_AXIS_ERR," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMinor = MINOR_AXIS," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMinorErr = MINOR_AXIS_ERR," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMag = MAG," \ -# + " ForcedGalaxyShape." + filter[1] + "GalMagErr = MAG_ERR," \ -# + " ForcedGalaxyShape." + filter[1] + "GalRa = RA," \ -# + " ForcedGalaxyShape." + filter[1] + "GalDec = DEC," \ -# + " ForcedGalaxyShape." + filter[1] + "GalRaErr = RA_ERR," \ -# + " ForcedGalaxyShape." + filter[1] + "GalDecErr = DEC_ERR," \ -# + " ForcedGalaxyShape." + filter[1] + "GalChisq = CHISQ " sqlLine = sqlUtility("UPDATE ForcedGalaxyShape AS a, " + cpqTable + " AS b SET") @@ -179,23 +152,18 @@ sqlLine.group("a." + filter[1] + "GalMag", "b.MAG") sqlLine.group("a." + filter[1] + "GalMagErr", "b.MAG_ERR") -# These fields don't exist. -# sqlLine.group("a." + filter[1] + "GalRa", "b.RA") -# sqlLine.group("a." + filter[1] + "GalDec", "b.DEC") -# sqlLine.group("a." + filter[1] + "GalRaErr", "b.RA_ERR") -# sqlLine.group("a." + filter[1] + "GalDecErr", "b.DEC_ERR") sqlLine.group("a." + filter[1] + "GalChisq", "b.CHISQ") + + # These fields don't exist. + # sqlLine.group("a." + filter[1] + "GalRa", "b.RA") + # sqlLine.group("a." + filter[1] + "GalDec", "b.DEC") + # sqlLine.group("a." + filter[1] + "GalRaErr", "b.RA_ERR") + # sqlLine.group("a." + filter[1] + "GalDecErr", "b.DEC_ERR") + sql = sqlLine.makeEquals("WHERE a.objID = b.OBJ_ID AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + photcode) - - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("failed update ForcedGalaxyShape", sql) - raise - - - - - ''' - Populate ForcedGalaxyShape table + + self.scratchDb.execute(sql) + + ''' + Populate ForcedGalaxyShape table with basic IDs ''' def populateForcedGalaxyShapeTable(self): @@ -212,6 +180,6 @@ # uniquePspsFGid is set elsewhere sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") - # batchID isn't in the schema. -# sqlLine.group("batchID", "'" + str(self.batchID) + "'") + # batchID isn't in the schema, should it be? + # sqlLine.group("batchID", "'" + str(self.batchID) + "'") # surveyID is set where? sqlLine.group("randomForcedGalID", "RAND("+str(self.batchID)+")") @@ -219,9 +187,5 @@ sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't populate Forced Galaxy table from cpt", sql) - raise + self.scratchDb.execute(sql) self.logger.infoPair("Adding 'row' columns to", "Forced Galaxy, cpt tables") @@ -234,5 +198,4 @@ self.logger.infoPair("update ForcedGalaxyShape from ","cpq table") self.updateForcedGalaxyShapeFromCpq(cpqTableName) - self.logger.infoPair("Dropping row column from ", "ForcedGalaxyShape table") @@ -242,5 +205,7 @@ self.setMinMaxObjID(["ForcedGalaxyShape"]) self.logger.infoPair("checking if we have detections", "ForcedGalaxyShape table") + rowcount = self.scratchDb.getRowCount("ForcedGalaxyShape") + if rowcount == 0: self.logger.infoPair("skipping this one", "ForcedGalaxyShape") @@ -256,5 +221,2 @@ if not self.populateForcedGalaxyShapeTable(): return False - - # My example is removing objID duplicates. Since ForcedGalaxyShape can have multiple models for a given object, I'm not doing that. - Index: /trunk/ippToPsps/jython/forcedobjectbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedobjectbatch.py (revision 39006) +++ /trunk/ippToPsps/jython/forcedobjectbatch.py (revision 39007) @@ -114,5 +114,4 @@ return True - ''' Updates table and generates pspsuniqueids @@ -125,7 +124,4 @@ self.logger.errorPair('failed sql',sql) raise - - - ''' @@ -195,10 +191,11 @@ SET " + filter[1] + "PSFMagErr = null \ WHERE " + filter[1] + "PSFMagErr > " + str(cut) -# taking out the cut since this is for mag, not flux - - # try: self.scratchDb.execute(sql) - # except: - # self.logger.errorPair("failed setting null meanmagerr","ForcedMeanObject") - # raise + + # NOTE: taking out the cut since this is for mag, not flux + # try: self.scratchDb.execute(sql) + # except: + # self.logger.errorPair("failed setting null meanmagerr","ForcedMeanObject") + # raise + self.logger.infoPair("Calculating nDetections from", "n[filters]") for filter in filters: @@ -275,5 +272,5 @@ " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanLensing.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ SET \ - ForcedMeanLensing." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ + ForcedMeanLensing." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ ,ForcedMeanLensing." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \ ,ForcedMeanLensing." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \ @@ -321,10 +318,11 @@ sqlLine = sqlUtility("INSERT INTO ForcedMeanObject (") - sqlLine.group("objID", "EXT_ID") - sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly - sqlLine.group("batchID", "'" + str(self.batchID) + "'") - sqlLine.group("randomForcedObjID", "RAND("+str(self.batchID)+")") - sqlLine.group("nDetections", "'0'") - sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + sqlLine.group("objID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("randomForcedObjID", "RAND("+str(self.batchID)+")") + sqlLine.group("nDetections", "'0'") + sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) @@ -344,5 +342,4 @@ self.updatePspsUniqueIDs() - self.logger.infoPair("update ForcedMeanObject from ","cps table") self.updateForcedMeanObjectFromCps(cpsTableName) @@ -350,11 +347,11 @@ sqlLine = sqlUtility("INSERT INTO ForcedMeanLensing (") - sqlLine.group("objID", "objID") - sqlLine.group("uniquePspsFOid","uniquePspsFOid") + sqlLine.group("objID", "objID") + sqlLine.group("uniquePspsFOid", "uniquePspsFOid") sqlLine.group("randomForcedObjID","randomForcedObjID") - sqlLine.group("ippObjID", "ippObjID") - sqlLine.group("batchID", "batchID") + sqlLine.group("ippObjID", "ippObjID") + sqlLine.group("batchID", "batchID") sqlLine.group("nDetections", "nDetections") - sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) sql = sqlLine.makeRaw(") SELECT ", " FROM ForcedMeanObject") @@ -375,10 +372,12 @@ self.logger.infoPair("Dropping row column from", "ForcedMeanObject table") self.scratchDb.dropColumn("ForcedMeanObject", "row") + self.logger.infoPair("Dropping row column from", "ForcedMeanLensing table") self.scratchDb.dropColumn("ForcedMeanLensing", "row") + + ##Don't do this till after MeanObject ##self.logger.infoPair("Purging from scratch Db", self.region + " region") - - ##Don't do this till after MeanObject ##self.dvoForcedObjects.purgeRegion(self.region) + self.logger.infoPair("setting minmaxobjid for ", "ForcedMeanObject table") self.setMinMaxObjID(["ForcedMeanObject"]) @@ -421,8 +420,3 @@ self.dvoForcedObjects.purgeRegion(self.region) - - #this is abuse of something but this is how I get the object batches to crash to further investigate them - -# rowCountAfter = self.scratchDb.getRowCount("Object") return True -# return False