Index: trunk/ippToPsps/jython/dvo.py =================================================================== --- trunk/ippToPsps/jython/dvo.py (revision 34107) +++ trunk/ippToPsps/jython/dvo.py (revision 35097) @@ -33,9 +33,11 @@ ''' - def __init__(self, logger, config, scratchDbName="ipptopsps_scratch"): + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None): # set up logging self.logger = logger self.config = config + self.skychunk = skychunk + self.ippToPspsDb = ippToPspsDb # set up empty lists @@ -46,18 +48,25 @@ self.regionsIngestedButOutOfDate = [] - # connect to the specified scratch database + # connect to the specified scratch database, if a name is given, otherwise use the first version try: - self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName) + self.scratchDb = ScratchDb(self.logger, self.config, '1', scratchDbName) except: raise + # we have (still) 3 modes of ingest from dvo cpt/cps/cpm files + # 1) dvograbber (useFullTables == FALSE), + # 2) jython/stilts (useStilts == TRUE, useFullTables == TRUE) + # 3) dvopsps (useStilts == FALSE, useFullTables == TRUE) + # my goal (EAM, 2013/01/31) is to migrate to only dvopsps + + self.useStilts = 0 self.scratchDb.setUseFullTables(True) # decide if we are using the right DVO for this scratchDb - self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation) + self.correctDvo = self.scratchDb.isCorrectDvo(self.skychunk.dvoLocation) # this scratch Db either has no DVO info ingested, or a different DVO ingested. Prompt user if not self.correctDvo: print "*******************************************************************************" - self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "'?") + self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.skychunk.dvoLocation + "'?") response = raw_input("(y/n) ") if response == "y": @@ -101,5 +110,5 @@ # check if we have up-to-date version - path = self.config.dvoLocation + "/Photcodes.dat" + path = self.skychunk.dvoLocation + "/Photcodes.dat" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Photcodes.dat file", "up-to-date") @@ -126,5 +135,5 @@ # check if we have up-to-date version - path = self.config.dvoLocation + "/Images.dat" + path = self.skychunk.dvoLocation + "/Images.dat" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Images.dat file", "up-to-date") @@ -151,5 +160,5 @@ if not self.correctDvo: return False - path = self.config.dvoLocation + "/SkyTable.fits" + path = self.skychunk.dvoLocation + "/SkyTable.fits" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO SkyTable.fits file", "up-to-date") @@ -158,8 +167,33 @@ self.logger.infoSeparator() self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") - self.importFits( + + if self.useStilts: + self.importFits( path, "R_MIN R_MAX D_MIN D_MAX INDEX NAME", self.scratchDb.dvoSkyTable) + else: + + # create dvoSkyTable + sql = "drop TABLE dvoSkyTable" + try: + self.scratchDb.execute(sql) + except: pass + + # create dvoSkyTable + sql = "CREATE TABLE dvoSkyTable (R_MIN REAL, R_MAX REAL, D_MIN REAL, D_MAX REAL, INDEX_ INT, NAME CHAR(18))" + self.scratchDb.execute(sql) + + # TODO path to DVO prog hardcoded temporarily + cmd = "dvopsps skytable" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) @@ -168,5 +202,5 @@ self.scratchDb.setImportedThisDvoTable(path) - self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation) + self.logger.infoPair("Finished importing SkyTable at", self.skychunk.dvoLocation) return True @@ -231,5 +265,5 @@ paths = [] for fileType in self.ingestFileTypes: - paths.append(self.config.dvoLocation + "/" + regionPath + "." + fileType) + paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType) # check for the existence of all interested file types @@ -304,5 +338,5 @@ # go no further if we've already partly ingested a different DVO if not self.correctDvo: - self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation) + self.logger.infoPair("Wrong DVO in use", self.skychunk.dvoLocation) return @@ -387,8 +421,10 @@ size = 0.0 for region in regions: - + self.logger.infoPair("sizes for region", region) # get combined size of all interested files for fileType in self.ingestFileTypes: - size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + "." + fileType) + size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType) + # EAM TEST I/O + self.logger.infoPair(fileType, size) return size @@ -519,5 +555,5 @@ try: if attempts > 0: self.logger.infoPair("Attempt %d to write" % attempts, table.name) - table.cmd_keepcols("CODE NAME TYPE_AS_INT").write(self.scratchDb.url + '#' + tableName) + table.cmd_keepcols("CODE NAME TYPE_AS_INT C_LAM K").write(self.scratchDb.url + '#' + tableName) break except: @@ -532,2 +568,79 @@ return tableName + + ''' + ingest skyregion into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize): + + # XXX put the chunk below in a separate method + # blow away existing dvoDetection table + + # clear the 'ingested' field for all boxes + self.ippToPspsDb.clearIngestedBoxes() + + # drop detections table + self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable) + self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable) + + # create detections table + self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable) + sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable) + return False + self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable) + + # add fileID column + sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable) + return False + + # add an index to the fileID column + self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID") + + # make sure we have an up-to-date Images table + self.loadImages() + + # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx + halfSize = boxSize / 2.0 + # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize + + # TODO path to DVO prog hardcoded temporarily + cmd = "dvopsps detections" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -region " + cmd += " " + str(raCenter-halfSize) + cmd += " " + str(raCenter+halfSize) + cmd += " " + str(decCenter-halfSize) + cmd += " " + str(decCenter+halfSize) + + if self.skychunk.parallel: + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + + self.ippToPspsDb.setIngestedBox(boxId) + + # add fileID column + sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("Unable to analyze mysql table", self.scratchDb.dvoDetectionTable) + return False + + # update lists after attempted sync + # self.printSummary() + + return True