Index: trunk/ippToPsps/jython/dvo.py =================================================================== --- trunk/ippToPsps/jython/dvo.py (revision 33177) +++ trunk/ippToPsps/jython/dvo.py (revision 33183) @@ -29,4 +29,9 @@ Constructor + - sets up logging + - checks we are using the correct DVO + - checks we have up-to-date Images.dat + - checks we have up-to-date SkyTable.fits + ''' def __init__(self, logger, doc): @@ -44,4 +49,7 @@ self.correctDvo = self.scratchDb.isThisDvoCurrentlyIngested(self.dvoLocation) + # using correct DVO, so now get up-to-date versions of Images and SkyTable tables + self.updateMetaTables() + ''' Destructor @@ -52,4 +60,12 @@ ''' + Updates two main meta tables, Images and SkyTable + ''' + def updateMetaTables(self): + + self.loadImages() + self.loadSkyTable() + + ''' Check we are importing the DVO currenly ingested ''' @@ -61,4 +77,5 @@ ''' def resetDetections(self): + self.logger.infoPair("Resetting", "just detections") self.scratchDb.resetOnlyDetectionDvoTables() @@ -67,4 +84,6 @@ ''' def resetAllTables(self): + + self.logger.infoPair("Resetting", "all tables") self.scratchDb.resetDvoToMysqlTables() self.correctDvo = True @@ -80,5 +99,10 @@ # check if we have up-to-date version path = self.dvoLocation + "/Images.dat" - if self.scratchDb.alreadyImportedThisDvoTable(path): return + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.infoPair("DVO Images.dat file", "up-to-date") + return + + self.logger.infoSeparator() + self.logger.infoPair("DVO Images.dat file", "NOT up-to-date") # first, delete old dvoMeta table @@ -86,5 +110,6 @@ sql = "DELETE FROM " + self.scratchDb.dvoMetaTable self.scratchDb.execute(sql) - + self.scratchDb.optimizeTable(self.scratchDb.dvoMetaTable) + self.importFits( path, @@ -128,5 +153,10 @@ path = self.dvoLocation + "/SkyTable.fits" - if self.scratchDb.alreadyImportedThisDvoTable(path): return + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.infoPair("DVO SkyTable.fits file", "up-to-date") + return + + self.logger.infoSeparator() + self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") self.importFits( path, @@ -152,8 +182,4 @@ return False - self.logger.infoPair("Finding DVO files for range", - "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec)) - - allRegions = self.scratchDb.getDvoRegionsForThisBox(minRa, maxRa, minDec, maxDec) allIngestedRegions = self.scratchDb.getIngestedDvoRegions() @@ -161,5 +187,5 @@ allPopulatedRegions = [] alreadyIngested = 0 - outOfDateIngested = [] + regionsIngestedButOutOfDate = [] for region in allRegions: @@ -182,5 +208,5 @@ if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath): alreadyIngested = alreadyIngested + 1 - outOfDateIngested.append(region) + regionsIngestedButOutOfDate.append(region) regionsToIngest.append(region) @@ -191,11 +217,20 @@ # now add out-of-date items to purge list - for item in outOfDateIngested: regionsToPurge.append(item) - - self.logger.infoPair("Total DVO regions", "%d" % len(allPopulatedRegions)) - self.logger.infoPair("Total DVO regions already ingested", "%d" % alreadyIngested) - self.logger.infoPair("Total DVO out-of-date regions", "%d" % len(outOfDateIngested)) - self.logger.infoPair("Total DVO regions to purge", "%d" % len(regionsToPurge)) - self.logger.infoPair("Total DVO regions to ingest", "%d" % len(regionsToIngest)) + for item in regionsIngestedButOutOfDate: regionsToPurge.append(item) + + sizeOfAllPopulatedRegions = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(allPopulatedRegions)) + sizeOfRegionsAlreadyIngested = self.getFileSizeStr(self.scratchDb.getTotalSizeOfIngestedDvoFiles()) + sizeOfRegionsIngestedButOutOfDate = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsIngestedButOutOfDate)) + sizeOfRegionsToPurge = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge)) + sizeOfRegionsToIngest = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToIngest)) + + self.logger.infoTitle("DVO Overview") + self.logger.infoPair("RA range", "%.2f -> %.2f" % (minRa, maxRa)) + self.logger.infoPair("Dec range", "%.2f -> %.2f" % (minDec, maxDec)) + self.logger.infoPair("Total regions", "%d (%s)" % (len(allPopulatedRegions), sizeOfAllPopulatedRegions)) + self.logger.infoPair("Total regions already ingested", "%d (%s)" % (alreadyIngested, sizeOfRegionsAlreadyIngested)) + self.logger.infoPair("Total out-of-date regions", "%d (%s)" % (len(regionsIngestedButOutOfDate), sizeOfRegionsIngestedButOutOfDate)) + self.logger.infoPair("Total regions to purge", "%d (%s)" % (len(regionsToPurge), sizeOfRegionsToPurge)) + self.logger.infoPair("Total regions to ingest", "%d (%s)" % (len(regionsToIngest), sizeOfRegionsToIngest)) @@ -204,4 +239,6 @@ ''' def purgeRegions(self, regions): + + if len(regions) < 1: return self.logger.infoPair("Purging", "%d regions" % len(regions)) @@ -237,13 +274,4 @@ ''' - Returns (and reports to log) total size of DVO files ingested to the scratch Db - ''' - def getSizeAlreadyIngested(self): - - size = self.scratchDb.getTotalSizeOfIngestedDvoFiles() - self.logger.infoPair("Total size of DVO already ingested", self.getFileSizeStr(size)) - - - ''' Populates dvoDetections and dvoMeta with FITS tables from DVO for the defined region Defaults to whole sky if limits are missing @@ -253,4 +281,6 @@ # go no further if we've already partly ingested a different DVO if not self.correctDvo: return + + self.updateMetaTables() regionsToIngest = [] @@ -258,15 +288,10 @@ self.getRegions(regionsToIngest, regionsToPurge, minRa, maxRa, minDec, maxDec) - - totalSize = self.getDiskSizeOfDvoRegions(regionsToIngest) - self.logger.infoPair("Total size of cpm/cpt files to ingest", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToIngest))) - self.logger.infoPair("Total size of cpm/cpt files to purge", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge))) + self.purgeRegions(regionsToPurge) + + if len(regionsToIngest) > 0: self.logger.infoSeparator() + + # now loop through all regions and ingest to MySQL count = 0 - - self.purgeRegions(regionsToPurge) - - self.logger.infoSeparator() - - # now loop through all regions and ingest to MySQL for region in regionsToIngest: @@ -274,10 +299,12 @@ cptPath = self.dvoLocation + "/" + region + ".cpt" - cpmTableName = self.getDatabaseFriendlyTableName(region + ".cpm") - cptTableName = self.getDatabaseFriendlyTableName(region + ".cpt") + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") # if we have not already ingested the cpm table, then do it now if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath): - + + self.logger.infoTitle("Region: " + region) + # import cpm table and index self.importFits( @@ -372,16 +399,4 @@ ''' - Create Db-friendly table name from file name - ''' - def getDatabaseFriendlyTableName(self, file): - - name = file - name = name.replace('.', '_') - name = name.replace('/', '_') - - return name - - - ''' Imports a FITS file. A regex filter lets you choose which tables to pull from the file An optional final argument lets you name the resulatant database table @@ -390,5 +405,5 @@ self.logger.infoPair("Importing file", path) - self.logger.infoPair("Writing to database table", tableName) + self.logger.debugPair("Writing to database table", tableName) tables = stilts.treads(path) @@ -401,5 +416,5 @@ # IPP FITS files are littered with infinities, so remove these - self.logger.infoPair("Removing", "infinity values") + self.logger.debugPair("Removing", "infinity values") table = stilts.tpipe(table, cmd='replaceval -Infinity null *') table = stilts.tpipe(table, cmd='replaceval Infinity null *') @@ -410,8 +425,9 @@ try: - self.logger.infoPair(" attempt", "%d" % attempts) + if attempts > 0: self.logger.infoPair("Attempt", "%d" % attempts) table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) break except: + # wait a few secs then try again time.sleep(2) attempts = attempts + 1 @@ -419,6 +435,5 @@ count = count + 1 - - self.logger.infoPair("Finished importing", tableName) + self.logger.debugPair("Finished importing", tableName) return tableName