Changeset 33183 for trunk/ippToPsps/jython/dvo.py

Timestamp:

Jan 31, 2012, 9:57:07 AM (14 years ago)

Author:

rhenders

Message:

more stats for what is and isn't ingested; improvments when resetting and starting from scratch; moved safe table naming method to mysql class; lots more logging

File:

• trunk/ippToPsps/jython/dvo.py (modified) (22 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -29 +29 @@
     Constructor
 
+    - sets up logging
+    - checks we are using the correct DVO
+    - checks we have up-to-date Images.dat
+    - checks we have up-to-date SkyTable.fits
+
     '''
     def __init__(self, logger, doc):
@@ -44 +49 @@
         self.correctDvo = self.scratchDb.isThisDvoCurrentlyIngested(self.dvoLocation)
 
+        # using correct DVO, so now get up-to-date versions of Images and SkyTable tables
+        self.updateMetaTables()
+
     '''
     Destructor
@@ -52 +60 @@
 
     '''
+    Updates two main meta tables, Images and SkyTable
+    '''
+    def updateMetaTables(self):
+
+        self.loadImages()
+        self.loadSkyTable()
+
+    '''
     Check we are importing the DVO currenly ingested
     '''
@@ -61 +77 @@
     '''
     def resetDetections(self):
+        self.logger.infoPair("Resetting", "just detections")
         self.scratchDb.resetOnlyDetectionDvoTables()
 
@@ -67 +84 @@
     '''
     def resetAllTables(self):
+
+        self.logger.infoPair("Resetting", "all tables")
         self.scratchDb.resetDvoToMysqlTables()
         self.correctDvo = True
@@ -80 +99 @@
         # check if we have up-to-date version
         path = self.dvoLocation + "/Images.dat"
-        if self.scratchDb.alreadyImportedThisDvoTable(path): return
+        if self.scratchDb.alreadyImportedThisDvoTable(path): 
+            self.logger.infoPair("DVO Images.dat file", "up-to-date")
+            return
+        
+        self.logger.infoSeparator()
+        self.logger.infoPair("DVO Images.dat file", "NOT up-to-date")
 
         # first, delete old dvoMeta table
@@ -86 +110 @@
         sql = "DELETE FROM " + self.scratchDb.dvoMetaTable
         self.scratchDb.execute(sql)
-
+        self.scratchDb.optimizeTable(self.scratchDb.dvoMetaTable)
+    
         self.importFits(
                 path,
@@ -128 +153 @@
 
         path =  self.dvoLocation + "/SkyTable.fits"
-        if self.scratchDb.alreadyImportedThisDvoTable(path): return
+        if self.scratchDb.alreadyImportedThisDvoTable(path): 
+            self.logger.infoPair("DVO SkyTable.fits file", "up-to-date")        
+            return
+
+        self.logger.infoSeparator()
+        self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date")        
         self.importFits(
                 path,
@@ -152 +182 @@
             return False        
 
-        self.logger.infoPair("Finding DVO files for range", 
-                "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec))
-
-
         allRegions = self.scratchDb.getDvoRegionsForThisBox(minRa, maxRa, minDec, maxDec)
         allIngestedRegions = self.scratchDb.getIngestedDvoRegions()
@@ -161 +187 @@
         allPopulatedRegions = []
         alreadyIngested = 0
-        outOfDateIngested = []
+        regionsIngestedButOutOfDate = []
 
         for region in allRegions:
@@ -182 +208 @@
            if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath): 
                alreadyIngested = alreadyIngested + 1
-               outOfDateIngested.append(region)
+               regionsIngestedButOutOfDate.append(region)
 
            regionsToIngest.append(region)
@@ -191 +217 @@
 
         # now add out-of-date items to purge list
-        for item in outOfDateIngested: regionsToPurge.append(item)
-
-        self.logger.infoPair("Total DVO regions", "%d" % len(allPopulatedRegions)) 
-        self.logger.infoPair("Total DVO regions already ingested", "%d" % alreadyIngested) 
-        self.logger.infoPair("Total DVO out-of-date regions", "%d" % len(outOfDateIngested)) 
-        self.logger.infoPair("Total DVO regions to purge", "%d" % len(regionsToPurge)) 
-        self.logger.infoPair("Total DVO regions to ingest", "%d" % len(regionsToIngest)) 
+        for item in regionsIngestedButOutOfDate: regionsToPurge.append(item)
+
+        sizeOfAllPopulatedRegions = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(allPopulatedRegions))
+        sizeOfRegionsAlreadyIngested = self.getFileSizeStr(self.scratchDb.getTotalSizeOfIngestedDvoFiles())
+        sizeOfRegionsIngestedButOutOfDate = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsIngestedButOutOfDate))
+        sizeOfRegionsToPurge = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge))
+        sizeOfRegionsToIngest = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToIngest))
+
+        self.logger.infoTitle("DVO Overview") 
+        self.logger.infoPair("RA range", "%.2f -> %.2f" % (minRa, maxRa))
+        self.logger.infoPair("Dec range", "%.2f -> %.2f" % (minDec, maxDec))
+        self.logger.infoPair("Total regions", "%d (%s)" % (len(allPopulatedRegions), sizeOfAllPopulatedRegions)) 
+        self.logger.infoPair("Total regions already ingested", "%d (%s)" % (alreadyIngested, sizeOfRegionsAlreadyIngested)) 
+        self.logger.infoPair("Total out-of-date regions", "%d (%s)" % (len(regionsIngestedButOutOfDate), sizeOfRegionsIngestedButOutOfDate)) 
+        self.logger.infoPair("Total regions to purge", "%d (%s)" % (len(regionsToPurge), sizeOfRegionsToPurge)) 
+        self.logger.infoPair("Total regions to ingest", "%d (%s)" % (len(regionsToIngest), sizeOfRegionsToIngest)) 
 
 
@@ -204 +239 @@
     '''
     def purgeRegions(self, regions):
+
+        if len(regions) < 1: return
 
         self.logger.infoPair("Purging", "%d regions" % len(regions)) 
@@ -237 +274 @@
 
     '''
-    Returns (and reports to log) total size of DVO files ingested to the scratch Db
-    '''
-    def getSizeAlreadyIngested(self):
-
-        size = self.scratchDb.getTotalSizeOfIngestedDvoFiles()
-        self.logger.infoPair("Total size of DVO already ingested", self.getFileSizeStr(size))
-
-
-    '''
     Populates dvoDetections and dvoMeta with FITS tables from DVO for the defined region
     Defaults to whole sky if limits are missing
@@ -253 +281 @@
         # go no further if we've already partly ingested a different DVO
         if not self.correctDvo: return
+
+        self.updateMetaTables()
 
         regionsToIngest = []
@@ -258 +288 @@
         self.getRegions(regionsToIngest, regionsToPurge, minRa, maxRa, minDec, maxDec)
 
-
-        totalSize = self.getDiskSizeOfDvoRegions(regionsToIngest)
-        self.logger.infoPair("Total size of cpm/cpt files to ingest", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToIngest)))
-        self.logger.infoPair("Total size of cpm/cpt files to purge", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge)))
+        self.purgeRegions(regionsToPurge)
+
+        if len(regionsToIngest) > 0: self.logger.infoSeparator()
+
+        # now loop through all regions and ingest to MySQL
         count = 0
-
-        self.purgeRegions(regionsToPurge)
-
-        self.logger.infoSeparator()
-
-        # now loop through all regions and ingest to MySQL
         for region in regionsToIngest:
 
@@ -274 +299 @@
            cptPath = self.dvoLocation + "/" + region + ".cpt"
 
-           cpmTableName = self.getDatabaseFriendlyTableName(region + ".cpm")
-           cptTableName = self.getDatabaseFriendlyTableName(region + ".cpt")
+           cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm")
+           cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt")
 
            # if we have not already ingested the cpm table, then do it now
            if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath):
-    
+   
+               self.logger.infoTitle("Region: " + region)
+
                # import cpm table and index
                self.importFits(
@@ -372 +399 @@
 
     '''
-    Create Db-friendly table name from file name
-    '''
-    def getDatabaseFriendlyTableName(self, file):
-
-        name = file
-        name = name.replace('.', '_')
-        name = name.replace('/', '_')
-
-        return name
-
-
-    '''
     Imports a FITS file. A regex filter lets you choose which tables to pull from the file
     An optional final argument lets you name the resulatant database table
@@ -390 +405 @@
 
       self.logger.infoPair("Importing file", path)
-      self.logger.infoPair("Writing to database table", tableName)
+      self.logger.debugPair("Writing to database table", tableName)
 
       tables = stilts.treads(path)
@@ -401 +416 @@
      
           # IPP FITS files are littered with infinities, so remove these
-          self.logger.infoPair("Removing", "infinity values")
+          self.logger.debugPair("Removing", "infinity values")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
@@ -410 +425 @@
     
               try:
-                  self.logger.infoPair("          attempt", "%d" % attempts)
+                  if attempts > 0: self.logger.infoPair("Attempt", "%d" % attempts)
                   table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
                   break
               except:
+                  # wait a few secs then try again
                   time.sleep(2)
                   attempts = attempts + 1
@@ -419 +435 @@
           count = count + 1
 
-
-      self.logger.infoPair("Finished importing", tableName)
+      self.logger.debugPair("Finished importing", tableName)
 
       return tableName