Changeset 33193 for trunk/ippToPsps

Timestamp:

Feb 1, 2012, 2:32:57 PM (15 years ago)

Author:

rhenders

Message:

improvements to auditing of regions; some convenience methods for loading regions with a box or box with border

File:

• trunk/ippToPsps/jython/dvo.py (modified) (10 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -54 +54 @@
     def __del__(self):
         self.logger.debug("Dvo destructor")
-
 
     '''
@@ -196 +195 @@
 
         allPopulatedRegions = []
-        alreadyIngested = 0
+        regionsAlreadyIngested = []
         regionsIngestedButOutOfDate = []
 
@@ -212 +211 @@
            # if we have already imported up-to-date versions of both the cpm and cpt then we skip this region
            if self.scratchDb.alreadyImportedThisDvoTable(cpmPath) and self.scratchDb.alreadyImportedThisDvoTable(cptPath): 
-               alreadyIngested = alreadyIngested + 1
+               regionsAlreadyIngested.append(region)
                continue
 
            # check if we have an out-of-date version of either region file, if so, add it to the purge list
-           if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath): 
-               alreadyIngested = alreadyIngested + 1
+           if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath):
                regionsIngestedButOutOfDate.append(region)
 
@@ -230 +228 @@
 
         sizeOfAllPopulatedRegions = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(allPopulatedRegions))
-        sizeOfRegionsAlreadyIngested = self.getFileSizeStr(self.scratchDb.getTotalSizeOfIngestedDvoFiles())
+        sizeOfRegionsAlreadyIngested = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsAlreadyIngested))
         sizeOfRegionsIngestedButOutOfDate = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsIngestedButOutOfDate))
         sizeOfRegionsToPurge = self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge))
@@ -239 +237 @@
         self.logger.infoPair("Dec range", "%.2f -> %.2f" % (minDec, maxDec))
         self.logger.infoPair("Total regions", "%d (%s)" % (len(allPopulatedRegions), sizeOfAllPopulatedRegions)) 
-        self.logger.infoPair("Total regions already ingested", "%d (%s)" % (alreadyIngested, sizeOfRegionsAlreadyIngested)) 
-        self.logger.infoPair("Total out-of-date regions", "%d (%s)" % (len(regionsIngestedButOutOfDate), sizeOfRegionsIngestedButOutOfDate)) 
+        self.logger.infoPair("Up-to-date regions already ingested", "%d (%s)" % (len(regionsAlreadyIngested), sizeOfRegionsAlreadyIngested)) 
+        self.logger.infoPair("Out-of-date regions ingested", "%d (%s)" % (len(regionsIngestedButOutOfDate), sizeOfRegionsIngestedButOutOfDate)) 
         self.logger.infoPair("Total regions to purge", "%d (%s)" % (len(regionsToPurge), sizeOfRegionsToPurge)) 
         self.logger.infoPair("Total regions to ingest", "%d (%s)" % (len(regionsToIngest), sizeOfRegionsToIngest)) 
@@ -287 +285 @@
 
     '''
+    Populates dvoDetections and dvoMeta with FITS tables from DVO for 
+    a RA/Dec box centered with these coords and with this side length PLUS a border
+    '''
+    def loadRegionInThisBoxWithBorder(self, ra=180., dec=0., side=181.0, border=0.):
+
+        halfSide = (side + (border*2.0)) /2.0
+        self.loadRegion(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide)
+    
+    '''
+    Populates dvoDetections and dvoMeta with FITS tables from DVO for 
+    a RA/Dec box centered with these coords and with this side length
+    '''
+    def loadRegionInThisBox(self, regionsToIngest, regionsToPurge, ra=180., dec=0., side=181.0):
+
+        halfSide = side/2.0
+        self.loadRegion(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide)
+    
+    '''
     Populates dvoDetections and dvoMeta with FITS tables from DVO for the defined region
     Defaults to whole sky if limits are missing
@@ -305 +321 @@
         self.purgeRegions(regionsToPurge)
 
-        if len(regionsToIngest) > 0: self.logger.infoSeparator()
+        totalRegions = len(regionsToIngest)
+        if totalRegions > 0: self.logger.infoSeparator()
 
         # now loop through all regions and ingest to MySQL
-        count = 0
+        regionCount = 0
         for region in regionsToIngest:
 
@@ -317 +334 @@
            cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt")
 
-           self.logger.infoTitle("Region: " + region)
+           self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
 
            # if we have not already ingested the cpm table, then do it now
@@ -341 +358 @@
                self.scratchDb.execute(sql)
 
-               # we can report that we have imported this table
+               # we can now report that we have imported this table
                self.scratchDb.setImportedThisDvoTable(cpmPath, region)
 
@@ -405 +422 @@
 
            self.scratchDb.setImportedThisDvoTable(cptPath, region)
-           count = count + 1
+           regionCount = regionCount + 1
 
         self.logger.infoSeparator()
 
-        self.logger.infoPair("Ingested", "%d DVO regions" % count)
+        self.logger.infoPair("Ingested", "%d DVO regions" % regionCount)
 
         return True