Changeset 33177

Timestamp:

Jan 27, 2012, 4:33:05 PM (14 years ago)

Author:

rhenders

Message:

changes to keep track of, and purge, ingested-but-out-of-date regions

Location:

trunk/ippToPsps/jython

Files:

• dvo.py (modified) (3 diffs)
• scratchdb.py (modified) (1 diff)

trunk/ippToPsps/jython/dvo.py

@@ -161 +161 @@
         allPopulatedRegions = []
         alreadyIngested = 0
+        outOfDateIngested = []
 
         for region in allRegions:
@@ -178 +179 @@
                continue
 
+           # check if we have an out-of-date version of either region file, if so, add it to the purge list
+           if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath): 
+               alreadyIngested = alreadyIngested + 1
+               outOfDateIngested.append(region)
+
            regionsToIngest.append(region)
 
-        # get list of ingested regions to remove
+        # add out-of-range regions for purge list
         items = list(set(allIngestedRegions) - set(allPopulatedRegions))
         for item in items: regionsToPurge.append(item)
 
-        self.logger.infoPair("Total populated DVO regions", "%d" % len(allPopulatedRegions)) 
+        # now add out-of-date items to purge list
+        for item in outOfDateIngested: regionsToPurge.append(item)
+
+        self.logger.infoPair("Total DVO regions", "%d" % len(allPopulatedRegions)) 
         self.logger.infoPair("Total DVO regions already ingested", "%d" % alreadyIngested) 
+        self.logger.infoPair("Total DVO out-of-date regions", "%d" % len(outOfDateIngested)) 
         self.logger.infoPair("Total DVO regions to purge", "%d" % len(regionsToPurge)) 
         self.logger.infoPair("Total DVO regions to ingest", "%d" % len(regionsToIngest)) 
@@ -269 +279 @@
            # if we have not already ingested the cpm table, then do it now
            if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath):
-           
+    
                # import cpm table and index
                self.importFits(

trunk/ippToPsps/jython/scratchdb.py

@@ -184 +184 @@
 
     '''
+    Do we have an out-of-date version of this file?
+    '''
+    def haveOutOfDateVersionOfThisDvoTable(self, path):
+
+        fileStat = os.stat(path)
+
+        sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate != " + str(fileStat.st_mtime) 
+
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            if rs.getInt(1) > 0:
+                self.logger.debugPair("Have out-to-date version of",  path)
+                return True
+            else:
+                return False
+        except:
+            self.logger.exception("Unable to check whether this DVO table is out-of-date")
+
+    '''
     Have we already imported this DVO table?
+    Checks path and modified date to be sure
     '''
     def alreadyImportedThisDvoTable(self, path):