Changeset 34041 for branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py

Timestamp:

Jun 19, 2012, 5:24:19 PM (14 years ago)

Author:

mhuber

Message:

merging latest r34040 trunk changes to branch

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/cleanup.py (modified) (9 diffs)

branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py

@@ -7 +7 @@
 import sys
 import time
+import traceback
 import logging.config
 
@@ -13 +14 @@
 from datastore import Datastore
 from dxlayer import DXLayer
-
 
 '''
@@ -26 +26 @@
         super(Cleanup, self).__init__(argv)
 
+        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
+        self.dxlayer = DXLayer(self.logger)
         if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) 
 
-
-    '''
-    Overrides base-class version
-    '''
-    def refreshConfig(self):
-        ret = super(Cleanup, self).refreshConfig()
-        self.dxlayer = DXLayer(self.logger)
-        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
-        return ret
 
     '''
@@ -46 +39 @@
         while True:
 
+            self.logger.infoSeparator()
+            self.logger.infoPair("Config", self.config.name)
             self.config.printDeletionPolicy()
+            self.clean("IN")
             self.clean("P2")
             self.clean("ST")
+            self.clean("OB")
 
             self.checkClientStatus()
             if not self.waitForPollTime(): break
    
+    '''
+    Reports results of cleanup for this set of IDs 
+    '''
+    def reportResults(self, name, ids, success):
+
+        if len(ids) > 0:
+            self.logger.infoSeparator()
+            self.logger.infoPair("Removed from " + name, "%d of %d" % (success, len(ids)))
+    
+
     '''
     Finds stuff to delete for this batch type, then:
@@ -66 +73 @@
         self.logger.infoTitle("Cleaning up batch type %s" % batchType)
 
-        loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel)
-        loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel)
-        mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel)
+        loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType)
+        deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType)
+        mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType)
 
-        purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel)
-        purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel)
+        purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType)
+        purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType)
 
-        self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs))
-        self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs))
-        self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs))
-        self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs))
-        self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs))
+        deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs
+        deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs
+
+        # report to log what we are going to do
+        self.logger.infoPair("To delete from datastore", "%d" % len(deleteFromDatastoreIDs))
+        self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs))
+        self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs))
     
+        # delete stuff from local disk
         if self.config.deleteLocal:
-            self.logger.infoSeparator()
             count = 0
-            ids = mergedLocalIDs + purgedLocalIDs
-            for id in ids:
+            for id in deleteFromLocalIDs:
         
                 if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id):
@@ -89 +97 @@
                     count = count + 1
     
-            self.logger.infoSeparator()
-            self.logger.infoPair("Removed from local disk", "%d" % count)
-            self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count))
-        
+            self.reportResults("local disk", deleteFromLocalIDs, count) 
     
+        # remove stuff from datastore
         if self.config.deleteDatastore:
-            self.logger.infoSeparator()
             count = 0
-            ids = loadedDatastoreIDs + purgedDatastoreIDs
-            for id in ids:
+            for id in deleteFromDatastoreIDs:
         
                 batchName = Batch.getNameFromID(id)
@@ -105 +109 @@
                     count = count + 1
     
-            self.logger.infoSeparator()
-            self.logger.infoPair("Removed from datastore", "%d" % count)
-            self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count))
+            self.reportResults("datastore", deleteFromDatastoreIDs, count) 
         
+        # remove stuff from DXLayer
         if self.config.deleteDxLayer:
-            self.logger.infoSeparator()
             count = 0
-            for id in loadedDxlayerIDs:
+            for id in deleteFromDxLayerIDs:
         
                 if self.dxlayer.deleteBatch(id):
@@ -118 +120 @@
                     count = count + 1
     
-            self.logger.infoSeparator()
-            self.logger.infoPair("Removed from DXLayer", "%d" % count)
-            self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count))
+            self.reportResults("DXLayer", deleteFromDxLayerIDs, count) 
        
-
     '''
     Overrides base-class version
     '''
     def printUsage(self):
-        super(Cleanup, self).printUsage()
-        print " [<poll time in hours>]"
+        super(Cleanup, self).printUsage("[<poll time in hours>]")
 
 
@@ -134 +132 @@
 Start of program
 '''
-cleanup = Cleanup(sys.argv)
-cleanup.run()
-cleanup.exitProgram("finished")
+try:
+    cleanup = Cleanup(sys.argv)
+    cleanup.run()
+    cleanup.exitProgram("completed")
+except Exception, e:
+    print str(e)
+    traceback.print_exc()