Changeset 33415 for branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/cleanup.py (modified) (1 diff)

branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py

@@ -6 +6 @@
 import re
 import sys
+import time
+import logging.config
 
-import logging.config
-from pslogger import PSLogger
-from ipptopspsdb import IppToPspsDb
+from ipptopsps import IppToPsps
 from batch import Batch
+from datastore import Datastore
+from dxlayer import DXLayer
 
 
-if len(sys.argv) > 1: CONFIG  = sys.argv[1]
-else:
-    print "\n** Usage: " + sys.argv[0] + " <configPath>\n"
-    sys.exit(1)
+'''
+Cleanup class
+'''
+class Cleanup(IppToPsps):
 
-# open config file
-configDoc = ElementTree(file=CONFIG)
+    '''
+    Constructor
+    '''
+    def __init__(self, argv):
+        super(Cleanup, self).__init__(argv)
 
-# set up logging
-logging.setLoggerClass(PSLogger)
-logger = logging.getLogger(sys.argv[0])
-logger.setup(configDoc, sys.argv[0])
-
-ippToPspsDb = IppToPspsDb(logger, configDoc)
-
-DVOLABEL = configDoc.find("dvo/gpc1Label").text
-EPOCH = configDoc.find("options/epoch").text
-
-ids = ippToPspsDb.getMergedButNotDeletedBatchIDs(EPOCH, DVOLABEL)
-
-logger.infoPair("Merged but un-deleted batches", "%d" % len(ids))
+        if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) 
 
 
+    '''
+    Overrides base-class version
+    '''
+    def refreshConfig(self):
+        ret = super(Cleanup, self).refreshConfig()
+        self.dxlayer = DXLayer(self.logger)
+        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
+        return ret
 
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")
-logger.info("|   Batch   | Removed from datastore  | Removed from local disk |   Removed from DXLayer  |")
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")
+    '''
+    Runs...
+    '''
+    def run(self):
 
-for id in ids:
+        if self.SECONDS: self.logger.infoPair("Performing clean-up every", "%f hours" % (self.SECONDS/60.0/60.0))
+        while True:
+
+            self.config.printDeletionPolicy()
+            self.clean("P2")
+            self.clean("ST")
+
+            self.checkClientStatus()
+            if not self.waitForPollTime(): break
+   
+    '''
+    Finds stuff to delete for this batch type, then:
+
+    - removes from the datastore
+    - deletes from local disk
+    - deletes files from DXLayer
+
+    '''
+    def clean(self, batchType):
+
+        self.logger.infoSeparator()
+        self.logger.infoTitle("Cleaning up batch type %s" % batchType)
+
+        loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel)
+        loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel)
+        mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel)
+
+        purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel)
+        purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel)
+
+        self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs))
+        self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs))
+        self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs))
+        self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs))
+        self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs))
     
-    batchName = Batch.getNameFromID(id)
-    removedFromDatastore = 0
-    removedFromLocalDisk = 0
-    removedFromDXLayer = 0
+        if self.config.deleteLocal:
+            self.logger.infoSeparator()
+            count = 0
+            ids = mergedLocalIDs + purgedLocalIDs
+            for id in ids:
+        
+                if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id):
+                    self.ippToPspsDb.updateDeletedLocal(id, 1)
+                    count = count + 1
+    
+            self.logger.infoSeparator()
+            self.logger.infoPair("Removed from local disk", "%d" % count)
+            self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count))
+        
+    
+        if self.config.deleteDatastore:
+            self.logger.infoSeparator()
+            count = 0
+            ids = loadedDatastoreIDs + purgedDatastoreIDs
+            for id in ids:
+        
+                batchName = Batch.getNameFromID(id)
+    
+                if self.datastore.remove(batchName):
+                    count = count + 1
+    
+            self.logger.infoSeparator()
+            self.logger.infoPair("Removed from datastore", "%d" % count)
+            self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count))
+        
+        if self.config.deleteDxLayer:
+            self.logger.infoSeparator()
+            count = 0
+            for id in loadedDxlayerIDs:
+        
+                if self.dxlayer.deleteBatch(id):
+                    self.ippToPspsDb.updateDeletedDxlayer(id, 1)
+                    count = count + 1
+    
+            self.logger.infoSeparator()
+            self.logger.infoPair("Removed from DXLayer", "%d" % count)
+            self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count))
+       
 
-    logger.info("| %9s | %23d | %23d | %23d |",
-            batchName,
-            removedFromDatastore,
-            removedFromLocalDisk,
-            removedFromDXLayer)
+    '''
+    Overrides base-class version
+    '''
+    def printUsage(self):
+        super(Cleanup, self).printUsage()
+        print " [<poll time in hours>]"
 
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")
+
+'''
+Start of program
+'''
+cleanup = Cleanup(sys.argv)
+cleanup.run()
+cleanup.exitProgram("finished")