Changeset 33098 for branches/meh_branches/ppsub_test/ippToPsps/jython/cleanup.py

Timestamp:

Jan 11, 2012, 11:19:21 PM (15 years ago)

Author:

mhuber

Message:

merged latest r33093 trunk changes to meh_branches/ppsub_test

Location:

branches/meh_branches/ppsub_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/cleanup.py (modified) (2 diffs)

branches/meh_branches/ppsub_test/ippToPsps/jython/cleanup.py

@@ -6 +6 @@
 import re
 import sys
+import time
+import logging.config
 
-import logging.config
 from pslogger import PSLogger
 from ipptopspsdb import IppToPspsDb
 from batch import Batch
+from datastore import Datastore
+from dxlayer import DXLayer
 
+'''
+Finds stuff to delete for this batch type, then:
+
+- removes from the datastore
+- deletes from local disk
+- deletes files from DXLayer
+
+'''
+def clean(batchType):
+
+    logger.infoSeparator()
+    logger.infoTitle("Cleaning up batch type %s" % batchType)
+
+    localIDs = ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, EPOCH, DVOLABEL)
+    datastoreIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, EPOCH, DVOLABEL)
+    dxlayerIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, EPOCH, DVOLABEL)
+    logger.infoPair("Merged but still on local disk", "%d" % len(localIDs))
+    logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(datastoreIDs))
+    logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(dxlayerIDs))
+
+    if DELETELOCAL:
+        count = 0
+        for id in localIDs:
+    
+            if Batch.deleteFromDisk(logger, BASEPATH, batchType, DVOLABEL, id):
+                ippToPspsDb.updateDeletedLocal(id, 1)
+                count = count + 1
+
+        logger.infoPair("Removed from local disk", "%d" % count)
+        logger.infoPair("Remaining on local disk", "%d" % (len(localIDs) - count))
+    
+
+    if DELETEDATASTORE:
+        count = 0
+        for id in datastoreIDs:
+    
+            batchName = Batch.getNameFromID(id)
+
+            if datastore.remove(batchName):
+                count = count + 1
+
+        logger.infoPair("Removed from datastore", "%d" % count)
+        logger.infoPair("Remaining on datastore", "%d" % (len(datastoreIDs) - count))
+    
+    if DELETEDXLAYER:
+        count = 0
+        for id in dxlayerIDs:
+    
+            if dxlayer.deleteBatch(id):
+                ippToPspsDb.updateDeletedDxlayer(id, 1)
+                count = count + 1
+
+        logger.infoPair("Removed from DXLayer", "%d" % count)
+        logger.infoPair("Remaining on DXLayer", "%d" % (len(dxlayerIDs) - count))
+    
+
+'''
+Start of program
+'''
 
 if len(sys.argv) > 1: CONFIG  = sys.argv[1]
 else:
-    print "\n** Usage: " + sys.argv[0] + " <configPath>\n"
+    print "\n** Usage: " + sys.argv[0] + " <configPath> [hours]\n"
     sys.exit(1)
+
+if len(sys.argv) > 2: 
+    HOURS = float(sys.argv[2])
+    SECONDS = HOURS * 60.0 * 60.0
+else: 
+    SECONDS = None
 
 # open config file
@@ -27 +95 @@
 
 ippToPspsDb = IppToPspsDb(logger, configDoc)
+datastore = Datastore(logger, configDoc, ippToPspsDb)
+dxlayer = DXLayer(logger)
 
 DVOLABEL = configDoc.find("dvo/gpc1Label").text
 EPOCH = configDoc.find("options/epoch").text
+BASEPATH = configDoc.find("localOutPath").text
+DELETELOCAL = int(configDoc.find("deletion/local").text)
+DELETEDATASTORE = int(configDoc.find("deletion/datastore").text)
+DELETEDXLAYER = int(configDoc.find("deletion/dxlayer").text)
 
-ids = ippToPspsDb.getMergedButNotDeletedBatchIDs(EPOCH, DVOLABEL)
+logger.infoTitle("ippToPsps clean-up")
+if SECONDS: logger.infoPair("Performing clean-up every", "%f hours" % (SECONDS/60.0/60.0))
+logger.infoBool("Deleting from local disk?", DELETELOCAL)
+logger.infoBool("Deleting from datastore?", DELETEDATASTORE)
+logger.infoBool("Deleting from DXLayer?", DELETEDXLAYER)
 
-logger.infoPair("Merged but un-deleted batches", "%d" % len(ids))
+while True:
 
+    clean("P2")
+    clean("ST")
 
-
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")
-logger.info("|   Batch   | Removed from datastore  | Removed from local disk |   Removed from DXLayer  |")
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")
-
-for id in ids:
+    if SECONDS:
+        logger.infoPair("Waiting for", "%f hours" % HOURS)
+        time.sleep(SECONDS)
+    else: break
     
-    batchName = Batch.getNameFromID(id)
-    removedFromDatastore = 0
-    removedFromLocalDisk = 0
-    removedFromDXLayer = 0
-
-    logger.info("| %9s | %23d | %23d | %23d |",
-            batchName,
-            removedFromDatastore,
-            removedFromLocalDisk,
-            removedFromDXLayer)
-
-logger.info("+-----------+-------------------------+-------------------------+-------------------------+")