Changeset 33351 for trunk/ippToPsps/jython/cleanup.py
- Timestamp:
- Feb 23, 2012, 4:14:32 PM (14 years ago)
- File:
-
- 1 edited
-
trunk/ippToPsps/jython/cleanup.py (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps/jython/cleanup.py
r33265 r33351 9 9 import logging.config 10 10 11 from config import Config 12 from pslogger import PSLogger 13 from ipptopspsdb import IppToPspsDb 11 from ipptopsps import IppToPsps 14 12 from batch import Batch 15 13 from datastore import Datastore 16 14 from dxlayer import DXLayer 17 15 18 '''19 Finds stuff to delete for this batch type, then:20 21 - removes from the datastore22 - deletes from local disk23 - deletes files from DXLayer24 16 25 17 ''' 26 def clean(batchType): 18 Cleanup class 19 ''' 20 class Cleanup(IppToPsps): 27 21 28 logger.infoSeparator() 29 logger.infoTitle("Cleaning up batch type %s" % batchType) 22 ''' 23 Constructor 24 ''' 25 def __init__(self, argv): 26 super(Cleanup, self).__init__(argv) 30 27 31 loadedDatastoreIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, config.epoch, config.dvoLabel) 32 loadedDxlayerIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, config.epoch, config.dvoLabel) 33 mergedLocalIDs = ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, config.epoch, config.dvoLabel) 28 if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) 34 29 35 purgedDatastoreIDs = ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, config.epoch, config.dvoLabel)36 purgedLocalIDs = ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, config.epoch, config.dvoLabel)37 30 38 logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) 39 logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) 40 logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) 41 logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) 42 logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) 31 ''' 32 Overrides base-class version 33 ''' 34 def refreshConfig(self): 35 ret = super(Cleanup, self).refreshConfig() 36 self.dxlayer = DXLayer(self.logger) 37 self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) 38 return ret 43 39 44 if config.deleteLocal: 45 logger.infoSeparator() 46 count = 0 47 ids = mergedLocalIDs + purgedLocalIDs 48 for id in ids: 40 ''' 41 Runs... 42 ''' 43 def run(self): 44 45 if self.SECONDS: self.logger.infoPair("Performing clean-up every", "%f hours" % (self.SECONDS/60.0/60.0)) 46 while True: 47 48 self.config.printDeletionPolicy() 49 self.clean("P2") 50 self.clean("ST") 51 52 self.checkClientStatus() 53 if not self.waitForPollTime(): break 54 55 ''' 56 Finds stuff to delete for this batch type, then: 57 58 - removes from the datastore 59 - deletes from local disk 60 - deletes files from DXLayer 61 62 ''' 63 def clean(self, batchType): 64 65 self.logger.infoSeparator() 66 self.logger.infoTitle("Cleaning up batch type %s" % batchType) 67 68 loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) 69 loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel) 70 mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) 71 72 purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) 73 purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) 74 75 self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) 76 self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) 77 self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) 78 self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) 79 self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) 49 80 50 if Batch.deleteFromDisk(logger, config.basePath, batchType, config.dvoLabel, id): 51 ippToPspsDb.updateDeletedLocal(id, 1) 52 count = count + 1 81 if self.config.deleteLocal: 82 self.logger.infoSeparator() 83 count = 0 84 ids = mergedLocalIDs + purgedLocalIDs 85 for id in ids: 86 87 if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): 88 self.ippToPspsDb.updateDeletedLocal(id, 1) 89 count = count + 1 90 91 self.logger.infoSeparator() 92 self.logger.infoPair("Removed from local disk", "%d" % count) 93 self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) 94 95 96 if self.config.deleteDatastore: 97 self.logger.infoSeparator() 98 count = 0 99 ids = loadedDatastoreIDs + purgedDatastoreIDs 100 for id in ids: 101 102 batchName = Batch.getNameFromID(id) 103 104 if self.datastore.remove(batchName): 105 count = count + 1 106 107 self.logger.infoSeparator() 108 self.logger.infoPair("Removed from datastore", "%d" % count) 109 self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count)) 110 111 if self.config.deleteDxLayer: 112 self.logger.infoSeparator() 113 count = 0 114 for id in loadedDxlayerIDs: 115 116 if self.dxlayer.deleteBatch(id): 117 self.ippToPspsDb.updateDeletedDxlayer(id, 1) 118 count = count + 1 119 120 self.logger.infoSeparator() 121 self.logger.infoPair("Removed from DXLayer", "%d" % count) 122 self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count)) 123 53 124 54 logger.infoSeparator() 55 logger.infoPair("Removed from local disk", "%d" % count) 56 logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) 57 125 ''' 126 Overrides base-class version 127 ''' 128 def printUsage(self): 129 super(Cleanup, self).printUsage() 130 print " [<poll time in hours>]" 58 131 59 if config.deleteDatastore:60 logger.infoSeparator()61 count = 062 ids = loadedDatastoreIDs + purgedDatastoreIDs63 for id in ids:64 65 batchName = Batch.getNameFromID(id)66 67 if datastore.remove(batchName):68 count = count + 169 70 logger.infoSeparator()71 logger.infoPair("Removed from datastore", "%d" % count)72 logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count))73 74 if config.deleteDxLayer:75 logger.infoSeparator()76 count = 077 for id in loadedDxlayerIDs:78 79 if dxlayer.deleteBatch(id):80 ippToPspsDb.updateDeletedDxlayer(id, 1)81 count = count + 182 83 logger.infoSeparator()84 logger.infoPair("Removed from DXLayer", "%d" % count)85 logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count))86 87 132 88 133 ''' 89 134 Start of program 90 135 ''' 91 92 if len(sys.argv) > 1: CONFIGPATH = sys.argv[1] 93 else: 94 print "\n** Usage: " + sys.argv[0] + " <configPath> [hours]\n" 95 sys.exit(1) 96 97 if len(sys.argv) > 2: 98 HOURS = float(sys.argv[2]) 99 SECONDS = HOURS * 60.0 * 60.0 100 else: 101 SECONDS = None 102 103 # create objects 104 config = Config(CONFIGPATH) 105 logger = config.getLogger(sys.argv[0]) 106 ippToPspsDb = IppToPspsDb(logger, config) 107 datastore = Datastore(logger, config, ippToPspsDb) 108 dxlayer = DXLayer(logger) 109 110 111 logger.infoTitle("ippToPsps clean-up") 112 if SECONDS: logger.infoPair("Performing clean-up every", "%f hours" % (SECONDS/60.0/60.0)) 113 while True: 114 115 config.printDeletionPolicy() 116 clean("P2") 117 clean("ST") 118 119 if SECONDS: 120 logger.infoPair("Waiting for", "%f hours" % HOURS) 121 time.sleep(SECONDS) 122 else: break 123 config.refresh() 124 136 cleanup = Cleanup(sys.argv) 137 cleanup.run() 138 cleanup.exitProgram("finished")
Note:
See TracChangeset
for help on using the changeset viewer.
