Index: trunk/ippm/FilesMonitoring/NodeRelated/deployment/generate_crontabs_and_deployment_script.py =================================================================== --- trunk/ippm/FilesMonitoring/NodeRelated/deployment/generate_crontabs_and_deployment_script.py (revision 35985) +++ (revision ) @@ -1,41 +1,0 @@ -#!/usr/bin/env python - -bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - -REFRESH_SCRIPT_LOCATION = '%s/refresh_table.py' % bin_directory -ROOT_DIRECTORY = '%s/deployment' % share_directory -PYTHONPATH = '%s/python' % share_directory -NODES_LIST = '%s/allStorageNodes' % share_directory -OUTPUT_DIRECTORY = '%s/scripts' % share_directory -DEPLOYMENT_SCRIPT = '%s/deploy.sh' % OUTPUT_DIRECTORY -NODE_SCRIPT_FORMAT = '%s/crontab.%%s' % OUTPUT_DIRECTORY - -if __name__ == '__main__': - nodes_file = open(NODES_LIST) - deployment_script_file = open(DEPLOYMENT_SCRIPT, 'w') - deployment_script_file.write('#!/bin/bash\n\n') - minutes = 1 - hours = 0 - for node in nodes_file: - node = node[:-1] - print '... Generating script for %s' % node - crontab_filename = NODE_SCRIPT_FORMAT % node - deployment_script_file.write("echo '%s'\n" % (node)) - deployment_script_file.write("ssh %s 'crontab %s'\n" % (node, crontab_filename)) - crontab_file = open(crontab_filename, 'w') - crontab_file.write("MAILTO=%s@ifa.hawaii.edu\n" % os.environ['USER'] ) - crontab_file.write("PYTHONPATH=%s\n" % PYTHONPATH) - crontab_file.write("%d %d * * * nice %s -disk 0\n" % (minutes, hours, REFRESH_SCRIPT_LOCATION)) - if not node.startswith('ipp'): #atrc or stsci nodes: add disks 1 and 2 - crontab_file.write("%d %d * * * nice %s -disk 1\n" % ((minutes+1)%60, hours, REFRESH_SCRIPT_LOCATION)) - crontab_file.write("%d %d * * * nice %s -disk 2\n" % ((minutes+2)%60, hours, REFRESH_SCRIPT_LOCATION)) - minutes += 7 - minutes %= 60 - hours += 1 - hours %= 24 - crontab_file.close() - nodes_file.close() - deployment_script_file.close() - print '\nNow run: %s\n' % DEPLOYMENT_SCRIPT - Index: trunk/ippm/FilesMonitoring/NodeRelated/refresh_table.py =================================================================== --- trunk/ippm/FilesMonitoring/NodeRelated/refresh_table.py (revision 35985) +++ (revision ) @@ -1,555 +1,0 @@ -#!/usr/bin/env python - -import sys -import os -import socket -import subprocess -import MySQLdb -import logging -import _mysql_exceptions -import datetime -import time - -################################### -# -class Constants: # Constants and/or default values - force = False # if we want to ignore the lock - limit_undefined = 10000 # the maximum number of undefined entries that we want to refresh - limit = 1000 # the maximum number of the oldest refreshed entries that we want to refresh - loggingLevel = logging.INFO # - skip_stages = [] -# Place we find all the installed IPP stuff - bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - -# Templates for things we'll either create or read - node_directory = '/export/%s.%s/ipp_filemon/' - mlocate_db_template = "%s/%s", (node_directory,'mlocate/%s.%s.%s.mlocate_db') - lock_directory = '/tmp/diskMonitoring/%s' # disk - lock_filename_template = '%s/lock.%s' - nebulous_directory_template = '/export/%s.%s/nebulous/%s' - -# Locations of the binaries we'll run - updatedb_binary = '%s/updatedb' % bin_directory - locate_binary = '%s/locate' % bin_directory - update_table_undefined_python_script = "%s/update_table_undefined.py" % bin_directory - update_table_python_script = '%s/update_table.py' % bin_directory - - disk = "0" # Most of the nodes have only one disk /export/.0 - next_directory = None - max_refresh_interval = 6*60*60 # Run the script if the last_modified value associated to the disk to refresh is older than that (6 hours) - # sleep = 60*10 # The time (in seconds) the script sleeps between two iterations - sleep = 1 # The time (in seconds) the script sleeps between two iterations - -def get_next_directory(current_directory): - """ - >>> get_next_directory("00") - '01' - >>> get_next_directory("ff") - '00' - >>> get_next_directory("ab") - 'ac' - >>> get_next_directory("cf") - 'd0' - """ - if Constants.next_directory is None: - Constants.next_directory = dict() - letters = '0123456789abcdef' - directories = [] - for letter1 in letters: - for letter2 in letters: - directories.append('%s%s'% (letter1, letter2)) - for index in range(len(directories)-1): - Constants.next_directory[directories[index]] = directories[index+1] - Constants.next_directory['ff'] = '00' - return Constants.next_directory[current_directory] - -################################### -# -class FilesDatabaseConnection: - def __init__(self, nodename, disk): - self.nodename = nodename - self.disk = disk - if nodename == 'neverland': - self.db = MySQLdb.connect('localhost', 'root', 'root', 'DisksMonitoring') - else: - self.db = MySQLdb.connect('localhost', 'root', '', 'DisksMonitoring') - self.count_inserts = 0 - - def close(self): - self.db.commit() - self.db.close() - - def find_instance(self, neb_ins_id, subdirname, filename): - query = 'SELECT neb_ins_id FROM neb_%s_%s WHERE neb_ins_id = %s' % (self.dirname, - self.disk, - neb_ins_id) - cursor = self.db.cursor() - cursor.execute(query) - rows = cursor.fetchall() - if len(rows) == 0: - print 'New entry' - elif len(rows) == 1: - print 'Existing entry' - else: - print 'Problem' - cursor.close() - - def get_instance_ids(self): - query = 'SELECT neb_ins_id, subdirname, filename FROM neb_%s_%s' % (self.disk, - self.dirname) - cursor = self.db.cursor() - cursor.execute(query) - ids = dict() - for row in cursor.fetchall(): - ids[row[0]] = (row[1], row[2]) - cursor.close() - return ids - - def add_new_file(self, filename, neb_ins_id, subdirname, debug = False): - query = """INSERT INTO neb_%s_%s ( - filename, - neb_ins_id, - subdirname, - nebkey, - creation_date) -VALUES ( - '%s', - %s, - '%s', - '%s', - NOW())""" % (self.disk, - self.dirname, - filename, - neb_ins_id, - subdirname, - generate_nebulous_key(filename)) - if debug: - logger.debug('%s;' % query) - return - cursor = self.db.cursor() - try: - cursor.execute(query) - except _mysql_exceptions.OperationalError, e: - logger.error(str(e)) - logger.info('Trying to replay query: %s' % query) - cursor.execute(query) - self.count_inserts += 1 - cursor.close() - if self.count_inserts >= 100: - self.db.commit() - self.count_inserts = 0 - - def mark_as_deleted(self, neb_ins_id): - table = "neb_%s_%s" % (self.disk, self.dirname) - logger.debug("Deleting entry %d from %s" % (neb_ins_id, table)) - cursor = self.db.cursor() - statement = "INSERT INTO %s_delete(filename, neb_ins_id, subdirname, nebkey, size, md5sum, role, stage, lifetime, status, creation_date, last_check) SELECT filename, neb_ins_id, subdirname, nebkey, size, md5sum, role, stage, lifetime, status, creation_date, last_check FROM %s WHERE neb_ins_id = %d" % (table, table, neb_ins_id) - logger.info('... Statement: (%s)' % (statement)) - try: - cursor.execute(statement) - except _mysql_exceptions.IntegrityError, e: - logger.error(e) - logger.error("Table is %s_delete" % (table)) - if 'Duplicate entry' in str(e): - pass - else: - raise e - except _mysql_exceptions.OperationalError, e: - logger.error(e) - if 'Lock wait timeout exceeded' in str(e): - pass - else: - raise e - statement = 'UPDATE %s_delete SET status=\'deleted\' WHERE neb_ins_id=%d' % (table, neb_ins_id) - logger.info('... Statement: (%s)' % (statement)) - cursor.execute(statement) - statement = 'DELETE FROM %s WHERE neb_ins_id=%d' % (table, neb_ins_id) - logger.info('... Statement: (%s)' % (statement)) - cursor.execute(statement) - cursor.close() - - def optimize_table(self): - cursor = self.db.cursor() - query = 'OPTIMIZE TABLE neb_%s_%s' % (self.disk, - self.dirname) - logger.debug('... Running statement "%s"' % (query)) - cursor.execute(query) - query = 'OPTIMIZE TABLE neb_%s_%s_delete' % (self.disk, - self.dirname) - logger.debug('... Running statement "%s"' % (query)) - cursor.execute(query) - cursor.close() - self.commit() - - def commit(self): - self.db.commit() - - def get_directory_to_refresh(self, force): - cursor = self.db.cursor() - cursor.execute('SELECT dirname, last_modified FROM refresh_status WHERE disk=%s' % self.disk) - for row in cursor.fetchall(): - (dirname, last_modified) = row - cursor.close() - then = last_modified - now = datetime.datetime.now() - td = now - then - delta_seconds = td.seconds + td.days * 24 * 3600 - if delta_seconds < Constants.max_refresh_interval and not force: - logger.warn("No need to restart the script") - sys.exit(0) - self.dirname = get_next_directory(dirname) - return self.dirname - - def update_refresh_status(self, dirname): - self.dirname = dirname - cursor = self.db.cursor() - cursor.execute("UPDATE refresh_status SET dirname = '%s' WHERE disk=%s" % (self.dirname, self.disk)) - self.commit() - cursor.close() - -# ################################### -# # -# class SummaryDatabaseConnection: -# def __init__(self): -# self.nodename = nodename -# self.dirname = dirname -# if nodename == 'neverland': -# self.db = MySQLdb.connect('localhost', 'root', 'root', 'DiskMonitoringSummary') -# else: -# self.db = MySQLdb.connect('localhost', 'ipp', 'ipp', 'DiskMonitoringSummary') - -################################### -# generate_nebulous_key() -def generate_nebulous_key(filename): - elements = filename.split('.') - neb_key = '.'.join(elements[1:]).replace(':', '/') - return neb_key - -################################### -# process_options() -def process_options(arguments): - # Default values: see usage() - force = Constants.force - limit = Constants.limit - skip_stages = Constants.skip_stages - index = 1 - disk = Constants.disk - while index] [-disk ] - %s -h - %s -unittest - - Refreshes files monitoring information about the - different /export/%s.0/nebulous/ nebulous storage - directories. is one of the 00..ff directories. - is obtained from the local database (DisksMonitoring/Status). - - The different stages are: - 0. Initializations. Exit if the information about the last refreshed - directory has been updated less than 24 hours ago. - 1. Get the directory to refresh - 2. Run updatedb on the current directory - 3. Remove deleted files from the FilesMonitoring database - 4. Add new files in the FilesMonitoring database - 5. Refresh the status of the oldest entries in the corresponding - table. - 6. Update the information about the directory. Goto 1 - - Arguments: - None: - : The nebulous directory to be refreshed 00..ff - - Options: - -f: - Force to ignore the lock. Beware of multiple instances of this - program running at the same time - -h, -help, --help: - Displays this help - -v: - Be verbose - -l/-limit : - Update up to new and then oldest files in the monitoring - database. By default, is %d - -disk : - Run for /export/./nebulous. - Default is /export/.%s/nebulous - -skip [,...]: - Debugging mode: Skip - You are supposed to know what you are doing when using this - option! - -unittest - Runs the unit tests associated to this program -""" - nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] - sys.stderr.write(usage.__doc__ % (sys.argv[0], sys.argv[0], sys.argv[0], - nodename, Constants.limit, Constants.disk)) - -def refresh(dirname): - logger.info('Refreshing [%s]' % dirname) - ####################### - # 1. Stage 1: Run updatedb on directory /export/%s.%s/nebulous/ - ####################### - logger.info('Stage 1: updatedb') - if '1' not in skip_stages: - directory = Constants.nebulous_directory_template % (nodename, disk, dirname) - mlocate_db = Constants.mlocate_db_template % (nodename, - disk, - nodename, - disk, - dirname) - mlocate_db_directory = "/".join(mlocate_db.split('/')[:-1]) - if not os.path.exists(mlocate_db_directory): - os.makedirs(mlocate_db_directory) - p = subprocess.Popen( [Constants.updatedb_binary, - '-l', '0', - '-U', directory, - '-o', mlocate_db], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - if p.wait()!=0: - logger.error('Error while running updatedb') - for line in p.stderr: - logger.error(line) - sys.exit(3) - logger.info('... Stage 1 complete') - else: - logger.info('... Stage 1 skipped') - - ####################### - # Stage 1.9: Collect all ins_id from the database and the mlocate file - ####################### - logger.info('Stage 1.9 (cannot be skipped): Get ins_id both from database and mlocate file') - if '2' in skip_stages and '3' in skip_stages: #Ignore if there is no database update - logger.info('... Stage 1.9 skipped since both 2 and 3 have to be skipped') - else: - # Collect ins_id from the database - ins_ids_in_db = filesMonitoring_database.get_instance_ids() - logger.debug('... Got %d entries from the database' % len(ins_ids_in_db)) - # Collect ins_id from the locate file - mlocate_db = Constants.mlocate_db_template % (nodename, - disk, - nodename, - disk, - dirname) - p = subprocess.Popen( [Constants.locate_binary, - '-d', mlocate_db, - '.'], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - p.poll() - count = 0 - acceptable = 0 - root = Constants.nebulous_directory_template % (nodename, - disk, - dirname) - ins_ids_in_mlocate = dict() - for line in p.stdout: - long_filename = line[:-1].replace(root, - '') - if long_filename.startswith('/'): - long_filename = long_filename[1:] - try: - (subdirname, filename) = long_filename.split('/') - if logger.isEnabledFor(logging.DEBUG): - logger.debug('... -> %s' % line[:-1]) - logger.debug('...... [%s] split in (%s,%s)' % (long_filename, - subdirname, - filename)) - fields = filename.split('.') - if fields[0] != '': - ins_ids_in_mlocate[int(fields[0])] = (filename, subdirname) - else: - logger.warn('... Check file [%s]' % line[:-1]) - acceptable += 1 - except ValueError: - pass # These are the directories - count += 1 - logger.debug('... Looked at %d entries in updatedb database' % count) - logger.debug('... Found %d acceptable entries in updatedb database' % acceptable) - if acceptable+257!=count: - # We expect to have 256 subdirectories and the root dir - # excluded, that is exactly count-257 acceptable entries - # in the updatedb database. If not, it means that there is - # an unexpected file in the root directory - logger.debug('... !!! There might be a problem in updatedb database !!!') - logger.debug('... Got %d entries from the mlocate file' % len(ins_ids_in_mlocate)) - # Differences - logger.debug('... Checking differences') - deleted_entries = dict() - existing_entries = dict() - new_entries = dict() - for key in ins_ids_in_db.keys(): - if ins_ids_in_mlocate.has_key(key): - existing_entries[key] = ins_ids_in_mlocate[key] - else: - if ins_ids_in_db.has_key(key): - logger.debug('Adding [%s, %s] to files to delete' % (ins_ids_in_db[key])) - deleted_entries[key] = ins_ids_in_db[key] - else: - logger.debug('No entry in database for key [%s]' % key) - for key in ins_ids_in_mlocate.keys(): - if ins_ids_in_db.has_key(key): - existing_entries[key] = ins_ids_in_db[key] - else: - new_entries[key] = ins_ids_in_mlocate[key] - logger.info('... New files.............: %d' % len(new_entries)) - logger.info('... Deleted files.........: %d' % len(deleted_entries)) - logger.info('... Already existing files: %d' % (len(existing_entries))) - logger.info('... Stage 1.9 complete') - - ####################### - # 2. Stage 2: Remove deleted files from the FilesMonitoring database - ####################### - logger.info('Stage 2: remove deleted files from the FilesMonitoring database') - if '2' not in skip_stages: - logger.info('... %d entries to delete' % len(deleted_entries)) - for neb_ins_id in deleted_entries.keys(): - filesMonitoring_database.mark_as_deleted(neb_ins_id) - filesMonitoring_database.commit() - logger.info('... Stage 2 complete') - else: - logger.info('... Stage 2 skipped') - - ####################### - # 3. Add new files to the FilesMonitoring database - ####################### - logger.info('Stage 3: add new files to the FilesMonitoring database') - if '3' not in skip_stages: - logger.info('... %d entries to add' % len(new_entries)) - for neb_ins_id in new_entries.keys(): - filesMonitoring_database.add_new_file(new_entries[neb_ins_id][0], - neb_ins_id, - new_entries[neb_ins_id][1]) - filesMonitoring_database.commit() - logger.info('... Stage 3 complete') - else: - logger.info('... Stage 3 skipped') - - ####################### - # 3.9. OPTIMIZE TABLE - ####################### - logger.info('Stage 3.9 (cannot be skipped): optimize table') - if '2' in skip_stages and '3' in skip_stages: #Ignore if no db update - logger.info('... Stage 3.9 skipped since both 2 and 3 have to be skipped') - else: - filesMonitoring_database.optimize_table() - logger.info('... Stage 3.9 complete') - - ####################### - # 4. Refresh the status of the oldest entries in the - # corresponding table - ####################### - logger.info('Stage 4: refresh the status of the oldest entries (limit = %d)' % limit) - if '4' not in skip_stages: - p = subprocess.Popen( [Constants.update_table_undefined_python_script, - '-disk', disk, - '-l', str(Constants.limit_undefined), - dirname], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - if p.wait()!=0: - logger.error('Error while running %s' % Constants.update_table_undefined_python_script) - for line in p.stderr: - logger.error(line) - sys.exit(3) - p = subprocess.Popen( [Constants.update_table_python_script, - '-disk', disk, - '-l', str(limit), - dirname], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - if p.wait()!=0: - logger.error('Error while running %s' % Constants.update_table_python_script) - for line in p.stderr: - logger.error(line) - sys.exit(3) - logger.info('... Stage 4 complete') - else: - logger.info('... Stage 4 skipped') - - ################################################## - # 5. Stage 5.9: Update information in refresh_status table - ################################################## - filesMonitoring_database.update_refresh_status(dirname) - -################################### -# Main -if __name__ == '__main__': - ####################### - # 0. Initializations - ####################### - # Get nodename - nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] - # 0.1. Setup logging - logger = logging.getLogger() - formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', - '%Y-%m-%dT%H:%M:%S') -# log_directory = '/data/%s.0/schastel/logs' % (nodename) - log_directory = ("%s/logs" % (node_directory)) % (nodename, disk) - log_filename = '%s/%s_refresh.log' % (log_directory, nodename) - try: - logging_output = logging.FileHandler(log_filename) - except IOError: - os.makedirs(log_directory) - logging_output = logging.FileHandler(log_filename) - logging_output.setFormatter(formatter) - logger.addHandler(logging_output) - logger.setLevel(Constants.loggingLevel) - # 0.2. Get program arguments - (force, limit, skip_stages, disk) = process_options(sys.argv) - logger.info('%s for [%s]/%s' % (sys.argv[0], nodename, disk)) - logger.info('Stage 0 (cannot be skipped): initializations') - # 0.3. Open a connection to the FilesMonitoring database - try: - filesMonitoring_database = FilesDatabaseConnection(nodename, disk) - except _mysql_exceptions.OperationalError, e: - logger.error('Caught exception while connecting to local MySQL server:\n\t%s' % str(e)) - sys.exit(1) - dirname = filesMonitoring_database.get_directory_to_refresh(force) - logger.info('Stage 0 complete') - while True: - refresh(dirname) - dirname = get_next_directory(dirname) - filesMonitoring_database.dirname = dirname # Dirty! - logger.info('Sleeping for a while (%d seconds)' % Constants.sleep) - time.sleep(Constants.sleep) Index: trunk/ippm/FilesMonitoring/NodeRelated/update_table.py =================================================================== --- trunk/ippm/FilesMonitoring/NodeRelated/update_table.py (revision 35985) +++ (revision ) @@ -1,204 +1,0 @@ -#!/usr/bin/env python - -import MySQLdb -import sys -import socket -import os -import datetime -import hashlib -from ipp.lifetime_templates import Template - -def md5sum(filename): - f = open(filename, "rb") - content = f.read() - f.close() - sum = hashlib.md5() - sum.update(content) - return sum.hexdigest() - -class DbEntry: - verbose = False - def __init__(self, db_key, full_filename): - self.db_key = db_key - self.full_filename = full_filename - - def __str__(self): - if not DbEntry.verbose and len(self.db_key) > 10: - db_key = '%s...' % self.db_key[:10] - full_filename = '...%s' % self.full_filename[-10:] - else: - db_key = self.db_key - full_filename = self.full_filename - if self.role != 'deleted': - return '[%s]: %s (%s/%s/%s), %d, %s, %s' % (db_key, - full_filename, - self.stage, - self.role, - self.lifetime, - self.size, - self.md5sum, - self.last_check) - else: - return '[%s]: %s (%s)' % (db_key, - full_filename, - self.role) - - def check(self): - try: - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - stat_values = os.stat(self.full_filename) - self.size = stat_values.st_size - try: - self.md5sum = md5sum(self.full_filename) - self.status = 'ok' - except IOError: - self.status = 'error' - self.md5sum = None - except OSError: - self.status = 'deleted' - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - self.last_check = datetime.datetime.now() - - def update(self, table): - if self.status == 'deleted': - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - stmt = """UPDATE %s -SET status=\'%s\', - last_check=\'%s\', - role=\'%s\', - stage=\'%s\', - lifetime=\'%s\' WHERE filename=\'%s\';""" % ( table, - self.status, - self.last_check, - self.role, - self.stage, - self.lifetime, - self.db_key ) - return stmt - stmt = """ -UPDATE %s -SET - size=%d, - md5sum=\'%s\', - role=\'%s\', - lifetime=\'%s\', - last_check=\'%s\', - stage=\'%s\', - status=\'%s\' -WHERE filename=\'%s\'; -""" % ( table, - self.size, - self.md5sum, - self.role, - self.lifetime, - self.last_check, - self.stage, - self.status, - self.db_key ) - return stmt - -def usage(): - print - print 'Usage: %s [-v] [-l | -limit ] ' % sys.argv[0] - print ' Update the 100 last checked entries of FilesMonitoring _ table' - print ' where is the name of the host where this command is run and' - print ' is a nebulous directory name (range: 00 - ff)' - print - print ' * -h: display help' - print ' * -v: be verbose' - print ' * -l , -limit : process no more than entries (default 10)' - print - print ' e.g.: %s [-v] 00' % sys.argv[0] - print ' e.g.: %s [-v] 01' % sys.argv[0] - print ' e.g.: ..' - print ' e.g.: %s [-v] ff' % sys.argv[0] - print - -def process_options(argv): - limit = 10 - directory = None - argc = 1 - while argc < len(argv): - if sys.argv[argc] == '-h': - usage() - sys.exit(0) - elif sys.argv[argc] == '-v': - DbEntry.verbose = True - argc += 1 - elif sys.argv[argc] == '-limit' or sys.argv[argc] == '-l' : - #TODO: Make this POSIX - limit = int(sys.argv[argc+1]) - argc += 2 - elif sys.argv[argc] == '-disk': - disk = sys.argv[argc+1] - argc += 2 - else: - directory = sys.argv[argc] - argc += 1 - if directory is None: - usage() - sys.exit(1) - return limit, directory, disk - -def process_batch(filesMonitoring, disk, table_name, limit): - cursor = filesMonitoring.cursor() - query = 'SELECT filename, subdirname FROM %s ORDER BY last_check LIMIT %d' % ( table_name, - limit) - if DbEntry.verbose: - sys.stderr.write('%s\n' % query) - cursor.execute(query) - rows = cursor.fetchall() - cursor.close() - entries = [ ] - for row in rows: - filename = row[0] - subdirname = row[1] - full_filename = '/export/%s.%s/nebulous/%s/%s/%s' % ( hostname, - disk, - directory, - subdirname, - filename) - entries.append(DbEntry(filename, full_filename)) - for entry in entries: - entry.check() - count_updates = 0 - cursor = filesMonitoring.cursor() - for entry in entries: - query = entry.update(table_name) - count_updates += 1 - cursor.execute(query) - if DbEntry.verbose: - print query - cursor.close() - filesMonitoring.commit() - sys.stderr.write('\t# of updates: %d\n' % count_updates) - -####################################################################################### -if __name__ == '__main__': - if len(sys.argv) == 1: - usage() - sys.exit(1) - userlimit, directory, disk = process_options(sys.argv) - hostname = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] - table_name = 'neb_%s_%s' % (disk, directory) - sys.stderr.write('Updating: %s %s %s\n' % (hostname, directory, table_name)) - if hostname == 'neverland': - filesMonitoring = MySQLdb.connect( 'localhost', - 'root', 'root', - 'DisksMonitoring' ) - else: - filesMonitoring = MySQLdb.connect( 'localhost', - 'root', '', - 'DisksMonitoring' ) - max_batch_size = 2500 - if userlimit > max_batch_size: - sys.stderr.write('Batching by sets of %d\n' % max_batch_size) - count = 0 - while count < userlimit: - sys.stderr.write('Processing %d out of %d\n' % (count, userlimit) ) - process_batch(filesMonitoring, disk, table_name, max_batch_size) - count += max_batch_size - else: - process_batch(filesMonitoring, disk, table_name, userlimit) - - filesMonitoring.close() Index: trunk/ippm/FilesMonitoring/NodeRelated/update_table_undefined.py =================================================================== --- trunk/ippm/FilesMonitoring/NodeRelated/update_table_undefined.py (revision 35985) +++ (revision ) @@ -1,214 +1,0 @@ -#!/usr/bin/env python - -import MySQLdb -import sys -import socket -import os -import datetime -import hashlib -try: - from ipp.lifetime_templates import Template -except ImportError: - sys.path.append('%s/%s/share/python/' % os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - sys.stderr.write('ImportError') - for folder in sys.path: - sys.stderr.write('\t%s\n' % folder) - from ipp.lifetime_templates import Template - -def md5sum(filename): - f = open(filename, "rb") - content = f.read() - f.close() - sum = hashlib.md5() - sum.update(content) - return sum.hexdigest() - -class DbEntry: - verbose = False - def __init__(self, db_key, full_filename): - self.db_key = db_key - self.full_filename = full_filename - - def __str__(self): - if not DbEntry.verbose and len(self.db_key) > 10: - db_key = '%s...' % self.db_key[:10] - full_filename = '...%s' % self.full_filename[-10:] - else: - db_key = self.db_key - full_filename = self.full_filename - if self.role != 'deleted': - return '[%s]: %s (%s/%s/%s), %d, %s, %s' % (db_key, - full_filename, - self.stage, - self.role, - self.lifetime, - self.size, - self.md5sum, - self.last_check) - else: - return '[%s]: %s (%s)' % (db_key, - full_filename, - self.role) - - def check(self): - try: - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - stat_values = os.stat(self.full_filename) - self.size = stat_values.st_size - try: - self.md5sum = md5sum(self.full_filename) - self.status = 'ok' - except IOError: - self.status = 'error' - self.md5sum = None - except OSError: - self.status = 'deleted' - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - self.last_check = datetime.datetime.now() - - def update(self, table): - if self.status == 'deleted': - self.stage, self.role, self.lifetime = Template.classify(self.db_key) - stmt = """UPDATE %s -SET status=\'%s\', - last_check=\'%s\', - role=\'%s\', - stage=\'%s\', - lifetime=\'%s\' WHERE filename=\'%s\';""" % ( table, - self.status, - self.last_check, - self.role, - self.stage, - self.lifetime, - self.db_key ) - return stmt - stmt = """ -UPDATE %s -SET - size=%d, - md5sum=\'%s\', - role=\'%s\', - lifetime=\'%s\', - last_check=\'%s\', - stage=\'%s\', - status=\'%s\' -WHERE filename=\'%s\'; -""" % ( table, - self.size, - self.md5sum, - self.role, - self.lifetime, - self.last_check, - self.stage, - self.status, - self.db_key ) - return stmt - -def usage(): - print - print 'Usage: %s [-v] [-l | -limit ] [-disk ] ' % sys.argv[0] - print ' Update the 10 last checked entries of FilesMonitoring _ table' - print ' where is the name of the host where this command is run and' - print ' is a nebulous directory name (range: 00 - ff)' - print - print ' * -h: display help' - print ' * -v: be verbose' - print ' * -l , -limit : process no more than entries (default 10)' - print ' * -disk : process disk /export/. (default: 0)' - print - print ' e.g.: %s [-v] 00' % sys.argv[0] - print ' e.g.: %s [-v] 01' % sys.argv[0] - print ' e.g.: ..' - print ' e.g.: %s [-v] ff' % sys.argv[0] - print - -def process_options(argv): - limit = 10 - directory = None - argc = 1 - disk = "0" - while argc < len(argv): - if sys.argv[argc] == '-h': - usage() - sys.exit(0) - elif sys.argv[argc] == '-v': - DbEntry.verbose = True - argc += 1 - elif sys.argv[argc] == '-limit' or sys.argv[argc] == '-l': - limit = int(sys.argv[argc+1]) - argc += 2 - elif sys.argv[argc] == '-disk': - disk = sys.argv[argc+1] - argc += 2 - else: - directory = sys.argv[argc] - argc += 1 - if directory is None: - usage() - sys.exit(1) - return limit, directory, disk - -def process_batch(filesMonitoring, disk, table_name, limit): - cursor = filesMonitoring.cursor() - query = 'SELECT filename, subdirname FROM %s WHERE status=\'undefined\' LIMIT %d' % ( table_name, - limit) - if DbEntry.verbose: - sys.stderr.write('%s\n' % query) - cursor.execute(query) - rows = cursor.fetchall() - cursor.close() - entries = [ ] - for row in rows: - filename = row[0] - subdirname = row[1] - full_filename = '/export/%s.%s/nebulous/%s/%s/%s' % ( hostname, - disk, - directory, - subdirname, - filename) - entries.append(DbEntry(filename, full_filename)) - for entry in entries: - entry.check() - count_updates = 0 - cursor = filesMonitoring.cursor() - for entry in entries: - query = entry.update(table_name) - count_updates += 1 - cursor.execute(query) - if DbEntry.verbose: - print query - cursor.close() - filesMonitoring.commit() - sys.stderr.write('\t# of updates: %d\n' % count_updates) - return count_updates - -####################################################################################### -if __name__ == '__main__': - if len(sys.argv) == 1: - usage() - sys.exit(1) - userlimit, directory, disk = process_options(sys.argv) - hostname = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] - table_name = 'neb_%s_%s' % (disk, directory) - sys.stderr.write('Updating: %s %s %s\n' % (hostname, directory, table_name)) - if hostname == 'neverland': - filesMonitoring = MySQLdb.connect( 'localhost', - 'root', 'root', - 'DisksMonitoring' ) - else: - filesMonitoring = MySQLdb.connect( 'localhost', - 'ipp', 'ipp', - 'DisksMonitoring' ) - max_batch_size = 2500 - if userlimit > max_batch_size: - sys.stderr.write('Batching by sets of %d\n' % max_batch_size) - count = 0 - while count < userlimit: - sys.stderr.write('\tProcessing: %d out of %d\n' % (count, userlimit) ) - if process_batch(filesMonitoring, disk, table_name, max_batch_size) == 0: - count = userlimit - count += max_batch_size - else: - process_batch(filesMonitoring, disk, table_name, userlimit) - - filesMonitoring.close() Index: trunk/ippm/FilesMonitoring/Summary/snapshot.py =================================================================== --- trunk/ippm/FilesMonitoring/Summary/snapshot.py (revision 35985) +++ (revision ) @@ -1,281 +1,0 @@ -#!/usr/bin/env python - -import MySQLdb -import sys -import socket -import _mysql_exceptions -import datetime -import logging - -def process_arguments(argv): - if len(argv) == 1: - usage() - sys.exit(1) - index = 1 - nodes = [ ] - while index < len(argv): - if argv[index] == '-v': - logger.setLevel(logging.DEBUG) - index += 1 - elif argv[index] == 'mhpcc': - nodesFilename = '%s/%s/share/filemon/ippMhpccStorageNodes' % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - nodesFile = open(nodesFilename) - nodes = [] - for line in nodesFile: - nodes.append((line[:-1]+".0")) - index += 1 - elif argv[index] == 'set0': - nodes.extend([ ("ipp004.0"), ("ipp005.0"), ("ipp006.0"), ("ipp007.0"), ("ipp008.0"), ("ipp009.0"), - ("ipp060.0"), ("ipp061.0"), ("ipp062.0"), ("ipp063.0"), ("ipp064.0"), ("ipp065.0"), - ("ipp066.0"), ] ) - index += 1 - elif argv[index] == 'set1': - nodes.extend([ ("ipp010.0"), ("ipp011.0"), ("ipp012.0"), ("ipp013.0"), ("ipp014.0"), - ("ipp015.0"), ("ipp016.0"), ("ipp017.0"), ("ipp018.0"), ("ipp019.0"), ] ) - index += 1 - elif argv[index] == 'set2': - nodes.extend( [ ("ipp020.0"), ("ipp021.0"), ("ipp023.0"), ("ipp024.0"), - ("ipp025.0"), ("ipp026.0"), ("ipp027.0"), ("ipp028.0"), ("ipp029.0"), ] ) - index += 1 - elif argv[index] == 'set3': - nodes.extend([ ("ipp030.0"), ("ipp031.0"), ("ipp032.0"), ("ipp033.0"), ("ipp034.0"), - ("ipp035.0"), ("ipp036.0"), ("ipp037.0"), ("ipp038.0"), ("ipp039.0"), ] ) - index += 1 - elif argv[index] == 'set4': - nodes.extend( [ ("ipp040.0"), ("ipp041.0"), ("ipp042.0"), ("ipp043.0"), ("ipp044.0"), - ("ipp045.0"), ("ipp046.0"), ("ipp047.0"), ("ipp048.0"), ("ipp049.0"), ] ) - index += 1 - elif argv[index] == 'set5': - nodes.extend( [ ("ipp050.0"), ("ipp051.0"), ("ipp052.0"), ("ipp053.0"), ("ipp054.0"), - ("ipp055.0"), ("ipp056.0"), ("ipp057.0"), ("ipp058.0"), ("ipp059.0"), ] ) - index += 1 - elif argv[index] == 'ippb': - nodes.extend( [ ("ippb00.0"), ("ippb00.1"), ("ippb00.2"), - ("ippb01.0"), ("ippb01.1"), ("ippb01.2"), - ("ippb02.0"), ("ippb02.1"), ("ippb02.2"), - ("ippb03.0"), ("ippb03.1"), ("ippb03.2"), ] ) - index += 1 - elif argv[index] == 'stsci': - nodes.extend( [ ("stsci00.0"), ("stsci00.1"), ("stsci00.2"), - ("stsci01.0"), ("stsci01.1"), ("stsci01.2"), - ("stsci02.0"), ("stsci02.1"), ("stsci02.2"), - ("stsci03.0"), ("stsci03.1"), ("stsci03.2"), - ("stsci04.0"), ("stsci04.1"), ("stsci04.2"), - ("stsci05.0"), ("stsci05.1"), ("stsci05.2"), - ("stsci06.0"), ("stsci06.1"), ("stsci06.2"), - ("stsci07.0"), ("stsci07.1"), ("stsci07.2"), - ("stsci08.0"), ("stsci08.1"), ("stsci08.2"), - ("stsci09.0"), ("stsci09.1"), ("stsci09.2"), - ("stsci10.0"), ("stsci10.1"), ("stsci10.2"), - ("stsci11.0"), ("stsci11.1"), ("stsci11.2"), - ("stsci12.0"), ("stsci12.1"), ("stsci12.2"), - ("stsci13.0"), ("stsci13.1"), ("stsci13.2"), - ("stsci14.0"), ("stsci14.1"), ("stsci14.2"), - ("stsci15.0"), ("stsci15.1"), ("stsci15.2"), - ("stsci16.0"), ("stsci16.1"), ("stsci16.2"), - ("stsci17.0"), ("stsci17.1"), ("stsci17.2"), - ("stsci18.0"), ("stsci18.1"), ("stsci18.2"), - ("stsci19.0"), ("stsci19.1"), ("stsci19.2"), ] ) - index += 1 - else: - node = argv[index] - if node.startswith('ippb'): - nodes.append((node+".0")) - nodes.append((node+".1")) - nodes.append((node+".2")) - elif node.startswith('stsci'): - nodes.append((node+".0")) - nodes.append((node+".1")) - nodes.append((node+".2")) - else: - nodes.append((node+".0")) - index += 1 - return nodes - -def get_snapshot(disk, dirname, fm_cursor, dms_cursor): - logger.debug("Getting snapshot of [%s]" % dirname) - select_query = 'SELECT role, stage, lifetime, SUM(size), COUNT(*), status FROM neb_%s_%s GROUP BY role, stage, lifetime, status' - insert_query = """INSERT INTO CurrentDiskUsage -SET node='%s', role='%s', stage='%s', dirname='%s', sizes=%d, counts=%d, lifetime='%s', last_update=NOW(), status='%s'""" - update_query = """UPDATE CurrentDiskUsage -SET sizes=%d, counts=%d, last_update=NOW() -WHERE node='%s' AND role='%s' AND stage='%s' AND dirname='%s' AND lifetime='%s' AND status='%s'""" - undefined_found = False - fm_cursor.execute(select_query % (disk, dirname)) - for (role, stage, lifetime, size, count, status) in fm_cursor.fetchall(): - if status == 'undefined': - undefined_found = True - if size < 0: - size=0 - try: - dms_cursor.execute(insert_query % (alias, - role, - stage, - dirname, - size, - count, - lifetime, - status) ) - except Exception, e: - dms_cursor.execute(update_query % (size, - count, - alias, - role, - stage, - dirname, - lifetime, - status) ) - if not undefined_found: - logger.debug('... No undefined found for %s' % dirname) - try: - logger.debug('... Trying to delete undefined entries') - dms_cursor.execute(delete_undefined_query % (alias) ) - except Exception, e: - logger.debug('... Trying to update') - dms_cursor.execute(update_query % (0, - 0, - alias, - 'undefined', - 'undefined', - dirname, - 'undefined', - 'undefined') ) - else: - logger.warn('... There are still undefined values for %s (dirname: %s)' % (alias, dirname)) - # Now delete all entries with status set as 'deleted' after the - # deleted entries have been measures - logger.debug('... Deleting deleted entries from %s (dirname: %s)' % (alias, dirname)) - stmt = "SELECT role, stage, lifetime, SUM(size), COUNT(*) FROM neb_%s_%s_delete GROUP BY role, stage, lifetime, status" - logger.debug(stmt % (disk, dirname)) - fm_cursor.execute(stmt % (disk, dirname)) - rows = fm_cursor.fetchall() - logger.debug("Got %d deleted entries" % len(rows)) - for (role, stage, lifetime, size, count) in rows: - try: - dms_cursor.execute(insert_query % (alias, - role, - stage, - dirname, - size, - count, - lifetime, - 'deleted') ) - except Exception, e: - dms_cursor.execute(update_query % (size, - count, - alias, - role, - stage, - dirname, - lifetime, - 'deleted') ) - try: - fm_cursor.execute("DELETE FROM neb_%s_%s_delete" % (disk, dirname)) - except _mysql_exceptions.OperationalError, e: - try: - # Try again?! - fm_cursor.execute("DELETE FROM neb_%s_%s_delete" % (disk, dirname)) - except _mysql_exceptions.OperationalError, e: - logger.warn("Couldn't delete 'deleted' entries from neb_%s_%s_delete on %s" % (dirname, disk, node)) - if not undefined_found: - logger.debug('... No undefined found for %s' % dirname) - delete_undefined_query = """DELETE FROM CurrentDiskUsage -WHERE node='%s' AND (status='undefined' OR role='undefined') -""" - try: - logger.debug('... Trying deleting undefined entries') - dms_cursor.execute(delete_undefined_query % (alias) ) - except Exception, e: - logger.debug('... Trying update') - dms_cursor.execute(update_query % (0, - 0, - alias, - 'undefined', - 'undefined', - dirname, - 'undefined', - 'undefined') ) - else: - logger.warn('... There are still undefined values for %s (dirname: %s)' % (alias, dirname)) - # Now move entries for this node which are older than the current date - logger.debug('... Moving old entries from CurrentDiskUsage for %s' % (alias)) - insert_select_statement = """INSERT IGNORE INTO DiskUsageTemporary(node, role, stage, sizes, counts, lifetime, dirname, status, last_update) -SELECT node, role, stage, sizes, counts, lifetime, dirname, status, last_update FROM CurrentDiskUsage -WHERE node='%s' AND last_update<'%s'""" - dms_cursor.execute(insert_select_statement % (node, snapshot_date)); - dms_cursor.execute("DELETE FROM CurrentDiskUsage WHERE node='%s' AND last_update<'%s'" % (alias, snapshot_date)); - -def update_daily_and_weekly(dms_cursor): - try: - stmt = 'CALL UpdateDailyByNode(\'%s\')' % alias - logger.info('... Calling UpdateDailyByNode: [%s]' % stmt) - dms_cursor.execute(stmt) - except _mysql_exceptions.OperationalError, e: - logger.warn('... MySQL Exception caught when calling UpdateDailyByNode(): ' + str(e) + '') - try: - logger.info('... Trying to guess if running UpdateWeeklyByNode makes sense') - today = datetime.datetime.today() - last_week = '%d-%d' % (today.year, today.isocalendar()[1]-1) - query = "SELECT COUNT(*), node, role, stage, lifetime, dirname, week, status FROM DiskUsageHistoryWeekly WHERE node = \'%s\' AND week = '%s' GROUP BY node, role, stage, lifetime, dirname, week, status" % (alias, last_week) - dms_cursor.execute(query) - row = dms_cursor.fetchone() - run_updateweekly = (row is None) - if run_updateweekly: - stmt = 'CALL UpdateWeeklyByNode(\'%s\')' % alias - logger.info('... Executing: %s' % stmt) - dms_cursor.execute(stmt) - else: - logger.info('... UpdateWeekly() call cancelled since last week entry already in database: %s' % last_week) - except _mysql_exceptions.OperationalError, e: - logger.warn('... MySQL Exception caught when calling UpdateWeekly(): ' + str(e) ) - -if __name__ == "__main__": - # Setup logging - logger = logging.getLogger() - formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', - '%Y-%m-%dT%H:%M:%S') - logging_output = logging.StreamHandler(sys.stderr) - logging_output.setFormatter(formatter) - logger.addHandler(logging_output) - logger.setLevel(logging.INFO) - - nodes = process_arguments(sys.argv) - letters = '0123456789abcdef' - - nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] - if nodename != 'neverland' and nodename != 'ian' : - dms_db = MySQLdb.connect('ippc11', 'ippuser', 'ipp', - 'DiskMonitoringSummary') - else: # Test platform - logger.info("On [%s]" % nodename) - if nodename == 'ian': - root_passwd = '' - else: - root_passwd = 'root' - dms_db = MySQLdb.connect('localhost', 'root', root_passwd, - 'DiskMonitoringSummary') - dms_cursor = dms_db.cursor() - - for alias in nodes: - node, disk = alias.split('.') - logger.info('Getting snapshot of node: %s (%s)' % (node, alias)) - try: - if node != 'neverland' and node != 'ian' : - fm_db = MySQLdb.connect(node, 'ipp', 'ipp', 'DisksMonitoring') - else: - fm_db = MySQLdb.connect('localhost', 'root', root_passwd, - 'DisksMonitoring') - fm_cursor = fm_db.cursor() - snapshot_date = datetime.datetime.now() - for letter1 in letters: - logger.info('... Getting snapshot for nebulous subdirectories starting with %s (%s0, %s1, ...)' % (letter1, - letter1, - letter1)) - for letter2 in letters: - dirname = '%s%s' % (letter1, letter2) - get_snapshot(disk, dirname, fm_cursor, dms_cursor) - update_daily_and_weekly(dms_cursor) - fm_db.commit() - dms_db.commit() - except _mysql_exceptions.OperationalError, e: - logger.error(str(e)) Index: trunk/ippm/FilesMonitoring/Supervision/cluster_status.py =================================================================== --- trunk/ippm/FilesMonitoring/Supervision/cluster_status.py (revision 35985) +++ (revision ) @@ -1,343 +1,0 @@ -#!/usr/bin/env python - -import sys -import MySQLdb -import subprocess -import _mysql_exceptions -import re - -toGigs = 1024.**3 -toGigsFromK = 1024.**2 - -bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - -def usage(): - sys.stderr.write('Usage: cluster_status.py [-sql] \n') - sys.stderr.write('Usage: cluster_status.py [-h|-help|--help]\n') - sys.stderr.write('\n') - sys.stderr.write('\t Nebulous disk usage about \n') - sys.stderr.write('\t is the usual node name, e.g. ipp008, ..., ipp064\n') - sys.stderr.write('\t can be:\n') - sys.stderr.write('\t\t * mhpcc: all storage nodes at mhpcc\n') - sys.stderr.write('\t\t * set0: ipp004, ipp005, ipp006, ipp007, ipp008, ipp009, ipp060, ipp061, ipp062, ipp063, ipp064, ipp065, ipp066.0\n') - sys.stderr.write('\t\t * set1: ipp01x\n') - sys.stderr.write('\t\t * set2: ipp02x\n') - sys.stderr.write('\t\t * set3: ipp03x\n') - sys.stderr.write('\t\t * set4: ipp04x\n') - sys.stderr.write('\t\t * set5: ipp05x\n') - sys.stderr.write('\t\t * atrc: ippb0x (all 3 partitions for each node)\n') - sys.stderr.write('\t\t * stcsi: stcsi0x (all 3 partitions for each node)\n') - sys.stderr.write('\t\t * all: all nodes\n') - sys.stderr.write('\n') - sys.stderr.write('\t\t [-sql] Check if the mysql server is running on the nodes\n') - -def process_arguments(argv): - sql_check = False - if len(argv) == 1: - usage() - sys.exit(1) - elif argv[1] == '-h': - usage() - sys.exit(1) - elif argv[1] == '-help': - usage() - sys.exit(1) - elif argv[1] == '--help': - usage() - sys.exit(1) - elif argv[1] == '-sql': - sql_check = True - elif argv[1] == 'mhpcc': - nodesFilename = '%s/ippMhpccStorageNodes' % share_directory - nodesFile = open(nodesFilename) - nodes = [] - for line in nodesFile: - nodes.append(line[:-1] + ".0") - elif argv[1] == 'set0': - nodes = [ "ipp004.0", "ipp005.0", "ipp006.0", "ipp007.0", "ipp008.0", "ipp009.0", - "ipp060.0", "ipp061.0", "ipp062.0", "ipp063.0", "ipp064.0", "ipp065.0", - "ipp066.0", ] - elif argv[1] == 'set1': - nodes = [ "ipp010.0", "ipp011.0", "ipp012.0", "ipp013.0", "ipp014.0", - "ipp015.0", "ipp016.0", "ipp017.0", "ipp018.0", "ipp019.0", ] - elif argv[1] == 'set2': - nodes = [ "ipp020.0", "ipp021.0", "ipp023.0", "ipp024.0", - "ipp025.0", "ipp026.0", "ipp027.0", "ipp028.0", "ipp029.0", ] - elif argv[1] == 'set3': - nodes = [ "ipp030.0", "ipp031.0", "ipp032.0", "ipp033.0", "ipp034.0", - "ipp035.0", "ipp036.0", "ipp037.0", "ipp038.0", "ipp039.0", ] - elif argv[1] == 'set4': - nodes = [ "ipp040.0", "ipp041.0", "ipp042.0", "ipp043.0", "ipp044.0", - "ipp045.0", "ipp046.0", "ipp047.0", "ipp048.0", "ipp049.0", ] - elif argv[1] == 'set5': - nodes = [ "ipp050.0", "ipp051.0", "ipp052.0", "ipp053.0", "ipp054.0", - "ipp055.0", "ipp056.0", "ipp057.0", "ipp058.0", "ipp059.0", ] - elif argv[1] == 'ippb': - nodes = [ "ippb00.0", "ippb00.1", "ippb00.2", - "ippb01.0", "ippb01.1", "ippb01.2", - "ippb02.0", "ippb02.1", "ippb02.2", - "ippb03.0", "ippb03.1", "ippb03.2", ] - elif argv[1] == 'stsci': - nodes = [ "stsci00.0", "stsci00.1", "stsci00.2", - "stsci01.0", "stsci01.1", "stsci01.2", - "stsci02.0", "stsci02.1", "stsci02.2", - "stsci03.0", "stsci03.1", "stsci03.2", - "stsci04.0", "stsci04.1", "stsci04.2", - "stsci05.0", "stsci05.1", "stsci05.2", - "stsci06.0", "stsci06.1", "stsci06.2", - "stsci07.0", "stsci07.1", "stsci07.2", - "stsci08.0", "stsci08.1", "stsci08.2", - "stsci09.0", "stsci09.1", "stsci09.2", - "stsci10.0", "stsci10.1", "stsci10.2", - "stsci11.0", "stsci11.1", "stsci11.2", - "stsci12.0", "stsci12.1", "stsci12.2", - "stsci13.0", "stsci13.1", "stsci13.2", - "stsci14.0", "stsci14.1", "stsci14.2", - "stsci15.0", "stsci15.1", "stsci15.2", - "stsci16.0", "stsci16.1", "stsci16.2", - "stsci17.0", "stsci17.1", "stsci17.2", - "stsci18.0", "stsci18.1", "stsci18.2", - "stsci19.0", "stsci19.1", "stsci19.2", ] - elif argv[1] == 'all': - nodesFilename = '%s/ippMhpccStorageNodes' % share_directory - nodesFile = open(nodesFilename) - nodes = [] - for line in nodesFile: - nodes.append(line[:-1] + ".0") - nodes.extend([ "ippb00.0", "ippb00.1", "ippb00.2", - "ippb01.0", "ippb01.1", "ippb01.2", - "ippb02.0", "ippb02.1", "ippb02.2", - "ippb03.0", "ippb03.1", "ippb03.2", ]) - nodes.extend( [ "stsci00.0", "stsci00.1", "stsci00.2", - "stsci01.0", "stsci01.1", "stsci01.2", - "stsci02.0", "stsci02.1", "stsci02.2", - "stsci03.0", "stsci03.1", "stsci03.2", - "stsci04.0", "stsci04.1", "stsci04.2", - "stsci05.0", "stsci05.1", "stsci05.2", - "stsci06.0", "stsci06.1", "stsci06.2", - "stsci07.0", "stsci07.1", "stsci07.2", - "stsci08.0", "stsci08.1", "stsci08.2", - "stsci09.0", "stsci09.1", "stsci09.2", - "stsci10.0", "stsci10.1", "stsci10.2", - "stsci11.0", "stsci11.1", "stsci11.2", - "stsci12.0", "stsci12.1", "stsci12.2", - "stsci13.0", "stsci13.1", "stsci13.2", - "stsci14.0", "stsci14.1", "stsci14.2", - "stsci15.0", "stsci15.1", "stsci15.2", - "stsci16.0", "stsci16.1", "stsci16.2", - "stsci17.0", "stsci17.1", "stsci17.2", - "stsci18.0", "stsci18.1", "stsci18.2", - "stsci19.0", "stsci19.1", "stsci19.2", ] - else: - nodes = [ ] - for node in argv[1:]: - nodes.append((node + ".0")) - return (sql_check, nodes) - -def get_nebdf_output(): - nebdf_output = dict() - p = subprocess.Popen( ['neb-df', '--names', ], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - if p.wait()!=0: - logger.error('Error while running neb-df') - for line in p.stderr: - logger.error(line) - sys.exit(3) - for line in p.stdout: - if not line.startswith('Filesystem') and not line.startswith('/summmary') and not line.startswith(' '): - line = re.sub('\s+', ':', line[:-1]) - (node, available, used, free, percentage, mountpoint) = line.split(':') - nebdf_output[node] = (float(available)/toGigsFromK, float(used)/toGigsFromK, float(free)/toGigsFromK) - return nebdf_output - -def is_mysql_server_running(node): - try: - MySQLdb.connect(node, 'ippuser', 'ipp', 'DisksMonitoring') - return "mysql server: Running" - except _mysql_exceptions.OperationalError: - return "mysql server: Stopped" - -def is_update_running(node): - p = subprocess.Popen( ['%s/node_status.py' % bin_directory, node], - stderr = subprocess.PIPE, - stdout = subprocess.PIPE) - if p.wait()!=0: - return "Can't tell if node's running update/refresh (ssh error for %s)" % node - for line in p.stdout: - return " ".join(line[:-1].split(' ')[1:]) - -if __name__ == '__main__': - (sql_check, nodes) = process_arguments(sys.argv) - nebdf = get_nebdf_output() - dms_db = MySQLdb.connect('ippc11', 'ippuser', 'ipp', - 'DiskMonitoringSummary') - dms_cursor = dms_db.cursor() - aggregate_by_role = dict() - aggregate_by_stage = dict() - nodes_sizes = dict() - aggregate = dict() - raw_aggregate = dict() - aggregate_percentage = 0. - for node in nodes: - try: - query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='deleted'" % node - dms_cursor.execute(query) - total = dms_cursor.fetchall()[0][0] - #print total, query - query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node - dms_cursor.execute(query) - defined = dms_cursor.fetchall()[0][0] - #print defined, query - percentage_done = float(defined) / float(total) - # - try: - query = "SELECT SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node - dms_cursor.execute(query) - nodes_sizes[node] = float(dms_cursor.fetchall()[0][0]) / percentage_done / toGigs - except TypeError, e: - print e - node_name = node.split('.')[0] - if sql_check: - print 'Node %s: %5.1f%% done | %20s | %40s' % (node, percentage_done * 100., - is_mysql_server_running(node_name), - is_update_running(node_name)) - else: - print 'Node %s: %5.1f%% done' % (node, percentage_done * 100.) - aggregate_percentage += percentage_done - query = "SELECT role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY role" % node - dms_cursor.execute(query) - for row in dms_cursor.fetchall(): - if row[0] != 'undefined': - try: - aggregate_by_role[row[0]] += (float(row[1])/percentage_done/toGigs) - except KeyError: - aggregate_by_role[row[0]] = (float(row[1])/percentage_done/toGigs) - query = "SELECT stage, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage" % node - dms_cursor.execute(query) - for row in dms_cursor.fetchall(): - if row[0] != 'undefined': - try: - aggregate_by_stage[row[0]] += (float(row[1])/percentage_done/toGigs) - except KeyError: - aggregate_by_stage[row[0]] = (float(row[1])/percentage_done/toGigs) - query = "SELECT stage, role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage, role" % node - dms_cursor.execute(query) - for row in dms_cursor.fetchall(): - if row[0] != 'undefined': - try: - aggregate[row[0]][row[1]] += (float(row[2])/percentage_done/toGigs) - raw_aggregate[row[0]][row[1]] += float(row[2])/toGigs - except KeyError: - try: - aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs) - raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs - except KeyError: - aggregate[row[0]] = dict() - raw_aggregate[row[0]] = dict() - aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs) - raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs - except TypeError, e: - print 'Node %s: being refreshed' % (node) - print e - aggregate_percentage /= len(nodes) - - print - print "### By role (in gigs / relative % / in teras)" - total = 0. - for size in aggregate_by_role.itervalues(): - total += size - for role, size in aggregate_by_role.iteritems(): - print "%10s: %10.1f %10.1f%% %10.1f" % (role, size, size/total*100., size/1024.) - print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.) - - print - print "### By stage (in gigs / relative % / in teras)" - total = 0. - for size in aggregate_by_stage.itervalues(): - total += size - for stage, size in aggregate_by_stage.iteritems(): - print "%10s: %10.1f %10.1f%% %10.1f" % (stage, size, size/total*100., size/1024.) - print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.) - print - - if aggregate_percentage == 1.: - print - print "### Interpolated Sizes (in gigs), if all nodes were all 100% done" - header = '%10s' % '' - roles = dict() - for stage in aggregate.iterkeys(): - header = '%s%10s' % (header, stage) - for role in aggregate[stage].iterkeys(): - roles[role] = '' - print header - for role in roles.iterkeys(): - line = '%10s' % role - for stage in aggregate.iterkeys(): - try: - line = '%10s%10.1f' % (line, aggregate[stage][role]) - except KeyError: - line = '%10s%10s' % (line, '-') - print line - - print - print "### Sizes (in gigs)" - header = '%10s' % '' - roles = dict() - for stage in aggregate.iterkeys(): - header = '%s%10s' % (header, stage) - for role in aggregate[stage].iterkeys(): - roles[role] = '' - print header - for role in roles.iterkeys(): - line = '%10s' % role - for stage in aggregate.iterkeys(): - try: - line = '%10s%10.1f' % (line, raw_aggregate[stage][role]) - except KeyError: - line = '%10s%10s' % (line, '-') - print line - - print - print "### By node: Mine \t neb-df \tDifference (compared to disk size)\t\"Non-nebulous\" space" - neb_total = 0. - nebdf_total = 0. - nonneb_total = 0. - available = 0. - for node in nodes: - try: - try: - neb_total += nodes_sizes[node] - except KeyError, e: - print 'No nodes_sizes[node]' - raise e - try: - nebdf_total += nebdf[node][1] - except KeyError, e: - print 'No nodes_nebdf[node]' - raise e - nonneb_total += (nebdf[node][1]-nodes_sizes[node]) - available += nebdf[node][0] - print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( node, nodes_sizes[node], nebdf[node][1], - (nodes_sizes[node]-nebdf[node][1])/nebdf[node][0]*100., - nebdf[node][1]-nodes_sizes[node] - ) - except KeyError: - print '%10s: Incomplete data' % node - print "---------------------------------------------------" - print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total", neb_total, nebdf_total, - (neb_total-nebdf_total)/available*100., - nonneb_total ) - exclude_nodes = ['ipp004.0', 'ipp005.0', 'ipp064.0', ] - for node in exclude_nodes: - if node in nodes_sizes: - neb_total -= nodes_sizes[node] - nebdf_total -= nebdf[node][1] - nonneb_total -= (nebdf[node][1]-nodes_sizes[node]) - available -= nebdf[node][0] - print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total*", neb_total, nebdf_total, - (neb_total-nebdf_total)/available*100., - nonneb_total ) - print "Total* = Total excluding nodes:", exclude_nodes - Index: trunk/ippm/FilesMonitoring/Supervision/node_status.py =================================================================== --- trunk/ippm/FilesMonitoring/Supervision/node_status.py (revision 35985) +++ (revision ) @@ -1,122 +1,0 @@ -#!/usr/bin/env python - -import sys -import warnings -warnings.simplefilter("ignore") # Importing paramiko displays the following warning: -#/home/panstarrs/schastel/local/lib/python2.5/site-packages/pycrypto-2.5-py2.5-linux-x86_64.egg/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability. -# _warn("Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.", PowmInsecureWarning) -import paramiko -warnings.simplefilter("always") -import datetime - -bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -node_directory = '/export/%s/ipp_filemon/' - -def process_arguments(argv): - if len(argv) == 1: - usage() - sys.exit(1) - elif argv[1] == 'all': - nodesFilename = '%s/ippMhpccStorageNodes' % share_directory - nodesFile = open(nodesFilename) - nodes = [] - for line in nodesFile: - nodes.append(line[:-1] + ".0") - elif argv[1] == 'set0': - nodes = [ "ipp004.0", "ipp005.0", "ipp006.0", "ipp007.0", "ipp008.0", "ipp009.0", - "ipp060.0", "ipp061.0", "ipp062.0", "ipp063.0", "ipp064.0", "ipp065.0", "ipp066.0", ] - elif argv[1] == 'set1': - nodes = [ "ipp010.0", "ipp011.0", "ipp012.0", "ipp013.0", "ipp014.0", - "ipp015.0", "ipp016.0", "ipp017.0", "ipp018.0", "ipp019.0", ] - elif argv[1] == 'set2': - nodes = [ "ipp020.0", "ipp021.0", "ipp023.0", "ipp024.0", - "ipp025.0", "ipp026.0", "ipp027.0", "ipp028.0", "ipp029.0", ] - elif argv[1] == 'set3': - nodes = [ "ipp030.0", "ipp031.0", "ipp032.0", "ipp033.0", "ipp034.0", - "ipp035.0", "ipp036.0", "ipp037.0", "ipp038.0", "ipp039.0", ] - elif argv[1] == 'set4': - nodes = [ "ipp040.0", "ipp041.0", "ipp042.0", "ipp043.0", "ipp044.0", - "ipp045.0", "ipp046.0", "ipp047.0", "ipp048.0", "ipp049.0", ] - elif argv[1] == 'set5': - nodes = [ "ipp050.0", "ipp051.0", "ipp052.0", "ipp053.0", "ipp054.0", - "ipp055.0", "ipp056.0", "ipp057.0", "ipp058.0", "ipp059.0", ] - else: - nodes = [] - for node in argv[1:]: - nodes.append(node + ".0") - return nodes - -def get_seconds_since_last_message(ssh, node): - logfilename = ('%s/%s' % (node_directory, '/logs/%s_refresh.log') % (node, node)) - stdin, stdout, stderr = ssh.exec_command('tail -n 1 %s' % (logfilename)) - for line in stdout: - try: - last_message_date = datetime.datetime.strptime(line[:-1].split(' ')[0], '%Y-%m-%dT%H:%M:%S') - except ValueError: - return 999999 - now = datetime.datetime.now() - try: - td = now - last_message_date - except UnboundLocalError: #local variable 'last_message_date' migh tbe referenced before assignment - return 999999 - return td.seconds + td.days * 24 * 3600 - -def get_crontab_status(ssh): - stdin, stdout, stderr = ssh.exec_command('crontab -l | grep refresh_host.sh') - crontab_installed = False - next_run = None - for line in stdout: - crontab_installed = True - if 'nice' not in line: - print 'In crontab but NOT nice' - schedule = line.split(' ') - next_run = schedule[1] + ':' + schedule[0] - return (crontab_installed, next_run) - -def guess_if_update_or_refresh_is_running(ssh): - stdin, stdout, stderr = ssh.exec_command('ps waux | grep update_host_undefined.sh | grep -v grep') - update_is_running = False - for line in stdout: - update_is_running = True - stdin, stdout, stderr = ssh.exec_command('ps waux | grep refresh_table.py | grep -v grep') - refresh_is_running = False - for line in stdout: - refresh_is_running = True - if update_is_running: - if refresh_is_running: - message = "Both Update and Refresh are running" - else: - message = "Update is running" - else: - if refresh_is_running: - message = "Refresh is running" - else: - message = "Nothing is running" - return message - -if __name__ == '__main__': - nodes = process_arguments(sys.argv) - show_crontab = False - for node in nodes: - node_name = node.split('.')[0] - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(node_name) - seconds_since_last_message = get_seconds_since_last_message(ssh, node) - (is_crontab_installed, next_run) = get_crontab_status(ssh) - print '%s: %s' % (node, guess_if_update_or_refresh_is_running(ssh)) - # if seconds_since_last_message > 1000 and not is_crontab_installed: - # print 'Node %s: Not installed in crontab and last run was %d seconds ago' % (node, seconds_since_last_message) - # else: - # if is_crontab_installed: - # if seconds_since_last_message < 1000: - # msg = "OK" - # else: - # msg = "Stopped or crashed? (next run at %s)" % next_run - # else: - # msg= "Running (but crontab is not installed)" - # show_crontab = True - # print 'Node %s: %s' % (node, msg) - # if show_crontab: - # print '58 13 * * * /usr/bin/nice -n 19 /home/panstarrs/schastel/dev/FilesMonitoring/refresh_host.sh' Index: trunk/ippm/FilesMonitoring/generate_tables.py =================================================================== --- trunk/ippm/FilesMonitoring/generate_tables.py (revision 35985) +++ (revision ) @@ -1,24 +1,0 @@ -#!/usr/bin/env python - -letters = '0123456789abcdef' -share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) - -nodes = [] -nodesFile = open('%s/ippMhpccStorageNodes' % share_directory) -for line in nodesFile: - nodes.append(line[:-1]) -nodesFile.close() - -template = "" -templateFile = open('%s/sql/CreateTablesTemplate.sql' % share_directory) -for line in templateFile: - template = "%s%s" % (template, line) -templateFile.close() - -for letter1 in letters: - for letter2 in letters: - directory = '%s%s' % (letter1, letter2) - for node in nodes: - print template.replace('@HOST@', - node).replace('@DIRECTORY@', - directory) Index: trunk/ippm/FilesMonitoring/ipp/lifetime_templates.py =================================================================== --- trunk/ippm/FilesMonitoring/ipp/lifetime_templates.py (revision 35985) +++ (revision ) @@ -1,399 +1,0 @@ -#!/usr/bin/env python - -import re - -class Stage: - UNKNOWN_STAGE = 'unknown' - STAGES = dict() - def __init__(self, name, str_patterns): - self.name = name - self.str_patterns = str_patterns - self.patterns = [ ] - for pattern in str_patterns: - self.patterns.append( re.compile(pattern) ) - @staticmethod - def add_stage(stage): - Stage.STAGES[stage.name] = stage - @staticmethod - def get_stage(filename): - for key, stage in Stage.STAGES.iteritems(): - for pattern in stage.patterns: - if pattern.match(filename): - return stage.name - return Stage.UNKNOWN_STAGE - -class Role: - UNKNOWN_ROLE = 'unknown' - ROLES = dict() - def __init__(self, name, str_patterns): - self.name = name - self.str_patterns = str_patterns - self.patterns = [ ] - for pattern in str_patterns: - self.patterns.append( re.compile(pattern) ) - @staticmethod - def add_role(role): - Role.ROLES[role.name] = role - @staticmethod - def get_role(filename): - default_role = Role.UNKNOWN_ROLE - for key, role in Role.ROLES.iteritems(): - for pattern in role.patterns: - if pattern.match(filename): - if role.name != 'fits': - return role.name - else: - default_role = role.name # i.e. 'fits' but there might be a 'better' match - return default_role - -class Lifetime: - UNKNOWN_LIFETIME = 'unknown' - LIFETIMES = dict() - def __init__(self, stage, role, duration): - if isinstance(stage, Stage): - _stage = stage - else: - _stage = Stage.STAGES[stage] - if isinstance(role, Role): - _role = role - else: - _role = Role.ROLES[role] - if duration not in ['permanent', 'raw', 'ephemeral']: - raise Exception("Lifetime duration must be one of: permanent', 'raw', 'ephemeral'") - self.stage = _stage - self.role = _role - self.duration = duration - @staticmethod - def add_lifetime(lifetime): - try: - subdict = Lifetime.LIFETIMES[lifetime.stage.name] - subdict[lifetime.role.name] = lifetime - except KeyError: - Lifetime.LIFETIMES[lifetime.stage.name] = dict() - Lifetime.add_lifetime(lifetime) - -def initialize_IppTemplates(): - # Define the stages (see http://docs.python.org/library/re.html) - # that will match the different stages - Stage.add_stage(Stage( 'cam', - [ '^.*\.cm\..*$', ] )) - Stage.add_stage(Stage( 'chip', - [ '^.*\.ch\..*$', ] )) - Stage.add_stage(Stage( 'destreak', - [ '^.*destreak.*$', ] )) - Stage.add_stage(Stage( 'detrend', - [ '^.*\.det.*$', ] )) - Stage.add_stage(Stage( 'diff', - [ '^.*\.dif\..*$', ] )) - Stage.add_stage(Stage( 'dist', - [ '^.*dist.*$', ] )) - Stage.add_stage(Stage( 'magic', - [ '^.*\.mgc\..*$', ] )) - Stage.add_stage(Stage( 'raw', - [ '^.*\.ota\d\d\..*$', - '^.*\.chip01\..*$', ] )) - Stage.add_stage(Stage( 'stack', - [ '^.*\.stk\..*$', ] )) - Stage.add_stage(Stage( 'warp', - [ '^.*\.wrp\..*$', ] )) - Stage.add_stage(Stage( 'skycal', - [ '^.*\.skycal\..*$', ] )) - Stage.add_stage(Stage( 'staticsky', - [ '^.*\.sky\..*$', ] )) - - #################### Roles and associated lifetimes - Role.add_role(Role( 'b1fits', - [ '^.*\.b1\.fits$', ] )) - Lifetime.add_lifetime( Lifetime('chip', 'b1fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('detrend', 'b1fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('diff', 'b1fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('stack', 'b1fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('warp', 'b1fits', 'permanent') ) - - Role.add_role(Role( 'b1jpg', - [ '^.*\.b1\.jpeg$', - '^.*\.b1\.jpg$', ] )) - Lifetime.add_lifetime( Lifetime('chip', 'b1jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('detrend', 'b1jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('diff', 'b1jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('stack', 'b1jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('warp', 'b1jpg', 'permanent') ) - - Role.add_role(Role( 'b2fits', - [ '^.*\.b2\.fits$', ] )) - Lifetime.add_lifetime( Lifetime('chip', 'b2fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('detrend', 'b2fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('diff', 'b2fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('stack', 'b2fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime('warp', 'b2fits', 'permanent') ) - - Role.add_role(Role( 'b2jpg', - [ '^.*\.b2\.jpeg$', - '^.*\.b2\.jpg$', ] )) - Lifetime.add_lifetime( Lifetime('cam', 'b2jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('detrend', 'b2jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('diff', 'b2jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('stack', 'b2jpg', 'permanent') ) - Lifetime.add_lifetime( Lifetime('warp', 'b2jpg', 'permanent') ) - - Role.add_role(Role( 'bttable', - [ '^.*\.burn\.tbl$', ] )) - Lifetime.add_lifetime( Lifetime('raw', 'bttable', 'raw') ) - - Role.add_role(Role( 'bundle', - [ '^.*\.tgz$', ] )) - Lifetime.add_lifetime( Lifetime('cam', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('chip', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('diff', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('dist', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('raw', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('stack', 'bundle', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime('warp', 'bundle', 'ephemeral') ) - - Role.add_role(Role( 'catalog', - [ '^.*\.smf$', - '^.*\.cmf$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'catalog', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'catalog', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'catalog', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'catalog', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'catalog', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'catalog', 'permanent') ) - - Role.add_role(Role( 'compfits', - [ '^.*\.fits\.fz$', ] )) - Lifetime.add_lifetime( Lifetime( 'raw', 'compfits', 'raw') ) - - Role.add_role(Role( 'kernel', - [ '^.*kernel$', ] )) - Lifetime.add_lifetime( Lifetime( 'diff', 'kernel', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'kernel', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'kernel', 'permanent') ) - - Role.add_role(Role( 'log', - [ '^.*\.log$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'detrend', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'dist', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'raw', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'log', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'log', 'permanent') ) - - Role.add_role(Role( 'logupdate', - [ '^.*\.log\.update$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'detrend', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'dist', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'raw', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'logupdate', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'logupdate', 'permanent') ) - - Role.add_role(Role( 'mask', - [ '^.*\.mk\.fits$', - '^.*\.mask\.fits$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'mask', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'mask', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'mask', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'mask', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'mask', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'mask', 'ephemeral') ) - - Role.add_role(Role( 'mdc', - [ '^.*\.mdc$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'dist', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'mdc', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'mdc', 'permanent') ) - - Role.add_role(Role( 'mdl', - [ '^.*\.mdl\.fits$', ] )) - Lifetime.add_lifetime( Lifetime( 'chip', 'mdl', 'ephemeral') ) - - Role.add_role(Role( 'miscfits', - [ '^.*\.exp\.fits$', - '^.*\.num\.fits$', - '^.*\.expwt\.fits$', ] )) - Lifetime.add_lifetime( Lifetime( 'stack', 'miscfits', 'permanent') ) - - Role.add_role(Role( 'png', - [ '^.*\.png$', ] )) - Lifetime.add_lifetime( Lifetime( 'chip', 'png', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'magic', 'png', 'ephemeral') ) - - Role.add_role(Role( 'psf', - [ '^.*\.psf$', ] )) - Lifetime.add_lifetime( Lifetime( 'chip', 'psf', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'psf', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'psf', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'psf', 'ephemeral') ) - - Role.add_role(Role( 'ptn', - [ '^.*\.ptn$', ] )) - Lifetime.add_lifetime( Lifetime( 'chip', 'ptn', 'ephemeral') ) - - Role.add_role(Role( 'skycell', - [ '^.*\.skycell$', ] )) - Lifetime.add_lifetime( Lifetime( 'warp', 'skycell', 'ephemeral') ) - - Role.add_role(Role( 'stats', - [ '^.*\.stats$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'detrend', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'magic', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'raw', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'stats', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'stats', 'permanent') ) - - Role.add_role(Role( 'trace', - [ '^.*\.trace$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'detrend', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'raw', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'trace', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'trace', 'permanent') ) - - Role.add_role(Role( 'weight', - [ '^.*\.wt\.fits$', ] )) - Lifetime.add_lifetime( Lifetime( 'chip', 'weight', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'weight', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'weight', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'weight', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'weight', 'ephemeral') ) - - Role.add_role(Role( 'fits', - [ '^.*\.fits$', ] )) - Lifetime.add_lifetime( Lifetime( 'cam', 'fits', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'chip', 'fits', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'destreak', 'fits', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'detrend', 'fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'diff', 'fits', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'magic', 'fits', 'ephemeral') ) - Lifetime.add_lifetime( Lifetime( 'raw', 'fits', 'raw') ) - Lifetime.add_lifetime( Lifetime( 'stack', 'fits', 'permanent') ) - Lifetime.add_lifetime( Lifetime( 'warp', 'fits', 'ephemeral') ) - -###################################################################################### -# -# DON'T TOUCH ANYTHING AFTER THIS LINE UNLESS YOU KNOW WHAT YOU'RE DOING -# -###################################################################################### -class Template: - """ - >>> print Template.classify('1110322.gpc1:20080610:o4627g0006o.17394:o4627g0006o.17394.reg.ota03.log') - ('raw', 'log', 'permanent') - >>> print Template.classify('165757062.gpc1:SweetSpot.nt:2010:02:22:o5249g0554o.139304:SR_o5249g0554o.139304.cm.49578.XY03.mk.fits') - ('cam', 'mask', 'permanent') - >>> print Template.classify('165578038.gpc1:ThreePi.nt:2010:02:23:o5250g0408o.139800:o5250g0408o.139800.ch.62962.XY54.stats') - ('chip', 'stats', 'permanent') - >>> print Template.classify('173285968.gpc1:IPP-MOPS:IPP-MOPS.15546.neg.mops') - ('unknown', 'unknown', 'unknown') - >>> print Template.classify('191293758.gpc1:20100308:o5263g0051f.145823:o5263g0051f.145823.reg.log') - ('unknown', 'log', 'unknown') - >>> print Template.classify('216601960.gpc1:ThreePi.nt:2010:02:22:o5249g0213o.138965:o5249g0213o.138965.wrp.34664.skycell.2038.080.trace.update') - ('warp', 'unknown', 'unknown') - >>> print Template.classify('500266344.gpc1:20101010:o5479g0217o.237556:o5479g0217o.237556.mgc.87689.original.png') - ('magic', 'png', 'ephemeral') - """ - initialize_IppTemplates() - - def __init__(self): - pass - - @staticmethod - def classify(filename): - # We have subclasses of FITS files: try to detect that first - stage = Stage.get_stage(filename) - role = Role.get_role(filename) - try: - return (stage, role, Lifetime.LIFETIMES[stage][role].duration) - except KeyError: - return (stage, role, Lifetime.UNKNOWN_LIFETIME) - - -def usage(progname): - return """ -: %s [test|wiki|] - - Provides the IPP files classification, that is (stage, role, lifetime duration) where: - * stage is the stage name when the file has been generated (chip, cam...) - * role is the role of the file (fits, log, mask...) - * its lifetime duration (raw, permanent, ephemeral) - Any undefined stage, role, or lifetime duration will be displayed as unknown. - - The complete list of stages is: - %s - - The complete list of roles is: - %s - -Arguments: - - test: Run the unit tests - - wiki: Generate the table for the "Cluster Storage Notes" IPP wikipage - See http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Cluster_Storage_Notes - - : Try to classify the , e.g.: - %s 218164796.gpc1:20100327:o5282g0065o.151926:o5282g0065o.151926.reg.log - should display: - ('unknown', 'log', 'unknown') - -""" % (progname, Stage.STAGES.keys(), Role.ROLES.keys(), progname) - -def generate_wiki_documentation(date, command_line, cwd): - # We start with an empty cell and display the stage names - wiki = """This table was generated on %s by the command {{{%s}}} from {{{%s}}}. Do not update it manually. - -Raw (R), Ephemeral (E), Should not be made (-), and Permanent Products (P) Files. -""" % (date, command_line, cwd) - wiki = "%s\n%s" % (wiki, "|| || Stage ") - stage_patterns_line = "|| Role || Pattern " - for stage in sorted(Stage.STAGES.keys()): - wiki = "%s || %10s" % (wiki, stage) - wiki_patterns = ('%s' % (Stage.STAGES[stage].str_patterns)) - stage_patterns_line = "%s || {{{%10s}}}" % (stage_patterns_line, wiki_patterns) - wiki = "%s||\n%s||\n" % (wiki, stage_patterns_line) - # Each line contains: the role name, then the role patterns, and then, - # for each stage, the lifetime duration - for role in sorted(Role.ROLES.keys()): - wiki = "%s|| %10s" % (wiki, role) - wiki = "%s|| {{{%10s}}}" % (wiki, Role.ROLES[role].str_patterns) - for stage in sorted(Stage.STAGES.keys()): - try: - wiki = "%s || %s" % (wiki, Lifetime.LIFETIMES[stage][role].duration[0].upper()) - except KeyError: - wiki = "%s || -" % (wiki) - wiki = "%s ||\n" % wiki - print wiki - -if __name__ == '__main__': - import sys - if len(sys.argv) != 2: - sys.stderr.write(usage(sys.argv[0])) - sys.exit(1) - if sys.argv[1] == 'test': - import doctest - doctest.testmod() - elif sys.argv[1] == 'wiki': - import datetime - import os - generate_wiki_documentation(str(datetime.datetime.now()), - " ".join(sys.argv), - os.getcwd()) - else: - print Template.classify(sys.argv[1]) Index: trunk/ippm/FilesMonitoring/populate_table.py =================================================================== --- trunk/ippm/FilesMonitoring/populate_table.py (revision 35985) +++ (revision ) @@ -1,58 +1,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -import os - -bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) -node_directory = '/export/%s.0/ipp_filemon/' - -def populate(node, directory): - root_dir = "%s/populate" % node_directory - try: - os.makedirs(('%s/sql' % root_dir) % node) - except OSError: - pass - output = open(('%s/%s' % (root_dir, 'sql/populate_%s_%s.sql')) % (node, node, directory), - 'w') - ignored = open(('%s/%s' % (root_dir, 'log/ignored_%s_%s')) % (node, node, directory), - 'w') - stdout = open(('%s/%s' % (root_dir, 'log/stdout_%s_%s')) % (node, node, directory), - 'w') - index_filename = ('%s/%s' % (node_directory,'/mlocate/%s.%s.mlocate_db')) % (node, node, directory) - p = subprocess.Popen(('%s/locate -d %s .' - % bin_directory,index_filename).split(' '), - stdout = subprocess.PIPE) - p.poll() - to_remove = '/export/%s.0/nebulous/%s' % (node, directory) - count = 1 - for line in p.stdout: - if count % 10000 == 0: - stdout.write('%s %s %s\n' % (node, directory, count)) - line = line[:-1].replace(to_remove, '') - fields = line.split('/') - if len(fields) == 3: - count += 1 - subdirectory = fields[1] - filename = fields[2] - elements = filename.split('.') - neb_key = '.'.join(elements[1:]).replace(':', '/') - output.write(insertTemplate % (node, directory, filename, elements[0], subdirectory, neb_key)) - else: - ignored.write('Ignored: %s %s %s\n' % (node, directory, line)) - output.close() - -letters = '0123456789abcdef' -insertTemplate="" -sqlFile = open('%s/sql/insert_template.sql' % share_directory) -for line in sqlFile: - insertTemplate = "%s%s" % (insertTemplate, line) -sqlFile.close() - -node = sys.argv[1] -for letter1 in letters: - for letter2 in letters: - directory = '%s%s' % (letter1, letter2) - populate(node, directory) - Index: trunk/ippm/FilesMonitoring/share/lifetime_templates.py =================================================================== --- trunk/ippm/FilesMonitoring/share/lifetime_templates.py (revision 35986) +++ trunk/ippm/FilesMonitoring/share/lifetime_templates.py (revision 35986) @@ -0,0 +1,399 @@ +#!/usr/bin/env python + +import re + +class Stage: + UNKNOWN_STAGE = 'unknown' + STAGES = dict() + def __init__(self, name, str_patterns): + self.name = name + self.str_patterns = str_patterns + self.patterns = [ ] + for pattern in str_patterns: + self.patterns.append( re.compile(pattern) ) + @staticmethod + def add_stage(stage): + Stage.STAGES[stage.name] = stage + @staticmethod + def get_stage(filename): + for key, stage in Stage.STAGES.iteritems(): + for pattern in stage.patterns: + if pattern.match(filename): + return stage.name + return Stage.UNKNOWN_STAGE + +class Role: + UNKNOWN_ROLE = 'unknown' + ROLES = dict() + def __init__(self, name, str_patterns): + self.name = name + self.str_patterns = str_patterns + self.patterns = [ ] + for pattern in str_patterns: + self.patterns.append( re.compile(pattern) ) + @staticmethod + def add_role(role): + Role.ROLES[role.name] = role + @staticmethod + def get_role(filename): + default_role = Role.UNKNOWN_ROLE + for key, role in Role.ROLES.iteritems(): + for pattern in role.patterns: + if pattern.match(filename): + if role.name != 'fits': + return role.name + else: + default_role = role.name # i.e. 'fits' but there might be a 'better' match + return default_role + +class Lifetime: + UNKNOWN_LIFETIME = 'unknown' + LIFETIMES = dict() + def __init__(self, stage, role, duration): + if isinstance(stage, Stage): + _stage = stage + else: + _stage = Stage.STAGES[stage] + if isinstance(role, Role): + _role = role + else: + _role = Role.ROLES[role] + if duration not in ['permanent', 'raw', 'ephemeral']: + raise Exception("Lifetime duration must be one of: permanent', 'raw', 'ephemeral'") + self.stage = _stage + self.role = _role + self.duration = duration + @staticmethod + def add_lifetime(lifetime): + try: + subdict = Lifetime.LIFETIMES[lifetime.stage.name] + subdict[lifetime.role.name] = lifetime + except KeyError: + Lifetime.LIFETIMES[lifetime.stage.name] = dict() + Lifetime.add_lifetime(lifetime) + +def initialize_IppTemplates(): + # Define the stages (see http://docs.python.org/library/re.html) + # that will match the different stages + Stage.add_stage(Stage( 'cam', + [ '^.*\.cm\..*$', ] )) + Stage.add_stage(Stage( 'chip', + [ '^.*\.ch\..*$', ] )) + Stage.add_stage(Stage( 'destreak', + [ '^.*destreak.*$', ] )) + Stage.add_stage(Stage( 'detrend', + [ '^.*\.det.*$', ] )) + Stage.add_stage(Stage( 'diff', + [ '^.*\.dif\..*$', ] )) + Stage.add_stage(Stage( 'dist', + [ '^.*dist.*$', ] )) + Stage.add_stage(Stage( 'magic', + [ '^.*\.mgc\..*$', ] )) + Stage.add_stage(Stage( 'raw', + [ '^.*\.ota\d\d\..*$', + '^.*\.chip01\..*$', ] )) + Stage.add_stage(Stage( 'stack', + [ '^.*\.stk\..*$', ] )) + Stage.add_stage(Stage( 'warp', + [ '^.*\.wrp\..*$', ] )) + Stage.add_stage(Stage( 'skycal', + [ '^.*\.skycal\..*$', ] )) + Stage.add_stage(Stage( 'staticsky', + [ '^.*\.sky\..*$', ] )) + + #################### Roles and associated lifetimes + Role.add_role(Role( 'b1fits', + [ '^.*\.b1\.fits$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b1fits', 'permanent') ) + + Role.add_role(Role( 'b1jpg', + [ '^.*\.b1\.jpeg$', + '^.*\.b1\.jpg$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b1jpg', 'permanent') ) + + Role.add_role(Role( 'b2fits', + [ '^.*\.b2\.fits$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b2fits', 'permanent') ) + + Role.add_role(Role( 'b2jpg', + [ '^.*\.b2\.jpeg$', + '^.*\.b2\.jpg$', ] )) + Lifetime.add_lifetime( Lifetime('cam', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b2jpg', 'permanent') ) + + Role.add_role(Role( 'bttable', + [ '^.*\.burn\.tbl$', ] )) + Lifetime.add_lifetime( Lifetime('raw', 'bttable', 'raw') ) + + Role.add_role(Role( 'bundle', + [ '^.*\.tgz$', ] )) + Lifetime.add_lifetime( Lifetime('cam', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('chip', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('diff', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('dist', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('raw', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('stack', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('warp', 'bundle', 'ephemeral') ) + + Role.add_role(Role( 'catalog', + [ '^.*\.smf$', + '^.*\.cmf$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'catalog', 'permanent') ) + + Role.add_role(Role( 'compfits', + [ '^.*\.fits\.fz$', ] )) + Lifetime.add_lifetime( Lifetime( 'raw', 'compfits', 'raw') ) + + Role.add_role(Role( 'kernel', + [ '^.*kernel$', ] )) + Lifetime.add_lifetime( Lifetime( 'diff', 'kernel', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'kernel', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'kernel', 'permanent') ) + + Role.add_role(Role( 'log', + [ '^.*\.log$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'log', 'permanent') ) + + Role.add_role(Role( 'logupdate', + [ '^.*\.log\.update$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'logupdate', 'permanent') ) + + Role.add_role(Role( 'mask', + [ '^.*\.mk\.fits$', + '^.*\.mask\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'mask', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'mask', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'mask', 'ephemeral') ) + + Role.add_role(Role( 'mdc', + [ '^.*\.mdc$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'mdc', 'permanent') ) + + Role.add_role(Role( 'mdl', + [ '^.*\.mdl\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'mdl', 'ephemeral') ) + + Role.add_role(Role( 'miscfits', + [ '^.*\.exp\.fits$', + '^.*\.num\.fits$', + '^.*\.expwt\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'stack', 'miscfits', 'permanent') ) + + Role.add_role(Role( 'png', + [ '^.*\.png$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'png', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'png', 'ephemeral') ) + + Role.add_role(Role( 'psf', + [ '^.*\.psf$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'psf', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'psf', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'psf', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'psf', 'ephemeral') ) + + Role.add_role(Role( 'ptn', + [ '^.*\.ptn$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'ptn', 'ephemeral') ) + + Role.add_role(Role( 'skycell', + [ '^.*\.skycell$', ] )) + Lifetime.add_lifetime( Lifetime( 'warp', 'skycell', 'ephemeral') ) + + Role.add_role(Role( 'stats', + [ '^.*\.stats$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'stats', 'permanent') ) + + Role.add_role(Role( 'trace', + [ '^.*\.trace$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'trace', 'permanent') ) + + Role.add_role(Role( 'weight', + [ '^.*\.wt\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'weight', 'ephemeral') ) + + Role.add_role(Role( 'fits', + [ '^.*\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'fits', 'raw') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'fits', 'ephemeral') ) + +###################################################################################### +# +# DON'T TOUCH ANYTHING AFTER THIS LINE UNLESS YOU KNOW WHAT YOU'RE DOING +# +###################################################################################### +class Template: + """ + >>> print Template.classify('1110322.gpc1:20080610:o4627g0006o.17394:o4627g0006o.17394.reg.ota03.log') + ('raw', 'log', 'permanent') + >>> print Template.classify('165757062.gpc1:SweetSpot.nt:2010:02:22:o5249g0554o.139304:SR_o5249g0554o.139304.cm.49578.XY03.mk.fits') + ('cam', 'mask', 'permanent') + >>> print Template.classify('165578038.gpc1:ThreePi.nt:2010:02:23:o5250g0408o.139800:o5250g0408o.139800.ch.62962.XY54.stats') + ('chip', 'stats', 'permanent') + >>> print Template.classify('173285968.gpc1:IPP-MOPS:IPP-MOPS.15546.neg.mops') + ('unknown', 'unknown', 'unknown') + >>> print Template.classify('191293758.gpc1:20100308:o5263g0051f.145823:o5263g0051f.145823.reg.log') + ('unknown', 'log', 'unknown') + >>> print Template.classify('216601960.gpc1:ThreePi.nt:2010:02:22:o5249g0213o.138965:o5249g0213o.138965.wrp.34664.skycell.2038.080.trace.update') + ('warp', 'unknown', 'unknown') + >>> print Template.classify('500266344.gpc1:20101010:o5479g0217o.237556:o5479g0217o.237556.mgc.87689.original.png') + ('magic', 'png', 'ephemeral') + """ + initialize_IppTemplates() + + def __init__(self): + pass + + @staticmethod + def classify(filename): + # We have subclasses of FITS files: try to detect that first + stage = Stage.get_stage(filename) + role = Role.get_role(filename) + try: + return (stage, role, Lifetime.LIFETIMES[stage][role].duration) + except KeyError: + return (stage, role, Lifetime.UNKNOWN_LIFETIME) + + +def usage(progname): + return """ +: %s [test|wiki|] + + Provides the IPP files classification, that is (stage, role, lifetime duration) where: + * stage is the stage name when the file has been generated (chip, cam...) + * role is the role of the file (fits, log, mask...) + * its lifetime duration (raw, permanent, ephemeral) + Any undefined stage, role, or lifetime duration will be displayed as unknown. + + The complete list of stages is: + %s + + The complete list of roles is: + %s + +Arguments: + + test: Run the unit tests + + wiki: Generate the table for the "Cluster Storage Notes" IPP wikipage + See http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Cluster_Storage_Notes + + : Try to classify the , e.g.: + %s 218164796.gpc1:20100327:o5282g0065o.151926:o5282g0065o.151926.reg.log + should display: + ('unknown', 'log', 'unknown') + +""" % (progname, Stage.STAGES.keys(), Role.ROLES.keys(), progname) + +def generate_wiki_documentation(date, command_line, cwd): + # We start with an empty cell and display the stage names + wiki = """This table was generated on %s by the command {{{%s}}} from {{{%s}}}. Do not update it manually. + +Raw (R), Ephemeral (E), Should not be made (-), and Permanent Products (P) Files. +""" % (date, command_line, cwd) + wiki = "%s\n%s" % (wiki, "|| || Stage ") + stage_patterns_line = "|| Role || Pattern " + for stage in sorted(Stage.STAGES.keys()): + wiki = "%s || %10s" % (wiki, stage) + wiki_patterns = ('%s' % (Stage.STAGES[stage].str_patterns)) + stage_patterns_line = "%s || {{{%10s}}}" % (stage_patterns_line, wiki_patterns) + wiki = "%s||\n%s||\n" % (wiki, stage_patterns_line) + # Each line contains: the role name, then the role patterns, and then, + # for each stage, the lifetime duration + for role in sorted(Role.ROLES.keys()): + wiki = "%s|| %10s" % (wiki, role) + wiki = "%s|| {{{%10s}}}" % (wiki, Role.ROLES[role].str_patterns) + for stage in sorted(Stage.STAGES.keys()): + try: + wiki = "%s || %s" % (wiki, Lifetime.LIFETIMES[stage][role].duration[0].upper()) + except KeyError: + wiki = "%s || -" % (wiki) + wiki = "%s ||\n" % wiki + print wiki + +if __name__ == '__main__': + import sys + if len(sys.argv) != 2: + sys.stderr.write(usage(sys.argv[0])) + sys.exit(1) + if sys.argv[1] == 'test': + import doctest + doctest.testmod() + elif sys.argv[1] == 'wiki': + import datetime + import os + generate_wiki_documentation(str(datetime.datetime.now()), + " ".join(sys.argv), + os.getcwd()) + else: + print Template.classify(sys.argv[1]) Index: trunk/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql =================================================================== --- trunk/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql (revision 35986) +++ trunk/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql (revision 35986) @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS Files_@HOST@_@DIRECTORY@; + +CREATE TABLE Files_@HOST@_@DIRECTORY@ ( + filename CHAR(255) PRIMARY KEY, + neb_ins_id BIGINT(20) NOT NULL, + subdirname CHAR(3) NOT NULL, + INDEX(subdirname), + nebkey CHAR(255), + INDEX(nebkey), + size INT DEFAULT -1, + md5sum CHAR(33) DEFAULT NULL, + role ENUM('undefined', 'unknown', 'deleted', 'b1fits', 'b1jpg', 'b2fits', + 'b2jpg', 'bttable', 'bundle', 'catalog', + 'fits', 'compfits', 'kernel', 'log', 'logupdate', 'mask', 'mdc', 'mdl', + 'miscfits', 'png', 'psf', 'ptn', 'skycell', 'stats', + 'trace', 'weight', 'other') + NOT NULL DEFAULT 'undefined', + INDEX(role), + stage ENUM( 'undefined', 'unknown', 'raw', 'chip', 'cam', 'fake', 'detrend', + 'warp', 'stack', 'diff', 'magic', 'destreak', + 'dist', 'pub', 'staticsky', 'chipbackground', + 'other' ) + NOT NULL DEFAULT 'undefined', + INDEX(stage), + lifetime ENUM('undefined', 'permanent', 'ephemeral', 'raw', 'unknown') NOT NULL DEFAULT 'undefined', + creation_date TIMESTAMP DEFAULT 0, + last_check TIMESTAMP DEFAULT 0 +); Index: trunk/ippm/FilesMonitoring/share/sql/insert_template.sql =================================================================== --- trunk/ippm/FilesMonitoring/share/sql/insert_template.sql (revision 35986) +++ trunk/ippm/FilesMonitoring/share/sql/insert_template.sql (revision 35986) @@ -0,0 +1,7 @@ +INSERT INTO Files_%s_%s ( + filename, + neb_ins_id, + subdirname, + nebkey, + creation_date) +VALUES ('%s', %s, '%s', '%s', NOW()); Index: trunk/ippm/FilesMonitoring/sql/CreateTablesTemplate.sql =================================================================== --- trunk/ippm/FilesMonitoring/sql/CreateTablesTemplate.sql (revision 35985) +++ (revision ) @@ -1,28 +1,0 @@ -DROP TABLE IF EXISTS Files_@HOST@_@DIRECTORY@; - -CREATE TABLE Files_@HOST@_@DIRECTORY@ ( - filename CHAR(255) PRIMARY KEY, - neb_ins_id BIGINT(20) NOT NULL, - subdirname CHAR(3) NOT NULL, - INDEX(subdirname), - nebkey CHAR(255), - INDEX(nebkey), - size INT DEFAULT -1, - md5sum CHAR(33) DEFAULT NULL, - role ENUM('undefined', 'unknown', 'deleted', 'b1fits', 'b1jpg', 'b2fits', - 'b2jpg', 'bttable', 'bundle', 'catalog', - 'fits', 'compfits', 'kernel', 'log', 'logupdate', 'mask', 'mdc', 'mdl', - 'miscfits', 'png', 'psf', 'ptn', 'skycell', 'stats', - 'trace', 'weight', 'other') - NOT NULL DEFAULT 'undefined', - INDEX(role), - stage ENUM( 'undefined', 'unknown', 'raw', 'chip', 'cam', 'fake', 'detrend', - 'warp', 'stack', 'diff', 'magic', 'destreak', - 'dist', 'pub', 'staticsky', 'chipbackground', - 'other' ) - NOT NULL DEFAULT 'undefined', - INDEX(stage), - lifetime ENUM('undefined', 'permanent', 'ephemeral', 'raw', 'unknown') NOT NULL DEFAULT 'undefined', - creation_date TIMESTAMP DEFAULT 0, - last_check TIMESTAMP DEFAULT 0 -); Index: trunk/ippm/FilesMonitoring/sql/insert_template.sql =================================================================== --- trunk/ippm/FilesMonitoring/sql/insert_template.sql (revision 35985) +++ (revision ) @@ -1,7 +1,0 @@ -INSERT INTO Files_%s_%s ( - filename, - neb_ins_id, - subdirname, - nebkey, - creation_date) -VALUES ('%s', %s, '%s', '%s', NOW()); Index: trunk/ippm/FilesMonitoring/tools/deneb-locate.py =================================================================== --- trunk/ippm/FilesMonitoring/tools/deneb-locate.py (revision 35985) +++ (revision ) @@ -1,184 +1,0 @@ -#!/usr/bin/env python - -import threading -import MySQLdb -import sys -import _mysql_exceptions - -def usage(progname): - print """ -Usage: %s [-domain ] [-size] - - Get the files on the cluster whose nebkey matches the - - Options: - -domain : with set to mhpcc (wrt atrc, wrt stsci), all (default: all) - Restrict the search to MHPCC (wrt ATRC, wrt STSCI) storages nodes - 'all': search on all nodes (default) - -size: display the size of the file - - Examples: - - Search a file on all nodes - %s gpc1/20100228/o5255g0304o/o5255g0304o.ota71.burn.tbl - - Search with SQL wildcard (can be really long) - %s gpc1/20100228/%%/o5255g0308o.ota11.burn.tbl (SQL wildcard) - - Search only on STSCI nodes - %s -domain stsci gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl - - Search only on ATRC nodes - %s -domain atrc gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl - - Search only on MHPCC nodes - %s -domain mhpcc gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl - - Search a file on all nodes and show their sizes - %s -size gpc1/20100228/o5255g0304o/o5255g0304o.ota71.burn.tbl - - Notes: - * Error messages as (2003, "Can't connect to MySQL server on '' (111)") - can be displayed if for some reason one of the MySQL servers (e.g. the one on ) - has crashed. - * Accuracy of the result is not asserted since the nebulous directory contents are - refreshed only once a day. - * If the file size is -1, it means that the file has been inserted very recently and not been been "refreshed" yet - """ % (progname, progname, progname, progname, progname, progname, progname) - -class Nodes: - mhpcc = [ ("ipp004", "0"), - ("ipp005", "0"), ("ipp006", "0"), ("ipp007", "0"), ("ipp008", "0"), ("ipp009", "0"), - ("ipp010", "0"), ("ipp011", "0"), ("ipp012", "0"), ("ipp013", "0"), ("ipp014", "0"), - ("ipp015", "0"), ("ipp016", "0"), ("ipp017", "0"), ("ipp018", "0"), ("ipp019", "0"), - ("ipp020", "0"), ("ipp021", "0"), ("ipp023", "0"), ("ipp024", "0"), - ("ipp025", "0"), ("ipp026", "0"), ("ipp027", "0"), ("ipp028", "0"), ("ipp029", "0"), - ("ipp030", "0"), ("ipp031", "0"), ("ipp032", "0"), ("ipp033", "0"), ("ipp034", "0"), - ("ipp035", "0"), ("ipp036", "0"), ("ipp037", "0"), ("ipp038", "0"), ("ipp039", "0"), - ("ipp040", "0"), ("ipp041", "0"), ("ipp042", "0"), ("ipp043", "0"), ("ipp044", "0"), - ("ipp045", "0"), ("ipp046", "0"), ("ipp047", "0"), ("ipp048", "0"), ("ipp049", "0"), - ("ipp050", "0"), ("ipp051", "0"), ("ipp052", "0"), ("ipp053", "0"), ("ipp054", "0"), - ("ipp055", "0"), ("ipp056", "0"), ("ipp057", "0"), ("ipp058", "0"), ("ipp059", "0"), - ("ipp060", "0"), ("ipp061", "0"), ("ipp062", "0"), ("ipp063", "0"), ("ipp064", "0"), - ("ipp065", "0"), ("ipp066", "0"), - ] - atrc = [ ("ippb00", "0"), ("ippb00", "1"), ("ippb00", "2"), - ("ippb01", "0"), ("ippb01", "1"), ("ippb01", "2"), - ("ippb02", "0"), ("ippb02", "1"), ("ippb02", "2"), - ("ippb03", "0"), ("ippb03", "1"), ("ippb03", "2"), - ] - stsci = [ ("stsci00", "0"), ("stsci00", "1"), ("stsci00", "2"), - ("stsci01", "0"), ("stsci01", "1"), ("stsci01", "2"), - ("stsci02", "0"), ("stsci02", "1"), ("stsci02", "2"), - ("stsci03", "0"), ("stsci03", "1"), ("stsci03", "2"), - ("stsci04", "0"), ("stsci04", "1"), ("stsci04", "2"), - ("stsci05", "0"), ("stsci05", "1"), ("stsci05", "2"), - ("stsci06", "0"), ("stsci06", "1"), ("stsci06", "2"), - ("stsci07", "0"), ("stsci07", "1"), ("stsci07", "2"), - ("stsci08", "0"), ("stsci08", "1"), ("stsci08", "2"), - ("stsci09", "0"), ("stsci09", "1"), ("stsci09", "2"), - ("stsci10", "0"), ("stsci10", "1"), ("stsci10", "2"), - ("stsci11", "0"), ("stsci11", "1"), ("stsci11", "2"), - ("stsci12", "0"), ("stsci12", "1"), ("stsci12", "2"), - ("stsci13", "0"), ("stsci13", "1"), ("stsci13", "2"), - ("stsci14", "0"), ("stsci14", "1"), ("stsci14", "2"), - ("stsci15", "0"), ("stsci15", "1"), ("stsci15", "2"), - ("stsci16", "0"), ("stsci16", "1"), ("stsci16", "2"), - ("stsci17", "0"), ("stsci17", "1"), ("stsci17", "2"), - ("stsci18", "0"), ("stsci18", "1"), ("stsci18", "2"), - ("stsci19", "0"), ("stsci19", "1"), ("stsci19", "2"), - ] - -class QueryingThread(threading.Thread): - count_query_exact = "SELECT COUNT(*) FROM %s WHERE nebkey = '%s'" - count_query_approximate = "SELECT COUNT(*) FROM %s WHERE nebkey LIKE '%s'" - query_exact = "SELECT subdirname, filename FROM %s WHERE nebkey = '%s'" - query_approximate = "SELECT subdirname, filename FROM %s WHERE nebkey LIKE '%s'" - query_exact_with_size = "SELECT subdirname, filename, size FROM %s WHERE nebkey = '%s'" - query_approximate_with_size = "SELECT subdirname, filename, size FROM %s WHERE nebkey LIKE '%s'" - letters = "0123456789abcdef" - - def __init__(self, node, disk, pattern, show_size): - threading.Thread.__init__(self) - self.node = node - self.disk = disk - self.show_size = show_size - self.db = MySQLdb.connect(host=node, user='ippuser', passwd='ipp', db='DisksMonitoring') - self.cursor = self.db.cursor() - if '%' in pattern: - if self.show_size: - self.count_query = QueryingThread.count_query_approximate - self.query = QueryingThread.query_approximate_with_size - else: - self.count_query = QueryingThread.count_query_approximate - self.query = QueryingThread.query_approximate - else: - if self.show_size: - self.count_query = QueryingThread.count_query_exact - self.query = QueryingThread.query_exact_with_size - else: - self.count_query = QueryingThread.count_query_exact - self.query = QueryingThread.query_exact - - def run(self): - try: - for letter1 in QueryingThread.letters: - for letter2 in QueryingThread.letters: - tablename = 'neb_%s_%s%s' % (self.disk, letter1, letter2) - self.cursor.execute(self.count_query % (tablename, pattern)) - rows = self.cursor.fetchall() - count = rows[0][0] - if count != 0: - self.cursor.execute(self.query % (tablename, pattern)) - for row in self.cursor.fetchall(): - if self.show_size: - print '%9d | /data/%s.%s/nebulous/%s%s/%s/%s' % (row[2], self.node, self.disk, letter1, letter2, row[0], row[1]) - else: - print '/data/%s.%s/nebulous/%s%s/%s/%s' % (self.node, self.disk, letter1, letter2, row[0], row[1]) - except _mysql_exceptions.ProgrammingError, e: - print e - print 'Problem with %s %s %s' % (self.node, self.disk, tablename) - except _mysql_exceptions.InternalError, e: - print "InternalError: %s %s" % (self.node, self.disk) - print e - print - -def process_options(arguments): - if len(arguments) == 1: - usage(arguments[0]) - sys.exit(1) - domain = 'all' - count = 1 - show_size = False - while count < len(arguments): - if arguments[count] == '-domain': - domain = arguments[count+1] - count += 2 - elif arguments[count] == '-size': - show_size = True - count += 1 - else: - pattern = arguments[count] - count += 1 - if domain == 'mhpcc': - nodes = Nodes.mhpcc - elif domain == 'atrc': - nodes = Nodes.atrc - elif domain == 'stsci': - nodes = Nodes.stsci - elif domain == 'all': - nodes = Nodes.mhpcc - nodes.extend(Nodes.atrc) - nodes.extend(Nodes.stsci) - else: - usage(arguments[0]) - sys.exit(1) - return pattern, nodes, show_size - -if __name__ == '__main__': - pattern, nodes, show_size = process_options(sys.argv) - queryingThreads = [] - for (node, disk) in nodes: - try: - queryingThreads.append(QueryingThread(node, disk, pattern, show_size)) - except _mysql_exceptions.OperationalError, e: - print e - for t in queryingThreads: - t.start() - for t in queryingThreads: - t.join() -