Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/Makefile.am (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/Makefile.am (revision 37068) @@ -0,0 +1,2 @@ +SUBDIRS = scripts share . + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/Makefile.am (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/Makefile.am (revision 37068) @@ -0,0 +1,11 @@ +bin_SCRIPTS = \ + cluster_status.py \ + deneb-locate.py \ + generate_crontabs_and_deployment_script.py \ + generate_tables.py \ + node_status.py \ + populate_table.py \ + refresh_table.py \ + snapshot.py \ + update_table.py \ + update_table_undefined.py Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/cluster_status.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/cluster_status.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/cluster_status.py (revision 37068) @@ -0,0 +1,344 @@ +#!/usr/bin/env python + +import os +import sys +import MySQLdb +import subprocess +import _mysql_exceptions +import re + +toGigs = 1024.**3 +toGigsFromK = 1024.**2 + +bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + +def usage(): + sys.stderr.write('Usage: cluster_status.py [-sql] \n') + sys.stderr.write('Usage: cluster_status.py [-h|-help|--help]\n') + sys.stderr.write('\n') + sys.stderr.write('\t Nebulous disk usage about \n') + sys.stderr.write('\t is the usual node name, e.g. ipp008, ..., ipp064\n') + sys.stderr.write('\t can be:\n') + sys.stderr.write('\t\t * mhpcc: all storage nodes at mhpcc\n') + sys.stderr.write('\t\t * set0: ipp004, ipp005, ipp006, ipp007, ipp008, ipp009, ipp060, ipp061, ipp062, ipp063, ipp064, ipp065, ipp066.0\n') + sys.stderr.write('\t\t * set1: ipp01x\n') + sys.stderr.write('\t\t * set2: ipp02x\n') + sys.stderr.write('\t\t * set3: ipp03x\n') + sys.stderr.write('\t\t * set4: ipp04x\n') + sys.stderr.write('\t\t * set5: ipp05x\n') + sys.stderr.write('\t\t * atrc: ippb0x (all 3 partitions for each node)\n') + sys.stderr.write('\t\t * stcsi: stcsi0x (all 3 partitions for each node)\n') + sys.stderr.write('\t\t * all: all nodes\n') + sys.stderr.write('\n') + sys.stderr.write('\t\t [-sql] Check if the mysql server is running on the nodes\n') + +def process_arguments(argv): + sql_check = False + if len(argv) == 1: + usage() + sys.exit(1) + elif argv[1] == '-h': + usage() + sys.exit(1) + elif argv[1] == '-help': + usage() + sys.exit(1) + elif argv[1] == '--help': + usage() + sys.exit(1) + elif argv[1] == '-sql': + sql_check = True + elif argv[1] == 'mhpcc': + nodesFilename = '%s/ippMhpccStorageNodes' % share_directory + nodesFile = open(nodesFilename) + nodes = [] + for line in nodesFile: + nodes.append(line[:-1] + ".0") + elif argv[1] == 'set0': + nodes = [ "ipp004.0", "ipp005.0", "ipp006.0", "ipp007.0", "ipp008.0", "ipp009.0", + "ipp060.0", "ipp061.0", "ipp062.0", "ipp063.0", "ipp064.0", "ipp065.0", + "ipp066.0", ] + elif argv[1] == 'set1': + nodes = [ "ipp010.0", "ipp011.0", "ipp012.0", "ipp013.0", "ipp014.0", + "ipp015.0", "ipp016.0", "ipp017.0", "ipp018.0", "ipp019.0", ] + elif argv[1] == 'set2': + nodes = [ "ipp020.0", "ipp021.0", "ipp023.0", "ipp024.0", + "ipp025.0", "ipp026.0", "ipp027.0", "ipp028.0", "ipp029.0", ] + elif argv[1] == 'set3': + nodes = [ "ipp030.0", "ipp031.0", "ipp032.0", "ipp033.0", "ipp034.0", + "ipp035.0", "ipp036.0", "ipp037.0", "ipp038.0", "ipp039.0", ] + elif argv[1] == 'set4': + nodes = [ "ipp040.0", "ipp041.0", "ipp042.0", "ipp043.0", "ipp044.0", + "ipp045.0", "ipp046.0", "ipp047.0", "ipp048.0", "ipp049.0", ] + elif argv[1] == 'set5': + nodes = [ "ipp050.0", "ipp051.0", "ipp052.0", "ipp053.0", "ipp054.0", + "ipp055.0", "ipp056.0", "ipp057.0", "ipp058.0", "ipp059.0", ] + elif argv[1] == 'ippb': + nodes = [ "ippb00.0", "ippb00.1", "ippb00.2", + "ippb01.0", "ippb01.1", "ippb01.2", + "ippb02.0", "ippb02.1", "ippb02.2", + "ippb03.0", "ippb03.1", "ippb03.2", ] + elif argv[1] == 'stsci': + nodes = [ "stsci00.0", "stsci00.1", "stsci00.2", + "stsci01.0", "stsci01.1", "stsci01.2", + "stsci02.0", "stsci02.1", "stsci02.2", + "stsci03.0", "stsci03.1", "stsci03.2", + "stsci04.0", "stsci04.1", "stsci04.2", + "stsci05.0", "stsci05.1", "stsci05.2", + "stsci06.0", "stsci06.1", "stsci06.2", + "stsci07.0", "stsci07.1", "stsci07.2", + "stsci08.0", "stsci08.1", "stsci08.2", + "stsci09.0", "stsci09.1", "stsci09.2", + "stsci10.0", "stsci10.1", "stsci10.2", + "stsci11.0", "stsci11.1", "stsci11.2", + "stsci12.0", "stsci12.1", "stsci12.2", + "stsci13.0", "stsci13.1", "stsci13.2", + "stsci14.0", "stsci14.1", "stsci14.2", + "stsci15.0", "stsci15.1", "stsci15.2", + "stsci16.0", "stsci16.1", "stsci16.2", + "stsci17.0", "stsci17.1", "stsci17.2", + "stsci18.0", "stsci18.1", "stsci18.2", + "stsci19.0", "stsci19.1", "stsci19.2", ] + elif argv[1] == 'all': + nodesFilename = '%s/ippMhpccStorageNodes' % share_directory + nodesFile = open(nodesFilename) + nodes = [] + for line in nodesFile: + nodes.append(line[:-1] + ".0") + nodes.extend([ "ippb00.0", "ippb00.1", "ippb00.2", + "ippb01.0", "ippb01.1", "ippb01.2", + "ippb02.0", "ippb02.1", "ippb02.2", + "ippb03.0", "ippb03.1", "ippb03.2", ]) + nodes.extend( [ "stsci00.0", "stsci00.1", "stsci00.2", + "stsci01.0", "stsci01.1", "stsci01.2", + "stsci02.0", "stsci02.1", "stsci02.2", + "stsci03.0", "stsci03.1", "stsci03.2", + "stsci04.0", "stsci04.1", "stsci04.2", + "stsci05.0", "stsci05.1", "stsci05.2", + "stsci06.0", "stsci06.1", "stsci06.2", + "stsci07.0", "stsci07.1", "stsci07.2", + "stsci08.0", "stsci08.1", "stsci08.2", + "stsci09.0", "stsci09.1", "stsci09.2", + "stsci10.0", "stsci10.1", "stsci10.2", + "stsci11.0", "stsci11.1", "stsci11.2", + "stsci12.0", "stsci12.1", "stsci12.2", + "stsci13.0", "stsci13.1", "stsci13.2", + "stsci14.0", "stsci14.1", "stsci14.2", + "stsci15.0", "stsci15.1", "stsci15.2", + "stsci16.0", "stsci16.1", "stsci16.2", + "stsci17.0", "stsci17.1", "stsci17.2", + "stsci18.0", "stsci18.1", "stsci18.2", + "stsci19.0", "stsci19.1", "stsci19.2", ] ) + else: + nodes = [ ] + for node in argv[1:]: + nodes.append((node + ".0")) + return (sql_check, nodes) + +def get_nebdf_output(): + nebdf_output = dict() + p = subprocess.Popen( ['neb-df', '--names', ], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + if p.wait()!=0: + logger.error('Error while running neb-df') + for line in p.stderr: + logger.error(line) + sys.exit(3) + for line in p.stdout: + if not line.startswith('Filesystem') and not line.startswith('/summmary') and not line.startswith(' '): + line = re.sub('\s+', ':', line[:-1]) + (node, available, used, free, percentage, mountpoint) = line.split(':') + nebdf_output[node] = (float(available)/toGigsFromK, float(used)/toGigsFromK, float(free)/toGigsFromK) + return nebdf_output + +def is_mysql_server_running(node): + try: + MySQLdb.connect(node, 'ippuser', 'ipp', 'DisksMonitoring') + return "mysql server: Running" + except _mysql_exceptions.OperationalError: + return "mysql server: Stopped" + +def is_update_running(node): + p = subprocess.Popen( ['%s/node_status.py' % bin_directory, node], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + if p.wait()!=0: + return "Can't tell if node's running update/refresh (ssh error for %s)" % node + for line in p.stdout: + return " ".join(line[:-1].split(' ')[1:]) + +if __name__ == '__main__': + (sql_check, nodes) = process_arguments(sys.argv) + nebdf = get_nebdf_output() + dms_db = MySQLdb.connect('ippc11', 'ippuser', 'ipp', + 'DiskMonitoringSummary') + dms_cursor = dms_db.cursor() + aggregate_by_role = dict() + aggregate_by_stage = dict() + nodes_sizes = dict() + aggregate = dict() + raw_aggregate = dict() + aggregate_percentage = 0. + for node in nodes: + try: + query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='deleted'" % node + dms_cursor.execute(query) + total = dms_cursor.fetchall()[0][0] + #print total, query + query = "SELECT SUM(counts) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node + dms_cursor.execute(query) + defined = dms_cursor.fetchall()[0][0] + #print defined, query + percentage_done = float(defined) / float(total) + # + try: + query = "SELECT SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted'" % node + dms_cursor.execute(query) + nodes_sizes[node] = float(dms_cursor.fetchall()[0][0]) / percentage_done / toGigs + except TypeError, e: + print e + node_name = node.split('.')[0] + if sql_check: + print 'Node %s: %5.1f%% done | %20s | %40s' % (node, percentage_done * 100., + is_mysql_server_running(node_name), + is_update_running(node_name)) + else: + print 'Node %s: %5.1f%% done' % (node, percentage_done * 100.) + aggregate_percentage += percentage_done + query = "SELECT role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY role" % node + dms_cursor.execute(query) + for row in dms_cursor.fetchall(): + if row[0] != 'undefined': + try: + aggregate_by_role[row[0]] += (float(row[1])/percentage_done/toGigs) + except KeyError: + aggregate_by_role[row[0]] = (float(row[1])/percentage_done/toGigs) + query = "SELECT stage, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage" % node + dms_cursor.execute(query) + for row in dms_cursor.fetchall(): + if row[0] != 'undefined': + try: + aggregate_by_stage[row[0]] += (float(row[1])/percentage_done/toGigs) + except KeyError: + aggregate_by_stage[row[0]] = (float(row[1])/percentage_done/toGigs) + query = "SELECT stage, role, SUM(sizes) FROM CurrentDiskUsage WHERE node='%s' AND status!='undefined' AND status!='deleted' GROUP BY stage, role" % node + dms_cursor.execute(query) + for row in dms_cursor.fetchall(): + if row[0] != 'undefined': + try: + aggregate[row[0]][row[1]] += (float(row[2])/percentage_done/toGigs) + raw_aggregate[row[0]][row[1]] += float(row[2])/toGigs + except KeyError: + try: + aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs) + raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs + except KeyError: + aggregate[row[0]] = dict() + raw_aggregate[row[0]] = dict() + aggregate[row[0]][row[1]] = (float(row[2])/percentage_done/toGigs) + raw_aggregate[row[0]][row[1]] = float(row[2])/toGigs + except TypeError, e: + print 'Node %s: being refreshed' % (node) + print e + aggregate_percentage /= len(nodes) + + print + print "### By role (in gigs / relative % / in teras)" + total = 0. + for size in aggregate_by_role.itervalues(): + total += size + for role, size in aggregate_by_role.iteritems(): + print "%10s: %10.1f %10.1f%% %10.1f" % (role, size, size/total*100., size/1024.) + print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.) + + print + print "### By stage (in gigs / relative % / in teras)" + total = 0. + for size in aggregate_by_stage.itervalues(): + total += size + for stage, size in aggregate_by_stage.iteritems(): + print "%10s: %10.1f %10.1f%% %10.1f" % (stage, size, size/total*100., size/1024.) + print "%10s: %10.1f %11s %10.1f" % ("Total", total, "", total/1024.) + print + + if aggregate_percentage == 1.: + print + print "### Interpolated Sizes (in gigs), if all nodes were all 100% done" + header = '%10s' % '' + roles = dict() + for stage in aggregate.iterkeys(): + header = '%s%10s' % (header, stage) + for role in aggregate[stage].iterkeys(): + roles[role] = '' + print header + for role in roles.iterkeys(): + line = '%10s' % role + for stage in aggregate.iterkeys(): + try: + line = '%10s%10.1f' % (line, aggregate[stage][role]) + except KeyError: + line = '%10s%10s' % (line, '-') + print line + + print + print "### Sizes (in gigs)" + header = '%10s' % '' + roles = dict() + for stage in aggregate.iterkeys(): + header = '%s%10s' % (header, stage) + for role in aggregate[stage].iterkeys(): + roles[role] = '' + print header + for role in roles.iterkeys(): + line = '%10s' % role + for stage in aggregate.iterkeys(): + try: + line = '%10s%10.1f' % (line, raw_aggregate[stage][role]) + except KeyError: + line = '%10s%10s' % (line, '-') + print line + + print + print "### By node: Mine \t neb-df \tDifference (compared to disk size)\t\"Non-nebulous\" space" + neb_total = 0. + nebdf_total = 0. + nonneb_total = 0. + available = 0. + for node in nodes: + try: + try: + neb_total += nodes_sizes[node] + except KeyError, e: + print 'No nodes_sizes[node]' + raise e + try: + nebdf_total += nebdf[node][1] + except KeyError, e: + print 'No nodes_nebdf[node]' + raise e + nonneb_total += (nebdf[node][1]-nodes_sizes[node]) + available += nebdf[node][0] + print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( node, nodes_sizes[node], nebdf[node][1], + (nodes_sizes[node]-nebdf[node][1])/nebdf[node][0]*100., + nebdf[node][1]-nodes_sizes[node] + ) + except KeyError: + print '%10s: Incomplete data' % node + print "---------------------------------------------------" + print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total", neb_total, nebdf_total, + (neb_total-nebdf_total)/available*100., + nonneb_total ) + exclude_nodes = ['ipp004.0', 'ipp005.0', 'ipp064.0', ] + for node in exclude_nodes: + if node in nodes_sizes: + neb_total -= nodes_sizes[node] + nebdf_total -= nebdf[node][1] + nonneb_total -= (nebdf[node][1]-nodes_sizes[node]) + available -= nebdf[node][0] + print '%10s: %10.2f %10.2f %10.2f%% %10.2f' % ( "Total*", neb_total, nebdf_total, + (neb_total-nebdf_total)/available*100., + nonneb_total ) + print "Total* = Total excluding nodes:", exclude_nodes + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/deneb-locate.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/deneb-locate.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/deneb-locate.py (revision 37068) @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +import os +import threading +import MySQLdb +import sys +import _mysql_exceptions + +def usage(progname): + print """ +Usage: %s [-domain ] [-size] + + Get the files on the cluster whose nebkey matches the + + Options: + -domain : with set to mhpcc (wrt atrc, wrt stsci), all (default: all) + Restrict the search to MHPCC (wrt ATRC, wrt STSCI) storages nodes + 'all': search on all nodes (default) + -size: display the size of the file + + Examples: + - Search a file on all nodes + %s gpc1/20100228/o5255g0304o/o5255g0304o.ota71.burn.tbl + - Search with SQL wildcard (can be really long) + %s gpc1/20100228/%%/o5255g0308o.ota11.burn.tbl (SQL wildcard) + - Search only on STSCI nodes + %s -domain stsci gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl + - Search only on ATRC nodes + %s -domain atrc gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl + - Search only on MHPCC nodes + %s -domain mhpcc gpc1/20100228/o5255g0308o/o5255g0308o.ota11.burn.tbl + - Search a file on all nodes and show their sizes + %s -size gpc1/20100228/o5255g0304o/o5255g0304o.ota71.burn.tbl + + Notes: + * Error messages as (2003, "Can't connect to MySQL server on '' (111)") + can be displayed if for some reason one of the MySQL servers (e.g. the one on ) + has crashed. + * Accuracy of the result is not asserted since the nebulous directory contents are + refreshed only once a day. + * If the file size is -1, it means that the file has been inserted very recently and not been been "refreshed" yet + """ % (progname, progname, progname, progname, progname, progname, progname) + +class Nodes: + mhpcc = [ ("ipp004", "0"), + ("ipp005", "0"), ("ipp006", "0"), ("ipp007", "0"), ("ipp008", "0"), ("ipp009", "0"), + ("ipp010", "0"), ("ipp011", "0"), ("ipp012", "0"), ("ipp013", "0"), ("ipp014", "0"), + ("ipp015", "0"), ("ipp016", "0"), ("ipp017", "0"), ("ipp018", "0"), ("ipp019", "0"), + ("ipp020", "0"), ("ipp021", "0"), ("ipp023", "0"), ("ipp024", "0"), + ("ipp025", "0"), ("ipp026", "0"), ("ipp027", "0"), ("ipp028", "0"), ("ipp029", "0"), + ("ipp030", "0"), ("ipp031", "0"), ("ipp032", "0"), ("ipp033", "0"), ("ipp034", "0"), + ("ipp035", "0"), ("ipp036", "0"), ("ipp037", "0"), ("ipp038", "0"), ("ipp039", "0"), + ("ipp040", "0"), ("ipp041", "0"), ("ipp042", "0"), ("ipp043", "0"), ("ipp044", "0"), + ("ipp045", "0"), ("ipp046", "0"), ("ipp047", "0"), ("ipp048", "0"), ("ipp049", "0"), + ("ipp050", "0"), ("ipp051", "0"), ("ipp052", "0"), ("ipp053", "0"), ("ipp054", "0"), + ("ipp055", "0"), ("ipp056", "0"), ("ipp057", "0"), ("ipp058", "0"), ("ipp059", "0"), + ("ipp060", "0"), ("ipp061", "0"), ("ipp062", "0"), ("ipp063", "0"), ("ipp064", "0"), + ("ipp065", "0"), ("ipp066", "0"), + ] + atrc = [ ("ippb00", "0"), ("ippb00", "1"), ("ippb00", "2"), + ("ippb01", "0"), ("ippb01", "1"), ("ippb01", "2"), + ("ippb02", "0"), ("ippb02", "1"), ("ippb02", "2"), + ("ippb03", "0"), ("ippb03", "1"), ("ippb03", "2"), + ] + stsci = [ ("stsci00", "0"), ("stsci00", "1"), ("stsci00", "2"), + ("stsci01", "0"), ("stsci01", "1"), ("stsci01", "2"), + ("stsci02", "0"), ("stsci02", "1"), ("stsci02", "2"), + ("stsci03", "0"), ("stsci03", "1"), ("stsci03", "2"), + ("stsci04", "0"), ("stsci04", "1"), ("stsci04", "2"), + ("stsci05", "0"), ("stsci05", "1"), ("stsci05", "2"), + ("stsci06", "0"), ("stsci06", "1"), ("stsci06", "2"), + ("stsci07", "0"), ("stsci07", "1"), ("stsci07", "2"), + ("stsci08", "0"), ("stsci08", "1"), ("stsci08", "2"), + ("stsci09", "0"), ("stsci09", "1"), ("stsci09", "2"), + ("stsci10", "0"), ("stsci10", "1"), ("stsci10", "2"), + ("stsci11", "0"), ("stsci11", "1"), ("stsci11", "2"), + ("stsci12", "0"), ("stsci12", "1"), ("stsci12", "2"), + ("stsci13", "0"), ("stsci13", "1"), ("stsci13", "2"), + ("stsci14", "0"), ("stsci14", "1"), ("stsci14", "2"), + ("stsci15", "0"), ("stsci15", "1"), ("stsci15", "2"), + ("stsci16", "0"), ("stsci16", "1"), ("stsci16", "2"), + ("stsci17", "0"), ("stsci17", "1"), ("stsci17", "2"), + ("stsci18", "0"), ("stsci18", "1"), ("stsci18", "2"), + ("stsci19", "0"), ("stsci19", "1"), ("stsci19", "2"), + ] + +class QueryingThread(threading.Thread): + count_query_exact = "SELECT COUNT(*) FROM %s WHERE nebkey = '%s'" + count_query_approximate = "SELECT COUNT(*) FROM %s WHERE nebkey LIKE '%s'" + query_exact = "SELECT subdirname, filename FROM %s WHERE nebkey = '%s'" + query_approximate = "SELECT subdirname, filename FROM %s WHERE nebkey LIKE '%s'" + query_exact_with_size = "SELECT subdirname, filename, size FROM %s WHERE nebkey = '%s'" + query_approximate_with_size = "SELECT subdirname, filename, size FROM %s WHERE nebkey LIKE '%s'" + letters = "0123456789abcdef" + + def __init__(self, node, disk, pattern, show_size): + threading.Thread.__init__(self) + self.node = node + self.disk = disk + self.show_size = show_size + self.db = MySQLdb.connect(host=node, user='ippuser', passwd='ipp', db='DisksMonitoring') + self.cursor = self.db.cursor() + if '%' in pattern: + if self.show_size: + self.count_query = QueryingThread.count_query_approximate + self.query = QueryingThread.query_approximate_with_size + else: + self.count_query = QueryingThread.count_query_approximate + self.query = QueryingThread.query_approximate + else: + if self.show_size: + self.count_query = QueryingThread.count_query_exact + self.query = QueryingThread.query_exact_with_size + else: + self.count_query = QueryingThread.count_query_exact + self.query = QueryingThread.query_exact + + def run(self): + try: + for letter1 in QueryingThread.letters: + for letter2 in QueryingThread.letters: + tablename = 'neb_%s_%s%s' % (self.disk, letter1, letter2) + self.cursor.execute(self.count_query % (tablename, pattern)) + rows = self.cursor.fetchall() + count = rows[0][0] + if count != 0: + self.cursor.execute(self.query % (tablename, pattern)) + for row in self.cursor.fetchall(): + if self.show_size: + print '%9d | /data/%s.%s/nebulous/%s%s/%s/%s' % (row[2], self.node, self.disk, letter1, letter2, row[0], row[1]) + else: + print '/data/%s.%s/nebulous/%s%s/%s/%s' % (self.node, self.disk, letter1, letter2, row[0], row[1]) + except _mysql_exceptions.ProgrammingError, e: + print e + print 'Problem with %s %s %s' % (self.node, self.disk, tablename) + except _mysql_exceptions.InternalError, e: + print "InternalError: %s %s" % (self.node, self.disk) + print e + print + +def process_options(arguments): + if len(arguments) == 1: + usage(arguments[0]) + sys.exit(1) + domain = 'all' + count = 1 + show_size = False + while count < len(arguments): + if arguments[count] == '-domain': + domain = arguments[count+1] + count += 2 + elif arguments[count] == '-size': + show_size = True + count += 1 + else: + pattern = arguments[count] + count += 1 + if domain == 'mhpcc': + nodes = Nodes.mhpcc + elif domain == 'atrc': + nodes = Nodes.atrc + elif domain == 'stsci': + nodes = Nodes.stsci + elif domain == 'all': + nodes = Nodes.mhpcc + nodes.extend(Nodes.atrc) + nodes.extend(Nodes.stsci) + else: + usage(arguments[0]) + sys.exit(1) + return pattern, nodes, show_size + +if __name__ == '__main__': + pattern, nodes, show_size = process_options(sys.argv) + queryingThreads = [] + for (node, disk) in nodes: + try: + queryingThreads.append(QueryingThread(node, disk, pattern, show_size)) + except _mysql_exceptions.OperationalError, e: + print e + for t in queryingThreads: + t.start() + for t in queryingThreads: + t.join() + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_crontabs_and_deployment_script.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_crontabs_and_deployment_script.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_crontabs_and_deployment_script.py (revision 37068) @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import os + +bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + +REFRESH_SCRIPT_LOCATION = '%s/refresh_table.py' % bin_directory +ROOT_DIRECTORY = '%s/deployment' % share_directory +PYTHONPATH = '%s/python' % share_directory +NODES_LIST = '%s/allStorageNodes' % share_directory +OUTPUT_DIRECTORY = '%s/scripts' % share_directory +DEPLOYMENT_SCRIPT = '%s/deploy.sh' % OUTPUT_DIRECTORY +NODE_SCRIPT_FORMAT = '%s/crontab.%%s' % OUTPUT_DIRECTORY + +if __name__ == '__main__': + nodes_file = open(NODES_LIST) + if not os.path.exists(OUTPUT_DIRECTORY): + os.makedirs(OUTPUT_DIRECTORY) + deployment_script_file = open(DEPLOYMENT_SCRIPT, 'w') + deployment_script_file.write('#!/bin/bash\n\n') + minutes = 1 + hours = 0 + for node in nodes_file: + node = node[:-1] + print '... Generating script for %s' % node + crontab_filename = NODE_SCRIPT_FORMAT % node + deployment_script_file.write("echo '%s'\n" % (node)) + deployment_script_file.write("ssh %s 'crontab %s'\n" % (node, crontab_filename)) + crontab_file = open(crontab_filename, 'w') + crontab_file.write("SHELL=/bin/tcsh\n") + crontab_file.write("MAILTO=%s@ifa.hawaii.edu\n" % os.environ['USER'] ) + crontab_file.write("PYTHONPATH=%s\n" % PYTHONPATH) + crontab_file.write("%d %d * * * nice %s -disk 0\n" % (minutes, hours, REFRESH_SCRIPT_LOCATION)) + if not node.startswith('ipp'): #atrc or stsci nodes: add disks 1 and 2 + crontab_file.write("%d %d * * * nice %s -disk 1\n" % ((minutes+1)%60, hours, REFRESH_SCRIPT_LOCATION)) + crontab_file.write("%d %d * * * nice %s -disk 2\n" % ((minutes+2)%60, hours, REFRESH_SCRIPT_LOCATION)) + minutes += 7 + minutes %= 60 + hours += 1 + hours %= 24 + crontab_file.close() + nodes_file.close() + deployment_script_file.close() + print '\nNow run: %s\n' % DEPLOYMENT_SCRIPT + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_tables.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_tables.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/generate_tables.py (revision 37068) @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import os + +letters = '0123456789abcdef' +share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + +nodes = [] +nodesFile = open('%s/ippMhpccStorageNodes' % share_directory) +for line in nodesFile: + nodes.append(line[:-1]) +nodesFile.close() + +template = "" +templateFile = open('%s/sql/CreateTablesTemplate.sql' % share_directory) +for line in templateFile: + template = "%s%s" % (template, line) +templateFile.close() + +for letter1 in letters: + for letter2 in letters: + directory = '%s%s' % (letter1, letter2) + for node in nodes: + print template.replace('@HOST@', + node).replace('@DIRECTORY@', + directory) Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/node_status.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/node_status.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/node_status.py (revision 37068) @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +import sys +import os +import warnings +warnings.simplefilter("ignore") # Importing paramiko displays the following warning: +#/home/panstarrs/schastel/local/lib/python2.5/site-packages/pycrypto-2.5-py2.5-linux-x86_64.egg/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability. +# _warn("Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.", PowmInsecureWarning) +import paramiko +warnings.simplefilter("always") +import datetime + +bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +node_directory = '/export/%s/ipp_filemon/' + +def process_arguments(argv): + if len(argv) == 1: + usage() + sys.exit(1) + elif argv[1] == 'all': + nodesFilename = '%s/ippMhpccStorageNodes' % share_directory + nodesFile = open(nodesFilename) + nodes = [] + for line in nodesFile: + nodes.append(line[:-1] + ".0") + elif argv[1] == 'set0': + nodes = [ "ipp004.0", "ipp005.0", "ipp006.0", "ipp007.0", "ipp008.0", "ipp009.0", + "ipp060.0", "ipp061.0", "ipp062.0", "ipp063.0", "ipp064.0", "ipp065.0", "ipp066.0", ] + elif argv[1] == 'set1': + nodes = [ "ipp010.0", "ipp011.0", "ipp012.0", "ipp013.0", "ipp014.0", + "ipp015.0", "ipp016.0", "ipp017.0", "ipp018.0", "ipp019.0", ] + elif argv[1] == 'set2': + nodes = [ "ipp020.0", "ipp021.0", "ipp023.0", "ipp024.0", + "ipp025.0", "ipp026.0", "ipp027.0", "ipp028.0", "ipp029.0", ] + elif argv[1] == 'set3': + nodes = [ "ipp030.0", "ipp031.0", "ipp032.0", "ipp033.0", "ipp034.0", + "ipp035.0", "ipp036.0", "ipp037.0", "ipp038.0", "ipp039.0", ] + elif argv[1] == 'set4': + nodes = [ "ipp040.0", "ipp041.0", "ipp042.0", "ipp043.0", "ipp044.0", + "ipp045.0", "ipp046.0", "ipp047.0", "ipp048.0", "ipp049.0", ] + elif argv[1] == 'set5': + nodes = [ "ipp050.0", "ipp051.0", "ipp052.0", "ipp053.0", "ipp054.0", + "ipp055.0", "ipp056.0", "ipp057.0", "ipp058.0", "ipp059.0", ] + else: + nodes = [] + for node in argv[1:]: + nodes.append(node + ".0") + return nodes + +def get_seconds_since_last_message(ssh, node): + logfilename = ('%s/%s' % (node_directory, '/logs/%s_refresh.log') % (node, node)) + stdin, stdout, stderr = ssh.exec_command('tail -n 1 %s' % (logfilename)) + for line in stdout: + try: + last_message_date = datetime.datetime.strptime(line[:-1].split(' ')[0], '%Y-%m-%dT%H:%M:%S') + except ValueError: + return 999999 + now = datetime.datetime.now() + try: + td = now - last_message_date + except UnboundLocalError: #local variable 'last_message_date' migh tbe referenced before assignment + return 999999 + return td.seconds + td.days * 24 * 3600 + +def get_crontab_status(ssh): + stdin, stdout, stderr = ssh.exec_command('crontab -l | grep refresh_host.sh') + crontab_installed = False + next_run = None + for line in stdout: + crontab_installed = True + if 'nice' not in line: + print 'In crontab but NOT nice' + schedule = line.split(' ') + next_run = schedule[1] + ':' + schedule[0] + return (crontab_installed, next_run) + +def guess_if_update_or_refresh_is_running(ssh): + stdin, stdout, stderr = ssh.exec_command('ps waux | grep update_host_undefined.sh | grep -v grep') + update_is_running = False + for line in stdout: + update_is_running = True + stdin, stdout, stderr = ssh.exec_command('ps waux | grep refresh_table.py | grep -v grep') + refresh_is_running = False + for line in stdout: + refresh_is_running = True + if update_is_running: + if refresh_is_running: + message = "Both Update and Refresh are running" + else: + message = "Update is running" + else: + if refresh_is_running: + message = "Refresh is running" + else: + message = "Nothing is running" + return message + +if __name__ == '__main__': + nodes = process_arguments(sys.argv) + show_crontab = False + for node in nodes: + node_name = node.split('.')[0] + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(node_name) + seconds_since_last_message = get_seconds_since_last_message(ssh, node) + (is_crontab_installed, next_run) = get_crontab_status(ssh) + print '%s: %s' % (node, guess_if_update_or_refresh_is_running(ssh)) + # if seconds_since_last_message > 1000 and not is_crontab_installed: + # print 'Node %s: Not installed in crontab and last run was %d seconds ago' % (node, seconds_since_last_message) + # else: + # if is_crontab_installed: + # if seconds_since_last_message < 1000: + # msg = "OK" + # else: + # msg = "Stopped or crashed? (next run at %s)" % next_run + # else: + # msg= "Running (but crontab is not installed)" + # show_crontab = True + # print 'Node %s: %s' % (node, msg) + # if show_crontab: + # print '58 13 * * * /usr/bin/nice -n 19 /home/panstarrs/schastel/dev/FilesMonitoring/refresh_host.sh' Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/populate_table.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/populate_table.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/populate_table.py (revision 37068) @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import subprocess +import sys +import os + +bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) +node_directory = '/export/%s.0/ipp_filemon/' + +def populate(node, directory): + root_dir = "%s/populate" % node_directory + try: + os.makedirs(('%s/sql' % root_dir) % node) + except OSError: + pass + output = open(('%s/%s' % (root_dir, 'sql/populate_%s_%s.sql')) % (node, node, directory), + 'w') + ignored = open(('%s/%s' % (root_dir, 'log/ignored_%s_%s')) % (node, node, directory), + 'w') + stdout = open(('%s/%s' % (root_dir, 'log/stdout_%s_%s')) % (node, node, directory), + 'w') + index_filename = ('%s/%s' % (node_directory,'/mlocate/%s.%s.mlocate_db')) % (node, node, directory) + p = subprocess.Popen(('%s/locate -d %s .' + % bin_directory,index_filename).split(' '), + stdout = subprocess.PIPE) + p.poll() + to_remove = '/export/%s.0/nebulous/%s' % (node, directory) + count = 1 + for line in p.stdout: + if count % 10000 == 0: + stdout.write('%s %s %s\n' % (node, directory, count)) + line = line[:-1].replace(to_remove, '') + fields = line.split('/') + if len(fields) == 3: + count += 1 + subdirectory = fields[1] + filename = fields[2] + elements = filename.split('.') + neb_key = '.'.join(elements[1:]).replace(':', '/') + output.write(insertTemplate % (node, directory, filename, elements[0], subdirectory, neb_key)) + else: + ignored.write('Ignored: %s %s %s\n' % (node, directory, line)) + output.close() + +letters = '0123456789abcdef' +insertTemplate="" +sqlFile = open('%s/sql/insert_template.sql' % share_directory) +for line in sqlFile: + insertTemplate = "%s%s" % (insertTemplate, line) +sqlFile.close() + +node = sys.argv[1] +for letter1 in letters: + for letter2 in letters: + directory = '%s%s' % (letter1, letter2) + populate(node, directory) + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/refresh_table.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/refresh_table.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/refresh_table.py (revision 37068) @@ -0,0 +1,555 @@ +#!/usr/bin/env python + +import sys +import os +import socket +import subprocess +import MySQLdb +import logging +import _mysql_exceptions +import datetime +import time + +################################### +# +class Constants: # Constants and/or default values + force = False # if we want to ignore the lock + limit_undefined = 10000 # the maximum number of undefined entries that we want to refresh + limit = 1000 # the maximum number of the oldest refreshed entries that we want to refresh + loggingLevel = logging.INFO # + skip_stages = [] +# Place we find all the installed IPP stuff + bin_directory = "%s/%s/bin" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + share_directory = "%s/%s/share/filemon/" % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + +# Templates for things we'll either create or read + node_directory = '/export/%s.%s/ipp_filemon/' + mlocate_db_template = "%s/%s" % (node_directory,'mlocate/%s.%s.%s.mlocate_db') + lock_directory = '/tmp/diskMonitoring/%s' # disk + lock_filename_template = '%s/lock.%s' + nebulous_directory_template = '/export/%s.%s/nebulous/%s' + +# Locations of the binaries we'll run + updatedb_binary = '%s/updatedb' % bin_directory + locate_binary = '%s/locate' % bin_directory + update_table_undefined_python_script = "%s/update_table_undefined.py" % bin_directory + update_table_python_script = '%s/update_table.py' % bin_directory + + disk = "0" # Most of the nodes have only one disk /export/.0 + next_directory = None + max_refresh_interval = 6*60*60 # Run the script if the last_modified value associated to the disk to refresh is older than that (6 hours) + # sleep = 60*10 # The time (in seconds) the script sleeps between two iterations + sleep = 1 # The time (in seconds) the script sleeps between two iterations + +def get_next_directory(current_directory): + """ + >>> get_next_directory("00") + '01' + >>> get_next_directory("ff") + '00' + >>> get_next_directory("ab") + 'ac' + >>> get_next_directory("cf") + 'd0' + """ + if Constants.next_directory is None: + Constants.next_directory = dict() + letters = '0123456789abcdef' + directories = [] + for letter1 in letters: + for letter2 in letters: + directories.append('%s%s'% (letter1, letter2)) + for index in range(len(directories)-1): + Constants.next_directory[directories[index]] = directories[index+1] + Constants.next_directory['ff'] = '00' + return Constants.next_directory[current_directory] + +################################### +# +class FilesDatabaseConnection: + def __init__(self, nodename, disk): + self.nodename = nodename + self.disk = disk + if nodename == 'neverland': + self.db = MySQLdb.connect('localhost', 'root', 'root', 'DisksMonitoring') + else: + self.db = MySQLdb.connect('localhost', 'root', '', 'DisksMonitoring') + self.count_inserts = 0 + + def close(self): + self.db.commit() + self.db.close() + + def find_instance(self, neb_ins_id, subdirname, filename): + query = 'SELECT neb_ins_id FROM neb_%s_%s WHERE neb_ins_id = %s' % (self.dirname, + self.disk, + neb_ins_id) + cursor = self.db.cursor() + cursor.execute(query) + rows = cursor.fetchall() + if len(rows) == 0: + print 'New entry' + elif len(rows) == 1: + print 'Existing entry' + else: + print 'Problem' + cursor.close() + + def get_instance_ids(self): + query = 'SELECT neb_ins_id, subdirname, filename FROM neb_%s_%s' % (self.disk, + self.dirname) + cursor = self.db.cursor() + cursor.execute(query) + ids = dict() + for row in cursor.fetchall(): + ids[row[0]] = (row[1], row[2]) + cursor.close() + return ids + + def add_new_file(self, filename, neb_ins_id, subdirname, debug = False): + query = """INSERT INTO neb_%s_%s ( + filename, + neb_ins_id, + subdirname, + nebkey, + creation_date) +VALUES ( + '%s', + %s, + '%s', + '%s', + NOW())""" % (self.disk, + self.dirname, + filename, + neb_ins_id, + subdirname, + generate_nebulous_key(filename)) + if debug: + logger.debug('%s;' % query) + return + cursor = self.db.cursor() + try: + cursor.execute(query) + except _mysql_exceptions.OperationalError, e: + logger.error(str(e)) + logger.info('Trying to replay query: %s' % query) + cursor.execute(query) + self.count_inserts += 1 + cursor.close() + if self.count_inserts >= 100: + self.db.commit() + self.count_inserts = 0 + + def mark_as_deleted(self, neb_ins_id): + table = "neb_%s_%s" % (self.disk, self.dirname) + logger.debug("Deleting entry %d from %s" % (neb_ins_id, table)) + cursor = self.db.cursor() + statement = "INSERT INTO %s_delete(filename, neb_ins_id, subdirname, nebkey, size, md5sum, role, stage, lifetime, status, creation_date, last_check) SELECT filename, neb_ins_id, subdirname, nebkey, size, md5sum, role, stage, lifetime, status, creation_date, last_check FROM %s WHERE neb_ins_id = %d" % (table, table, neb_ins_id) + logger.info('... Statement: (%s)' % (statement)) + try: + cursor.execute(statement) + except _mysql_exceptions.IntegrityError, e: + logger.error(e) + logger.error("Table is %s_delete" % (table)) + if 'Duplicate entry' in str(e): + pass + else: + raise e + except _mysql_exceptions.OperationalError, e: + logger.error(e) + if 'Lock wait timeout exceeded' in str(e): + pass + else: + raise e + statement = 'UPDATE %s_delete SET status=\'deleted\' WHERE neb_ins_id=%d' % (table, neb_ins_id) + logger.info('... Statement: (%s)' % (statement)) + cursor.execute(statement) + statement = 'DELETE FROM %s WHERE neb_ins_id=%d' % (table, neb_ins_id) + logger.info('... Statement: (%s)' % (statement)) + cursor.execute(statement) + cursor.close() + + def optimize_table(self): + cursor = self.db.cursor() + query = 'OPTIMIZE TABLE neb_%s_%s' % (self.disk, + self.dirname) + logger.debug('... Running statement "%s"' % (query)) + cursor.execute(query) + query = 'OPTIMIZE TABLE neb_%s_%s_delete' % (self.disk, + self.dirname) + logger.debug('... Running statement "%s"' % (query)) + cursor.execute(query) + cursor.close() + self.commit() + + def commit(self): + self.db.commit() + + def get_directory_to_refresh(self, force): + cursor = self.db.cursor() + cursor.execute('SELECT dirname, last_modified FROM refresh_status WHERE disk=%s' % self.disk) + for row in cursor.fetchall(): + (dirname, last_modified) = row + cursor.close() + then = last_modified + now = datetime.datetime.now() + td = now - then + delta_seconds = td.seconds + td.days * 24 * 3600 + if delta_seconds < Constants.max_refresh_interval and not force: + logger.warn("No need to restart the script") + sys.exit(0) + self.dirname = get_next_directory(dirname) + return self.dirname + + def update_refresh_status(self, dirname): + self.dirname = dirname + cursor = self.db.cursor() + cursor.execute("UPDATE refresh_status SET dirname = '%s' WHERE disk=%s" % (self.dirname, self.disk)) + self.commit() + cursor.close() + +# ################################### +# # +# class SummaryDatabaseConnection: +# def __init__(self): +# self.nodename = nodename +# self.dirname = dirname +# if nodename == 'neverland': +# self.db = MySQLdb.connect('localhost', 'root', 'root', 'DiskMonitoringSummary') +# else: +# self.db = MySQLdb.connect('localhost', 'ipp', 'ipp', 'DiskMonitoringSummary') + +################################### +# generate_nebulous_key() +def generate_nebulous_key(filename): + elements = filename.split('.') + neb_key = '.'.join(elements[1:]).replace(':', '/') + return neb_key + +################################### +# process_options() +def process_options(arguments): + # Default values: see usage() + force = Constants.force + limit = Constants.limit + skip_stages = Constants.skip_stages + index = 1 + disk = Constants.disk + while index] [-disk ] + %s -h + %s -unittest + + Refreshes files monitoring information about the + different /export/%s.0/nebulous/ nebulous storage + directories. is one of the 00..ff directories. + is obtained from the local database (DisksMonitoring/Status). + + The different stages are: + 0. Initializations. Exit if the information about the last refreshed + directory has been updated less than 24 hours ago. + 1. Get the directory to refresh + 2. Run updatedb on the current directory + 3. Remove deleted files from the FilesMonitoring database + 4. Add new files in the FilesMonitoring database + 5. Refresh the status of the oldest entries in the corresponding + table. + 6. Update the information about the directory. Goto 1 + + Arguments: + None: + : The nebulous directory to be refreshed 00..ff + + Options: + -f: + Force to ignore the lock. Beware of multiple instances of this + program running at the same time + -h, -help, --help: + Displays this help + -v: + Be verbose + -l/-limit : + Update up to new and then oldest files in the monitoring + database. By default, is %d + -disk : + Run for /export/./nebulous. + Default is /export/.%s/nebulous + -skip [,...]: + Debugging mode: Skip + You are supposed to know what you are doing when using this + option! + -unittest + Runs the unit tests associated to this program +""" + nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + sys.stderr.write(usage.__doc__ % (sys.argv[0], sys.argv[0], sys.argv[0], + nodename, Constants.limit, Constants.disk)) + +def refresh(dirname): + logger.info('Refreshing [%s]' % dirname) + ####################### + # 1. Stage 1: Run updatedb on directory /export/%s.%s/nebulous/ + ####################### + logger.info('Stage 1: updatedb') + if '1' not in skip_stages: + directory = Constants.nebulous_directory_template % (nodename, disk, dirname) + mlocate_db = Constants.mlocate_db_template % (nodename, + disk, + nodename, + disk, + dirname) + mlocate_db_directory = "/".join(mlocate_db.split('/')[:-1]) + if not os.path.exists(mlocate_db_directory): + os.makedirs(mlocate_db_directory) + p = subprocess.Popen( [Constants.updatedb_binary, + '-l', '0', + '-U', directory, + '-o', mlocate_db], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + if p.wait()!=0: + logger.error('Error while running updatedb') + for line in p.stderr: + logger.error(line) + sys.exit(3) + logger.info('... Stage 1 complete') + else: + logger.info('... Stage 1 skipped') + + ####################### + # Stage 1.9: Collect all ins_id from the database and the mlocate file + ####################### + logger.info('Stage 1.9 (cannot be skipped): Get ins_id both from database and mlocate file') + if '2' in skip_stages and '3' in skip_stages: #Ignore if there is no database update + logger.info('... Stage 1.9 skipped since both 2 and 3 have to be skipped') + else: + # Collect ins_id from the database + ins_ids_in_db = filesMonitoring_database.get_instance_ids() + logger.debug('... Got %d entries from the database' % len(ins_ids_in_db)) + # Collect ins_id from the locate file + mlocate_db = Constants.mlocate_db_template % (nodename, + disk, + nodename, + disk, + dirname) + p = subprocess.Popen( [Constants.locate_binary, + '-d', mlocate_db, + '.'], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + p.poll() + count = 0 + acceptable = 0 + root = Constants.nebulous_directory_template % (nodename, + disk, + dirname) + ins_ids_in_mlocate = dict() + for line in p.stdout: + long_filename = line[:-1].replace(root, + '') + if long_filename.startswith('/'): + long_filename = long_filename[1:] + try: + (subdirname, filename) = long_filename.split('/') + if logger.isEnabledFor(logging.DEBUG): + logger.debug('... -> %s' % line[:-1]) + logger.debug('...... [%s] split in (%s,%s)' % (long_filename, + subdirname, + filename)) + fields = filename.split('.') + if fields[0] != '': + ins_ids_in_mlocate[int(fields[0])] = (filename, subdirname) + else: + logger.warn('... Check file [%s]' % line[:-1]) + acceptable += 1 + except ValueError: + pass # These are the directories + count += 1 + logger.debug('... Looked at %d entries in updatedb database' % count) + logger.debug('... Found %d acceptable entries in updatedb database' % acceptable) + if acceptable+257!=count: + # We expect to have 256 subdirectories and the root dir + # excluded, that is exactly count-257 acceptable entries + # in the updatedb database. If not, it means that there is + # an unexpected file in the root directory + logger.debug('... !!! There might be a problem in updatedb database !!!') + logger.debug('... Got %d entries from the mlocate file' % len(ins_ids_in_mlocate)) + # Differences + logger.debug('... Checking differences') + deleted_entries = dict() + existing_entries = dict() + new_entries = dict() + for key in ins_ids_in_db.keys(): + if ins_ids_in_mlocate.has_key(key): + existing_entries[key] = ins_ids_in_mlocate[key] + else: + if ins_ids_in_db.has_key(key): + logger.debug('Adding [%s, %s] to files to delete' % (ins_ids_in_db[key])) + deleted_entries[key] = ins_ids_in_db[key] + else: + logger.debug('No entry in database for key [%s]' % key) + for key in ins_ids_in_mlocate.keys(): + if ins_ids_in_db.has_key(key): + existing_entries[key] = ins_ids_in_db[key] + else: + new_entries[key] = ins_ids_in_mlocate[key] + logger.info('... New files.............: %d' % len(new_entries)) + logger.info('... Deleted files.........: %d' % len(deleted_entries)) + logger.info('... Already existing files: %d' % (len(existing_entries))) + logger.info('... Stage 1.9 complete') + + ####################### + # 2. Stage 2: Remove deleted files from the FilesMonitoring database + ####################### + logger.info('Stage 2: remove deleted files from the FilesMonitoring database') + if '2' not in skip_stages: + logger.info('... %d entries to delete' % len(deleted_entries)) + for neb_ins_id in deleted_entries.keys(): + filesMonitoring_database.mark_as_deleted(neb_ins_id) + filesMonitoring_database.commit() + logger.info('... Stage 2 complete') + else: + logger.info('... Stage 2 skipped') + + ####################### + # 3. Add new files to the FilesMonitoring database + ####################### + logger.info('Stage 3: add new files to the FilesMonitoring database') + if '3' not in skip_stages: + logger.info('... %d entries to add' % len(new_entries)) + for neb_ins_id in new_entries.keys(): + filesMonitoring_database.add_new_file(new_entries[neb_ins_id][0], + neb_ins_id, + new_entries[neb_ins_id][1]) + filesMonitoring_database.commit() + logger.info('... Stage 3 complete') + else: + logger.info('... Stage 3 skipped') + + ####################### + # 3.9. OPTIMIZE TABLE + ####################### + logger.info('Stage 3.9 (cannot be skipped): optimize table') + if '2' in skip_stages and '3' in skip_stages: #Ignore if no db update + logger.info('... Stage 3.9 skipped since both 2 and 3 have to be skipped') + else: + filesMonitoring_database.optimize_table() + logger.info('... Stage 3.9 complete') + + ####################### + # 4. Refresh the status of the oldest entries in the + # corresponding table + ####################### + logger.info('Stage 4: refresh the status of the oldest entries (limit = %d)' % limit) + if '4' not in skip_stages: + p = subprocess.Popen( [Constants.update_table_undefined_python_script, + '-disk', disk, + '-l', str(Constants.limit_undefined), + dirname], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + if p.wait()!=0: + logger.error('Error while running %s' % Constants.update_table_undefined_python_script) + for line in p.stderr: + logger.error(line) + sys.exit(3) + p = subprocess.Popen( [Constants.update_table_python_script, + '-disk', disk, + '-l', str(limit), + dirname], + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + if p.wait()!=0: + logger.error('Error while running %s' % Constants.update_table_python_script) + for line in p.stderr: + logger.error(line) + sys.exit(3) + logger.info('... Stage 4 complete') + else: + logger.info('... Stage 4 skipped') + + ################################################## + # 5. Stage 5.9: Update information in refresh_status table + ################################################## + filesMonitoring_database.update_refresh_status(dirname) + +################################### +# Main +if __name__ == '__main__': + ####################### + # 0. Initializations + ####################### + # Get nodename + nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + # 0.1. Setup logging + logger = logging.getLogger() + formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', + '%Y-%m-%dT%H:%M:%S') +# log_directory = '/data/%s.0/schastel/logs' % (nodename) + log_directory = ("%s/logs" % (Constants.node_directory)) % (nodename, Constants.disk) + log_filename = '%s/%s_refresh.log' % (log_directory, nodename) + try: + logging_output = logging.FileHandler(log_filename) + except IOError: + os.makedirs(log_directory) + logging_output = logging.FileHandler(log_filename) + logging_output.setFormatter(formatter) + logger.addHandler(logging_output) + logger.setLevel(Constants.loggingLevel) + # 0.2. Get program arguments + (force, limit, skip_stages, disk) = process_options(sys.argv) + logger.info('%s for [%s]/%s' % (sys.argv[0], nodename, disk)) + logger.info('Stage 0 (cannot be skipped): initializations') + # 0.3. Open a connection to the FilesMonitoring database + try: + filesMonitoring_database = FilesDatabaseConnection(nodename, disk) + except _mysql_exceptions.OperationalError, e: + logger.error('Caught exception while connecting to local MySQL server:\n\t%s' % str(e)) + sys.exit(1) + dirname = filesMonitoring_database.get_directory_to_refresh(force) + logger.info('Stage 0 complete') + while True: + refresh(dirname) + dirname = get_next_directory(dirname) + filesMonitoring_database.dirname = dirname # Dirty! + logger.info('Sleeping for a while (%d seconds)' % Constants.sleep) + time.sleep(Constants.sleep) Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/snapshot.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/snapshot.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/snapshot.py (revision 37068) @@ -0,0 +1,282 @@ +#!/usr/bin/env python + +import MySQLdb +import sys +import socket +import _mysql_exceptions +import datetime +import logging +import os + +def process_arguments(argv): + if len(argv) == 1: + usage() + sys.exit(1) + index = 1 + nodes = [ ] + while index < len(argv): + if argv[index] == '-v': + logger.setLevel(logging.DEBUG) + index += 1 + elif argv[index] == 'mhpcc': + nodesFilename = '%s/%s/share/filemon/ippMhpccStorageNodes' % (os.environ['PSCONFDIR'], os.environ['PSCONFIG']) + nodesFile = open(nodesFilename) + nodes = [] + for line in nodesFile: + nodes.append((line[:-1]+".0")) + index += 1 + elif argv[index] == 'set0': + nodes.extend([ ("ipp004.0"), ("ipp005.0"), ("ipp006.0"), ("ipp007.0"), ("ipp008.0"), ("ipp009.0"), + ("ipp060.0"), ("ipp061.0"), ("ipp062.0"), ("ipp063.0"), ("ipp064.0"), ("ipp065.0"), + ("ipp066.0"), ] ) + index += 1 + elif argv[index] == 'set1': + nodes.extend([ ("ipp010.0"), ("ipp011.0"), ("ipp012.0"), ("ipp013.0"), ("ipp014.0"), + ("ipp015.0"), ("ipp016.0"), ("ipp017.0"), ("ipp018.0"), ("ipp019.0"), ] ) + index += 1 + elif argv[index] == 'set2': + nodes.extend( [ ("ipp020.0"), ("ipp021.0"), ("ipp023.0"), ("ipp024.0"), + ("ipp025.0"), ("ipp026.0"), ("ipp027.0"), ("ipp028.0"), ("ipp029.0"), ] ) + index += 1 + elif argv[index] == 'set3': + nodes.extend([ ("ipp030.0"), ("ipp031.0"), ("ipp032.0"), ("ipp033.0"), ("ipp034.0"), + ("ipp035.0"), ("ipp036.0"), ("ipp037.0"), ("ipp038.0"), ("ipp039.0"), ] ) + index += 1 + elif argv[index] == 'set4': + nodes.extend( [ ("ipp040.0"), ("ipp041.0"), ("ipp042.0"), ("ipp043.0"), ("ipp044.0"), + ("ipp045.0"), ("ipp046.0"), ("ipp047.0"), ("ipp048.0"), ("ipp049.0"), ] ) + index += 1 + elif argv[index] == 'set5': + nodes.extend( [ ("ipp050.0"), ("ipp051.0"), ("ipp052.0"), ("ipp053.0"), ("ipp054.0"), + ("ipp055.0"), ("ipp056.0"), ("ipp057.0"), ("ipp058.0"), ("ipp059.0"), ] ) + index += 1 + elif argv[index] == 'ippb': + nodes.extend( [ ("ippb00.0"), ("ippb00.1"), ("ippb00.2"), + ("ippb01.0"), ("ippb01.1"), ("ippb01.2"), + ("ippb02.0"), ("ippb02.1"), ("ippb02.2"), + ("ippb03.0"), ("ippb03.1"), ("ippb03.2"), ] ) + index += 1 + elif argv[index] == 'stsci': + nodes.extend( [ ("stsci00.0"), ("stsci00.1"), ("stsci00.2"), + ("stsci01.0"), ("stsci01.1"), ("stsci01.2"), + ("stsci02.0"), ("stsci02.1"), ("stsci02.2"), + ("stsci03.0"), ("stsci03.1"), ("stsci03.2"), + ("stsci04.0"), ("stsci04.1"), ("stsci04.2"), + ("stsci05.0"), ("stsci05.1"), ("stsci05.2"), + ("stsci06.0"), ("stsci06.1"), ("stsci06.2"), + ("stsci07.0"), ("stsci07.1"), ("stsci07.2"), + ("stsci08.0"), ("stsci08.1"), ("stsci08.2"), + ("stsci09.0"), ("stsci09.1"), ("stsci09.2"), + ("stsci10.0"), ("stsci10.1"), ("stsci10.2"), + ("stsci11.0"), ("stsci11.1"), ("stsci11.2"), + ("stsci12.0"), ("stsci12.1"), ("stsci12.2"), + ("stsci13.0"), ("stsci13.1"), ("stsci13.2"), + ("stsci14.0"), ("stsci14.1"), ("stsci14.2"), + ("stsci15.0"), ("stsci15.1"), ("stsci15.2"), + ("stsci16.0"), ("stsci16.1"), ("stsci16.2"), + ("stsci17.0"), ("stsci17.1"), ("stsci17.2"), + ("stsci18.0"), ("stsci18.1"), ("stsci18.2"), + ("stsci19.0"), ("stsci19.1"), ("stsci19.2"), ] ) + index += 1 + else: + node = argv[index] + if node.startswith('ippb'): + nodes.append((node+".0")) + nodes.append((node+".1")) + nodes.append((node+".2")) + elif node.startswith('stsci'): + nodes.append((node+".0")) + nodes.append((node+".1")) + nodes.append((node+".2")) + else: + nodes.append((node+".0")) + index += 1 + return nodes + +def get_snapshot(disk, dirname, fm_cursor, dms_cursor): + logger.debug("Getting snapshot of [%s]" % dirname) + select_query = 'SELECT role, stage, lifetime, SUM(size), COUNT(*), status FROM neb_%s_%s GROUP BY role, stage, lifetime, status' + insert_query = """INSERT INTO CurrentDiskUsage +SET node='%s', role='%s', stage='%s', dirname='%s', sizes=%d, counts=%d, lifetime='%s', last_update=NOW(), status='%s'""" + update_query = """UPDATE CurrentDiskUsage +SET sizes=%d, counts=%d, last_update=NOW() +WHERE node='%s' AND role='%s' AND stage='%s' AND dirname='%s' AND lifetime='%s' AND status='%s'""" + undefined_found = False + fm_cursor.execute(select_query % (disk, dirname)) + for (role, stage, lifetime, size, count, status) in fm_cursor.fetchall(): + if status == 'undefined': + undefined_found = True + if size < 0: + size=0 + try: + dms_cursor.execute(insert_query % (alias, + role, + stage, + dirname, + size, + count, + lifetime, + status) ) + except Exception, e: + dms_cursor.execute(update_query % (size, + count, + alias, + role, + stage, + dirname, + lifetime, + status) ) + if not undefined_found: + logger.debug('... No undefined found for %s' % dirname) + try: + logger.debug('... Trying to delete undefined entries') + dms_cursor.execute(delete_undefined_query % (alias) ) + except Exception, e: + logger.debug('... Trying to update') + dms_cursor.execute(update_query % (0, + 0, + alias, + 'undefined', + 'undefined', + dirname, + 'undefined', + 'undefined') ) + else: + logger.warn('... There are still undefined values for %s (dirname: %s)' % (alias, dirname)) + # Now delete all entries with status set as 'deleted' after the + # deleted entries have been measures + logger.debug('... Deleting deleted entries from %s (dirname: %s)' % (alias, dirname)) + stmt = "SELECT role, stage, lifetime, SUM(size), COUNT(*) FROM neb_%s_%s_delete GROUP BY role, stage, lifetime, status" + logger.debug(stmt % (disk, dirname)) + fm_cursor.execute(stmt % (disk, dirname)) + rows = fm_cursor.fetchall() + logger.debug("Got %d deleted entries" % len(rows)) + for (role, stage, lifetime, size, count) in rows: + try: + dms_cursor.execute(insert_query % (alias, + role, + stage, + dirname, + size, + count, + lifetime, + 'deleted') ) + except Exception, e: + dms_cursor.execute(update_query % (size, + count, + alias, + role, + stage, + dirname, + lifetime, + 'deleted') ) + try: + fm_cursor.execute("DELETE FROM neb_%s_%s_delete" % (disk, dirname)) + except _mysql_exceptions.OperationalError, e: + try: + # Try again?! + fm_cursor.execute("DELETE FROM neb_%s_%s_delete" % (disk, dirname)) + except _mysql_exceptions.OperationalError, e: + logger.warn("Couldn't delete 'deleted' entries from neb_%s_%s_delete on %s" % (dirname, disk, node)) + if not undefined_found: + logger.debug('... No undefined found for %s' % dirname) + delete_undefined_query = """DELETE FROM CurrentDiskUsage +WHERE node='%s' AND (status='undefined' OR role='undefined') +""" + try: + logger.debug('... Trying deleting undefined entries') + dms_cursor.execute(delete_undefined_query % (alias) ) + except Exception, e: + logger.debug('... Trying update') + dms_cursor.execute(update_query % (0, + 0, + alias, + 'undefined', + 'undefined', + dirname, + 'undefined', + 'undefined') ) + else: + logger.warn('... There are still undefined values for %s (dirname: %s)' % (alias, dirname)) + # Now move entries for this node which are older than the current date + logger.debug('... Moving old entries from CurrentDiskUsage for %s' % (alias)) + insert_select_statement = """INSERT IGNORE INTO DiskUsageTemporary(node, role, stage, sizes, counts, lifetime, dirname, status, last_update) +SELECT node, role, stage, sizes, counts, lifetime, dirname, status, last_update FROM CurrentDiskUsage +WHERE node='%s' AND last_update<'%s'""" + dms_cursor.execute(insert_select_statement % (node, snapshot_date)); + dms_cursor.execute("DELETE FROM CurrentDiskUsage WHERE node='%s' AND last_update<'%s'" % (alias, snapshot_date)); + +def update_daily_and_weekly(dms_cursor): + try: + stmt = 'CALL UpdateDailyByNode(\'%s\')' % alias + logger.info('... Calling UpdateDailyByNode: [%s]' % stmt) + dms_cursor.execute(stmt) + except _mysql_exceptions.OperationalError, e: + logger.warn('... MySQL Exception caught when calling UpdateDailyByNode(): ' + str(e) + '') + try: + logger.info('... Trying to guess if running UpdateWeeklyByNode makes sense') + today = datetime.datetime.today() + last_week = '%d-%d' % (today.year, today.isocalendar()[1]-1) + query = "SELECT COUNT(*), node, role, stage, lifetime, dirname, week, status FROM DiskUsageHistoryWeekly WHERE node = \'%s\' AND week = '%s' GROUP BY node, role, stage, lifetime, dirname, week, status" % (alias, last_week) + dms_cursor.execute(query) + row = dms_cursor.fetchone() + run_updateweekly = (row is None) + if run_updateweekly: + stmt = 'CALL UpdateWeeklyByNode(\'%s\')' % alias + logger.info('... Executing: %s' % stmt) + dms_cursor.execute(stmt) + else: + logger.info('... UpdateWeekly() call cancelled since last week entry already in database: %s' % last_week) + except _mysql_exceptions.OperationalError, e: + logger.warn('... MySQL Exception caught when calling UpdateWeekly(): ' + str(e) ) + +if __name__ == "__main__": + # Setup logging + logger = logging.getLogger() + formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', + '%Y-%m-%dT%H:%M:%S') + logging_output = logging.StreamHandler(sys.stderr) + logging_output.setFormatter(formatter) + logger.addHandler(logging_output) + logger.setLevel(logging.INFO) + + nodes = process_arguments(sys.argv) + letters = '0123456789abcdef' + + nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + if nodename != 'neverland' and nodename != 'ian' : + dms_db = MySQLdb.connect('ippc11', 'ippuser', 'ipp', + 'DiskMonitoringSummary') + else: # Test platform + logger.info("On [%s]" % nodename) + if nodename == 'ian': + root_passwd = '' + else: + root_passwd = 'root' + dms_db = MySQLdb.connect('localhost', 'root', root_passwd, + 'DiskMonitoringSummary') + dms_cursor = dms_db.cursor() + + for alias in nodes: + node, disk = alias.split('.') + logger.info('Getting snapshot of node: %s (%s)' % (node, alias)) + try: + if node != 'neverland' and node != 'ian' : + fm_db = MySQLdb.connect(node, 'ipp', 'ipp', 'DisksMonitoring') + else: + fm_db = MySQLdb.connect('localhost', 'root', root_passwd, + 'DisksMonitoring') + fm_cursor = fm_db.cursor() + snapshot_date = datetime.datetime.now() + for letter1 in letters: + logger.info('... Getting snapshot for nebulous subdirectories starting with %s (%s0, %s1, ...)' % (letter1, + letter1, + letter1)) + for letter2 in letters: + dirname = '%s%s' % (letter1, letter2) + get_snapshot(disk, dirname, fm_cursor, dms_cursor) + update_daily_and_weekly(dms_cursor) + fm_db.commit() + dms_db.commit() + except _mysql_exceptions.OperationalError, e: + logger.error(str(e)) Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table.py (revision 37068) @@ -0,0 +1,211 @@ +#!/usr/bin/env python + +import MySQLdb +import sys +import socket +import os +import datetime +import hashlib +try: + from lifetime_templates import Template +except ImportError: + sys.path.append('%s/%s/share/filemon/' % (os.environ['PSCONFDIR'], os.environ['PSCONFIG'])) + sys.stderr.write('ImportError') + for folder in sys.path: + sys.stderr.write('\t%s\n' % folder) + from lifetime_templates import Template + +def md5sum(filename): + f = open(filename, "rb") + content = f.read() + f.close() + sum = hashlib.md5() + sum.update(content) + return sum.hexdigest() + +class DbEntry: + verbose = False + def __init__(self, db_key, full_filename): + self.db_key = db_key + self.full_filename = full_filename + + def __str__(self): + if not DbEntry.verbose and len(self.db_key) > 10: + db_key = '%s...' % self.db_key[:10] + full_filename = '...%s' % self.full_filename[-10:] + else: + db_key = self.db_key + full_filename = self.full_filename + if self.role != 'deleted': + return '[%s]: %s (%s/%s/%s), %d, %s, %s' % (db_key, + full_filename, + self.stage, + self.role, + self.lifetime, + self.size, + self.md5sum, + self.last_check) + else: + return '[%s]: %s (%s)' % (db_key, + full_filename, + self.role) + + def check(self): + try: + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + stat_values = os.stat(self.full_filename) + self.size = stat_values.st_size + try: + self.md5sum = md5sum(self.full_filename) + self.status = 'ok' + except IOError: + self.status = 'error' + self.md5sum = None + except OSError: + self.status = 'deleted' + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + self.last_check = datetime.datetime.now() + + def update(self, table): + if self.status == 'deleted': + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + stmt = """UPDATE %s +SET status=\'%s\', + last_check=\'%s\', + role=\'%s\', + stage=\'%s\', + lifetime=\'%s\' WHERE filename=\'%s\';""" % ( table, + self.status, + self.last_check, + self.role, + self.stage, + self.lifetime, + self.db_key ) + return stmt + stmt = """ +UPDATE %s +SET + size=%d, + md5sum=\'%s\', + role=\'%s\', + lifetime=\'%s\', + last_check=\'%s\', + stage=\'%s\', + status=\'%s\' +WHERE filename=\'%s\'; +""" % ( table, + self.size, + self.md5sum, + self.role, + self.lifetime, + self.last_check, + self.stage, + self.status, + self.db_key ) + return stmt + +def usage(): + print + print 'Usage: %s [-v] [-l | -limit ] ' % sys.argv[0] + print ' Update the 100 last checked entries of FilesMonitoring _ table' + print ' where is the name of the host where this command is run and' + print ' is a nebulous directory name (range: 00 - ff)' + print + print ' * -h: display help' + print ' * -v: be verbose' + print ' * -l , -limit : process no more than entries (default 10)' + print + print ' e.g.: %s [-v] 00' % sys.argv[0] + print ' e.g.: %s [-v] 01' % sys.argv[0] + print ' e.g.: ..' + print ' e.g.: %s [-v] ff' % sys.argv[0] + print + +def process_options(argv): + limit = 10 + directory = None + argc = 1 + while argc < len(argv): + if sys.argv[argc] == '-h': + usage() + sys.exit(0) + elif sys.argv[argc] == '-v': + DbEntry.verbose = True + argc += 1 + elif sys.argv[argc] == '-limit' or sys.argv[argc] == '-l' : + #TODO: Make this POSIX + limit = int(sys.argv[argc+1]) + argc += 2 + elif sys.argv[argc] == '-disk': + disk = sys.argv[argc+1] + argc += 2 + else: + directory = sys.argv[argc] + argc += 1 + if directory is None: + usage() + sys.exit(1) + return limit, directory, disk + +def process_batch(filesMonitoring, disk, table_name, limit): + cursor = filesMonitoring.cursor() + query = 'SELECT filename, subdirname FROM %s ORDER BY last_check LIMIT %d' % ( table_name, + limit) + if DbEntry.verbose: + sys.stderr.write('%s\n' % query) + cursor.execute(query) + rows = cursor.fetchall() + cursor.close() + entries = [ ] + for row in rows: + filename = row[0] + subdirname = row[1] + full_filename = '/export/%s.%s/nebulous/%s/%s/%s' % ( hostname, + disk, + directory, + subdirname, + filename) + entries.append(DbEntry(filename, full_filename)) + for entry in entries: + entry.check() + count_updates = 0 + cursor = filesMonitoring.cursor() + for entry in entries: + query = entry.update(table_name) + count_updates += 1 + cursor.execute(query) + if DbEntry.verbose: + print query + cursor.close() + filesMonitoring.commit() + sys.stderr.write('\t# of updates: %d\n' % count_updates) + +####################################################################################### +if __name__ == '__main__': + if len(sys.argv) == 1: + usage() + sys.exit(1) + userlimit, directory, disk = process_options(sys.argv) + hostname = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + table_name = 'neb_%s_%s' % (disk, directory) + sys.stderr.write('Updating: %s %s %s\n' % (hostname, directory, table_name)) + if hostname == 'neverland': + filesMonitoring = MySQLdb.connect( 'localhost', + 'root', 'root', + 'DisksMonitoring' ) + else: + filesMonitoring = MySQLdb.connect( 'localhost', + 'root', '', + 'DisksMonitoring' ) + max_batch_size = 2500 + if userlimit > max_batch_size: + sys.stderr.write('Batching by sets of %d\n' % max_batch_size) + count = 0 + while count < userlimit: + sys.stderr.write('Processing %d out of %d\n' % (count, userlimit) ) + process_batch(filesMonitoring, disk, table_name, max_batch_size) + count += max_batch_size + else: + process_batch(filesMonitoring, disk, table_name, userlimit) + + filesMonitoring.close() Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table_undefined.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table_undefined.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/scripts/update_table_undefined.py (revision 37068) @@ -0,0 +1,215 @@ +#!/usr/bin/env python + +import MySQLdb +import sys +import socket +import os +import datetime +import hashlib +try: + from lifetime_templates import Template +except ImportError: + sys.path.append('%s/%s/share/filemon/' % (os.environ['PSCONFDIR'], os.environ['PSCONFIG'])) + sys.stderr.write('ImportError') + for folder in sys.path: + sys.stderr.write('\t%s\n' % folder) + from lifetime_templates import Template + +def md5sum(filename): + f = open(filename, "rb") + content = f.read() + f.close() + sum = hashlib.md5() + sum.update(content) + return sum.hexdigest() + +class DbEntry: + verbose = False + def __init__(self, db_key, full_filename): + self.db_key = db_key + self.full_filename = full_filename + + def __str__(self): + if not DbEntry.verbose and len(self.db_key) > 10: + db_key = '%s...' % self.db_key[:10] + full_filename = '...%s' % self.full_filename[-10:] + else: + db_key = self.db_key + full_filename = self.full_filename + if self.role != 'deleted': + return '[%s]: %s (%s/%s/%s), %d, %s, %s' % (db_key, + full_filename, + self.stage, + self.role, + self.lifetime, + self.size, + self.md5sum, + self.last_check) + else: + return '[%s]: %s (%s)' % (db_key, + full_filename, + self.role) + + def check(self): + try: + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + stat_values = os.stat(self.full_filename) + self.size = stat_values.st_size + try: + self.md5sum = md5sum(self.full_filename) + self.status = 'ok' + except IOError: + self.status = 'error' + self.md5sum = None + except OSError: + self.status = 'deleted' + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + self.last_check = datetime.datetime.now() + + def update(self, table): + if self.status == 'deleted': + self.stage, self.role, self.lifetime = Template.classify(self.db_key) + stmt = """UPDATE %s +SET status=\'%s\', + last_check=\'%s\', + role=\'%s\', + stage=\'%s\', + lifetime=\'%s\' WHERE filename=\'%s\';""" % ( table, + self.status, + self.last_check, + self.role, + self.stage, + self.lifetime, + self.db_key ) + return stmt + stmt = """ +UPDATE %s +SET + size=%d, + md5sum=\'%s\', + role=\'%s\', + lifetime=\'%s\', + last_check=\'%s\', + stage=\'%s\', + status=\'%s\' +WHERE filename=\'%s\'; +""" % ( table, + self.size, + self.md5sum, + self.role, + self.lifetime, + self.last_check, + self.stage, + self.status, + self.db_key ) + return stmt + +def usage(): + print + print 'Usage: %s [-v] [-l | -limit ] [-disk ] ' % sys.argv[0] + print ' Update the 10 last checked entries of FilesMonitoring _ table' + print ' where is the name of the host where this command is run and' + print ' is a nebulous directory name (range: 00 - ff)' + print + print ' * -h: display help' + print ' * -v: be verbose' + print ' * -l , -limit : process no more than entries (default 10)' + print ' * -disk : process disk /export/. (default: 0)' + print + print ' e.g.: %s [-v] 00' % sys.argv[0] + print ' e.g.: %s [-v] 01' % sys.argv[0] + print ' e.g.: ..' + print ' e.g.: %s [-v] ff' % sys.argv[0] + print + +def process_options(argv): + limit = 10 + directory = None + argc = 1 + disk = "0" + while argc < len(argv): + if sys.argv[argc] == '-h': + usage() + sys.exit(0) + elif sys.argv[argc] == '-v': + DbEntry.verbose = True + argc += 1 + elif sys.argv[argc] == '-limit' or sys.argv[argc] == '-l': + limit = int(sys.argv[argc+1]) + argc += 2 + elif sys.argv[argc] == '-disk': + disk = sys.argv[argc+1] + argc += 2 + else: + directory = sys.argv[argc] + argc += 1 + if directory is None: + usage() + sys.exit(1) + return limit, directory, disk + +def process_batch(filesMonitoring, disk, table_name, limit): + cursor = filesMonitoring.cursor() + query = 'SELECT filename, subdirname FROM %s WHERE status=\'undefined\' LIMIT %d' % ( table_name, + limit) + if DbEntry.verbose: + sys.stderr.write('%s\n' % query) + cursor.execute(query) + rows = cursor.fetchall() + cursor.close() + entries = [ ] + for row in rows: + filename = row[0] + subdirname = row[1] + full_filename = '/export/%s.%s/nebulous/%s/%s/%s' % ( hostname, + disk, + directory, + subdirname, + filename) + entries.append(DbEntry(filename, full_filename)) + for entry in entries: + entry.check() + count_updates = 0 + cursor = filesMonitoring.cursor() + for entry in entries: + query = entry.update(table_name) + count_updates += 1 + cursor.execute(query) + if DbEntry.verbose: + print query + cursor.close() + filesMonitoring.commit() + sys.stderr.write('\t# of updates: %d\n' % count_updates) + return count_updates + +####################################################################################### +if __name__ == '__main__': + if len(sys.argv) == 1: + usage() + sys.exit(1) + userlimit, directory, disk = process_options(sys.argv) + hostname = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + table_name = 'neb_%s_%s' % (disk, directory) + sys.stderr.write('Updating: %s %s %s\n' % (hostname, directory, table_name)) + if hostname == 'neverland': + filesMonitoring = MySQLdb.connect( 'localhost', + 'root', 'root', + 'DisksMonitoring' ) + else: +# filesMonitoring = MySQLdb.connect( 'localhost', +# 'ipp', 'ipp', +# 'DisksMonitoring' ) + filesMonitoring = MySQLdb.connect('localhost', 'root', '', 'DisksMonitoring') + max_batch_size = 2500 + if userlimit > max_batch_size: + sys.stderr.write('Batching by sets of %d\n' % max_batch_size) + count = 0 + while count < userlimit: + sys.stderr.write('\tProcessing: %d out of %d\n' % (count, userlimit) ) + if process_batch(filesMonitoring, disk, table_name, max_batch_size) == 0: + count = userlimit + count += max_batch_size + else: + process_batch(filesMonitoring, disk, table_name, userlimit) + + filesMonitoring.close() Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/Makefile.am (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/Makefile.am (revision 37068) @@ -0,0 +1,11 @@ +SUBDIRS = sql . + +pkgdatadir = $(datadir)/filemon + +pkgdata_DATA = \ + lifetime_templates.py \ + allStorageNodes \ + ippMhpccStorageNodes \ + ippComputingNodesNames \ + ippAllNodesNamesButDbs \ + ippAllNodesNames Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/allStorageNodes =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/allStorageNodes (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/allStorageNodes (revision 37068) @@ -0,0 +1,86 @@ +ippb00 +ippb01 +ippb02 +ippb03 +ipp004 +ipp005 +ipp006 +ipp007 +ipp008 +ipp009 +ipp010 +ipp011 +ipp012 +ipp013 +ipp014 +ipp015 +ipp016 +ipp017 +ipp018 +ipp019 +ipp020 +ipp021 +ipp023 +ipp024 +ipp025 +ipp026 +ipp027 +ipp028 +ipp029 +ipp030 +ipp031 +ipp032 +ipp033 +ipp034 +ipp035 +ipp036 +ipp037 +ipp038 +ipp039 +ipp040 +ipp041 +ipp042 +ipp043 +ipp044 +ipp045 +ipp046 +ipp047 +ipp048 +ipp049 +ipp050 +ipp051 +ipp052 +ipp053 +ipp054 +ipp055 +ipp056 +ipp057 +ipp058 +ipp059 +ipp060 +ipp061 +ipp062 +ipp063 +ipp064 +ipp065 +ipp066 +stsci00 +stsci01 +stsci02 +stsci03 +stsci04 +stsci05 +stsci06 +stsci07 +stsci08 +stsci09 +stsci10 +stsci11 +stsci12 +stsci13 +stsci14 +stsci15 +stsci16 +stsci17 +stsci18 +stsci19 Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNames =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNames (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNames (revision 37068) @@ -0,0 +1,155 @@ +ipp004 +ipp005 +ipp006 +ipp007 +ipp008 +ipp009 +ipp010 +ipp011 +ipp012 +ipp013 +ipp014 +ipp015 +ipp016 +ipp017 +ipp018 +ipp019 +ipp020 +ipp021 +ipp023 +ipp024 +ipp025 +ipp026 +ipp027 +ipp028 +ipp029 +ipp030 +ipp031 +ipp032 +ipp033 +ipp034 +ipp035 +ipp036 +ipp037 +ipp038 +ipp039 +ipp040 +ipp041 +ipp042 +ipp043 +ipp044 +ipp045 +ipp046 +ipp047 +ipp048 +ipp049 +ipp050 +ipp051 +ipp052 +ipp053 +ipp054 +ipp055 +ipp056 +ipp057 +ipp058 +ipp059 +ipp060 +ipp061 +ipp062 +ipp063 +ipp064 +ipp065 +ipp066 +ippc01 +ippc02 +ippc03 +ippc04 +ippc05 +ippc06 +ippc07 +ippc08 +ippc09 +ippc10 +ippc11 +ippc12 +ippc13 +ippc14 +ippc15 +ippc16 +ippc17 +ippc18 +ippc19 +ippc20 +ippc21 +ippc22 +ippc23 +ippc24 +ippc25 +ippc26 +ippc27 +ippc28 +ippc29 +ippc30 +ippc31 +ippc32 +ippc33 +ippc34 +ippc35 +ippc36 +ippc37 +ippc38 +ippc39 +ippc40 +ippc41 +ippc42 +ippc43 +ippc44 +ippc45 +ippc46 +ippc47 +ippc48 +ippc49 +ippc50 +ippc51 +ippc52 +ippc53 +ippc54 +ippc55 +ippc56 +ippc57 +ippc58 +ippc59 +ippc60 +ippc61 +ippc62 +ippc63 +stare00 +stare01 +stare02 +stare03 +stare04 +ippdb00 +ippdb01 +ippdb02 +ippdb03 +stsci00 +stsci01 +stsci02 +stsci03 +stsci04 +stsci05 +stsci06 +stsci07 +stsci08 +stsci09 +stsci10 +stsci11 +stsci12 +stsci13 +stsci14 +stsci15 +stsci16 +stsci17 +stsci18 +stsci19 + Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNamesButDbs =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNamesButDbs (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippAllNodesNamesButDbs (revision 37068) @@ -0,0 +1,150 @@ +ipp004 +ipp005 +ipp006 +ipp007 +ipp008 +ipp009 +ipp010 +ipp011 +ipp012 +ipp013 +ipp014 +ipp015 +ipp016 +ipp017 +ipp018 +ipp019 +ipp020 +ipp021 +ipp023 +ipp024 +ipp025 +ipp026 +ipp027 +ipp028 +ipp029 +ipp030 +ipp031 +ipp032 +ipp033 +ipp034 +ipp035 +ipp036 +ipp037 +ipp038 +ipp039 +ipp040 +ipp041 +ipp042 +ipp043 +ipp044 +ipp045 +ipp046 +ipp047 +ipp048 +ipp049 +ipp050 +ipp051 +ipp052 +ipp053 +ipp054 +ipp055 +ipp056 +ipp057 +ipp058 +ipp059 +ipp060 +ipp061 +ipp062 +ipp063 +ipp064 +ipp065 +ipp066 +ippc01 +ippc02 +ippc03 +ippc04 +ippc05 +ippc06 +ippc07 +ippc08 +ippc09 +ippc10 +ippc11 +ippc12 +ippc13 +ippc14 +ippc15 +ippc16 +ippc17 +ippc18 +ippc19 +ippc20 +ippc21 +ippc22 +ippc23 +ippc24 +ippc25 +ippc26 +ippc27 +ippc28 +ippc29 +ippc30 +ippc31 +ippc32 +ippc33 +ippc34 +ippc35 +ippc36 +ippc37 +ippc38 +ippc39 +ippc40 +ippc41 +ippc42 +ippc43 +ippc44 +ippc45 +ippc46 +ippc47 +ippc48 +ippc49 +ippc50 +ippc51 +ippc52 +ippc53 +ippc54 +ippc55 +ippc56 +ippc57 +ippc58 +ippc59 +ippc60 +ippc61 +ippc62 +ippc63 +stare00 +stare01 +stare02 +stare03 +stare04 +stsci00 +stsci01 +stsci02 +stsci03 +stsci04 +stsci05 +stsci06 +stsci07 +stsci08 +stsci09 +stsci10 +stsci11 +stsci12 +stsci13 +stsci14 +stsci15 +stsci16 +stsci17 +stsci18 +stsci19 Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippComputingNodesNames =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippComputingNodesNames (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippComputingNodesNames (revision 37068) @@ -0,0 +1,83 @@ +ipp004 +ipp005 +ipp006 +ipp007 +ipp008 +ipp009 +ipp010 +ipp011 +ipp012 +ipp013 +ipp014 +ipp015 +ipp016 +ipp017 +ipp018 +ipp019 +ipp020 +ipp021 +ipp023 +ipp024 +ipp025 +ipp026 +ipp027 +ipp028 +ipp029 +ipp030 +ipp031 +ipp032 +ipp033 +ipp034 +ipp035 +ipp036 +ipp037 +ipp038 +ipp039 +ipp040 +ipp041 +ipp042 +ipp043 +ipp044 +ipp045 +ipp046 +ipp047 +ipp048 +ipp049 +ipp050 +ipp051 +ipp052 +ipp053 +ippc01 +ippc02 +ippc03 +ippc04 +ippc05 +ippc06 +ippc07 +ippc08 +ippc09 +ippc10 +ippc11 +ippc12 +ippc13 +ippc14 +ippc15 +ippc16 +ippc17 +ippc18 +ippc19 +ippc20 +ippc21 +ippc22 +ippc23 +ippc24 +ippc25 +ippc26 +ippc27 +ippc28 +ippc29 +stare00 +stare01 +stare02 +stare03 +stare04 Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippMhpccStorageNodes =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippMhpccStorageNodes (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/ippMhpccStorageNodes (revision 37068) @@ -0,0 +1,62 @@ +ipp004 +ipp005 +ipp006 +ipp007 +ipp008 +ipp009 +ipp010 +ipp011 +ipp012 +ipp013 +ipp014 +ipp015 +ipp016 +ipp017 +ipp018 +ipp019 +ipp020 +ipp021 +ipp023 +ipp024 +ipp025 +ipp026 +ipp027 +ipp028 +ipp029 +ipp030 +ipp031 +ipp032 +ipp033 +ipp034 +ipp035 +ipp036 +ipp037 +ipp038 +ipp039 +ipp040 +ipp041 +ipp042 +ipp043 +ipp044 +ipp045 +ipp046 +ipp047 +ipp048 +ipp049 +ipp050 +ipp051 +ipp052 +ipp053 +ipp054 +ipp055 +ipp056 +ipp057 +ipp058 +ipp059 +ipp060 +ipp061 +ipp062 +ipp063 +ipp064 +ipp065 +ipp066 Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/lifetime_templates.py =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/lifetime_templates.py (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/lifetime_templates.py (revision 37068) @@ -0,0 +1,399 @@ +#!/usr/bin/env python + +import re + +class Stage: + UNKNOWN_STAGE = 'unknown' + STAGES = dict() + def __init__(self, name, str_patterns): + self.name = name + self.str_patterns = str_patterns + self.patterns = [ ] + for pattern in str_patterns: + self.patterns.append( re.compile(pattern) ) + @staticmethod + def add_stage(stage): + Stage.STAGES[stage.name] = stage + @staticmethod + def get_stage(filename): + for key, stage in Stage.STAGES.iteritems(): + for pattern in stage.patterns: + if pattern.match(filename): + return stage.name + return Stage.UNKNOWN_STAGE + +class Role: + UNKNOWN_ROLE = 'unknown' + ROLES = dict() + def __init__(self, name, str_patterns): + self.name = name + self.str_patterns = str_patterns + self.patterns = [ ] + for pattern in str_patterns: + self.patterns.append( re.compile(pattern) ) + @staticmethod + def add_role(role): + Role.ROLES[role.name] = role + @staticmethod + def get_role(filename): + default_role = Role.UNKNOWN_ROLE + for key, role in Role.ROLES.iteritems(): + for pattern in role.patterns: + if pattern.match(filename): + if role.name != 'fits': + return role.name + else: + default_role = role.name # i.e. 'fits' but there might be a 'better' match + return default_role + +class Lifetime: + UNKNOWN_LIFETIME = 'unknown' + LIFETIMES = dict() + def __init__(self, stage, role, duration): + if isinstance(stage, Stage): + _stage = stage + else: + _stage = Stage.STAGES[stage] + if isinstance(role, Role): + _role = role + else: + _role = Role.ROLES[role] + if duration not in ['permanent', 'raw', 'ephemeral']: + raise Exception("Lifetime duration must be one of: permanent', 'raw', 'ephemeral'") + self.stage = _stage + self.role = _role + self.duration = duration + @staticmethod + def add_lifetime(lifetime): + try: + subdict = Lifetime.LIFETIMES[lifetime.stage.name] + subdict[lifetime.role.name] = lifetime + except KeyError: + Lifetime.LIFETIMES[lifetime.stage.name] = dict() + Lifetime.add_lifetime(lifetime) + +def initialize_IppTemplates(): + # Define the stages (see http://docs.python.org/library/re.html) + # that will match the different stages + Stage.add_stage(Stage( 'cam', + [ '^.*\.cm\..*$', ] )) + Stage.add_stage(Stage( 'chip', + [ '^.*\.ch\..*$', ] )) + Stage.add_stage(Stage( 'destreak', + [ '^.*destreak.*$', ] )) + Stage.add_stage(Stage( 'detrend', + [ '^.*\.det.*$', ] )) + Stage.add_stage(Stage( 'diff', + [ '^.*\.dif\..*$', ] )) + Stage.add_stage(Stage( 'dist', + [ '^.*dist.*$', ] )) + Stage.add_stage(Stage( 'magic', + [ '^.*\.mgc\..*$', ] )) + Stage.add_stage(Stage( 'raw', + [ '^.*\.ota\d\d\..*$', + '^.*\.chip01\..*$', ] )) + Stage.add_stage(Stage( 'stack', + [ '^.*\.stk\..*$', ] )) + Stage.add_stage(Stage( 'warp', + [ '^.*\.wrp\..*$', ] )) + Stage.add_stage(Stage( 'skycal', + [ '^.*\.skycal\..*$', ] )) + Stage.add_stage(Stage( 'staticsky', + [ '^.*\.sky\..*$', ] )) + + #################### Roles and associated lifetimes + Role.add_role(Role( 'b1fits', + [ '^.*\.b1\.fits$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b1fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b1fits', 'permanent') ) + + Role.add_role(Role( 'b1jpg', + [ '^.*\.b1\.jpeg$', + '^.*\.b1\.jpg$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b1jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b1jpg', 'permanent') ) + + Role.add_role(Role( 'b2fits', + [ '^.*\.b2\.fits$', ] )) + Lifetime.add_lifetime( Lifetime('chip', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b2fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b2fits', 'permanent') ) + + Role.add_role(Role( 'b2jpg', + [ '^.*\.b2\.jpeg$', + '^.*\.b2\.jpg$', ] )) + Lifetime.add_lifetime( Lifetime('cam', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('detrend', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('diff', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('stack', 'b2jpg', 'permanent') ) + Lifetime.add_lifetime( Lifetime('warp', 'b2jpg', 'permanent') ) + + Role.add_role(Role( 'bttable', + [ '^.*\.burn\.tbl$', ] )) + Lifetime.add_lifetime( Lifetime('raw', 'bttable', 'raw') ) + + Role.add_role(Role( 'bundle', + [ '^.*\.tgz$', ] )) + Lifetime.add_lifetime( Lifetime('cam', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('chip', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('diff', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('dist', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('raw', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('stack', 'bundle', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime('warp', 'bundle', 'ephemeral') ) + + Role.add_role(Role( 'catalog', + [ '^.*\.smf$', + '^.*\.cmf$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'catalog', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'catalog', 'permanent') ) + + Role.add_role(Role( 'compfits', + [ '^.*\.fits\.fz$', ] )) + Lifetime.add_lifetime( Lifetime( 'raw', 'compfits', 'raw') ) + + Role.add_role(Role( 'kernel', + [ '^.*kernel$', ] )) + Lifetime.add_lifetime( Lifetime( 'diff', 'kernel', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'kernel', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'kernel', 'permanent') ) + + Role.add_role(Role( 'log', + [ '^.*\.log$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'log', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'log', 'permanent') ) + + Role.add_role(Role( 'logupdate', + [ '^.*\.log\.update$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'logupdate', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'logupdate', 'permanent') ) + + Role.add_role(Role( 'mask', + [ '^.*\.mk\.fits$', + '^.*\.mask\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'mask', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'mask', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'mask', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'mask', 'ephemeral') ) + + Role.add_role(Role( 'mdc', + [ '^.*\.mdc$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'dist', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'mdc', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'mdc', 'permanent') ) + + Role.add_role(Role( 'mdl', + [ '^.*\.mdl\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'mdl', 'ephemeral') ) + + Role.add_role(Role( 'miscfits', + [ '^.*\.exp\.fits$', + '^.*\.num\.fits$', + '^.*\.expwt\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'stack', 'miscfits', 'permanent') ) + + Role.add_role(Role( 'png', + [ '^.*\.png$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'png', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'png', 'ephemeral') ) + + Role.add_role(Role( 'psf', + [ '^.*\.psf$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'psf', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'psf', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'psf', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'psf', 'ephemeral') ) + + Role.add_role(Role( 'ptn', + [ '^.*\.ptn$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'ptn', 'ephemeral') ) + + Role.add_role(Role( 'skycell', + [ '^.*\.skycell$', ] )) + Lifetime.add_lifetime( Lifetime( 'warp', 'skycell', 'ephemeral') ) + + Role.add_role(Role( 'stats', + [ '^.*\.stats$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'stats', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'stats', 'permanent') ) + + Role.add_role(Role( 'trace', + [ '^.*\.trace$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'trace', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'trace', 'permanent') ) + + Role.add_role(Role( 'weight', + [ '^.*\.wt\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'chip', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'weight', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'weight', 'ephemeral') ) + + Role.add_role(Role( 'fits', + [ '^.*\.fits$', ] )) + Lifetime.add_lifetime( Lifetime( 'cam', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'chip', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'destreak', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'detrend', 'fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'diff', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'magic', 'fits', 'ephemeral') ) + Lifetime.add_lifetime( Lifetime( 'raw', 'fits', 'raw') ) + Lifetime.add_lifetime( Lifetime( 'stack', 'fits', 'permanent') ) + Lifetime.add_lifetime( Lifetime( 'warp', 'fits', 'ephemeral') ) + +###################################################################################### +# +# DON'T TOUCH ANYTHING AFTER THIS LINE UNLESS YOU KNOW WHAT YOU'RE DOING +# +###################################################################################### +class Template: + """ + >>> print Template.classify('1110322.gpc1:20080610:o4627g0006o.17394:o4627g0006o.17394.reg.ota03.log') + ('raw', 'log', 'permanent') + >>> print Template.classify('165757062.gpc1:SweetSpot.nt:2010:02:22:o5249g0554o.139304:SR_o5249g0554o.139304.cm.49578.XY03.mk.fits') + ('cam', 'mask', 'permanent') + >>> print Template.classify('165578038.gpc1:ThreePi.nt:2010:02:23:o5250g0408o.139800:o5250g0408o.139800.ch.62962.XY54.stats') + ('chip', 'stats', 'permanent') + >>> print Template.classify('173285968.gpc1:IPP-MOPS:IPP-MOPS.15546.neg.mops') + ('unknown', 'unknown', 'unknown') + >>> print Template.classify('191293758.gpc1:20100308:o5263g0051f.145823:o5263g0051f.145823.reg.log') + ('unknown', 'log', 'unknown') + >>> print Template.classify('216601960.gpc1:ThreePi.nt:2010:02:22:o5249g0213o.138965:o5249g0213o.138965.wrp.34664.skycell.2038.080.trace.update') + ('warp', 'unknown', 'unknown') + >>> print Template.classify('500266344.gpc1:20101010:o5479g0217o.237556:o5479g0217o.237556.mgc.87689.original.png') + ('magic', 'png', 'ephemeral') + """ + initialize_IppTemplates() + + def __init__(self): + pass + + @staticmethod + def classify(filename): + # We have subclasses of FITS files: try to detect that first + stage = Stage.get_stage(filename) + role = Role.get_role(filename) + try: + return (stage, role, Lifetime.LIFETIMES[stage][role].duration) + except KeyError: + return (stage, role, Lifetime.UNKNOWN_LIFETIME) + + +def usage(progname): + return """ +: %s [test|wiki|] + + Provides the IPP files classification, that is (stage, role, lifetime duration) where: + * stage is the stage name when the file has been generated (chip, cam...) + * role is the role of the file (fits, log, mask...) + * its lifetime duration (raw, permanent, ephemeral) + Any undefined stage, role, or lifetime duration will be displayed as unknown. + + The complete list of stages is: + %s + + The complete list of roles is: + %s + +Arguments: + + test: Run the unit tests + + wiki: Generate the table for the "Cluster Storage Notes" IPP wikipage + See http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Cluster_Storage_Notes + + : Try to classify the , e.g.: + %s 218164796.gpc1:20100327:o5282g0065o.151926:o5282g0065o.151926.reg.log + should display: + ('unknown', 'log', 'unknown') + +""" % (progname, Stage.STAGES.keys(), Role.ROLES.keys(), progname) + +def generate_wiki_documentation(date, command_line, cwd): + # We start with an empty cell and display the stage names + wiki = """This table was generated on %s by the command {{{%s}}} from {{{%s}}}. Do not update it manually. + +Raw (R), Ephemeral (E), Should not be made (-), and Permanent Products (P) Files. +""" % (date, command_line, cwd) + wiki = "%s\n%s" % (wiki, "|| || Stage ") + stage_patterns_line = "|| Role || Pattern " + for stage in sorted(Stage.STAGES.keys()): + wiki = "%s || %10s" % (wiki, stage) + wiki_patterns = ('%s' % (Stage.STAGES[stage].str_patterns)) + stage_patterns_line = "%s || {{{%10s}}}" % (stage_patterns_line, wiki_patterns) + wiki = "%s||\n%s||\n" % (wiki, stage_patterns_line) + # Each line contains: the role name, then the role patterns, and then, + # for each stage, the lifetime duration + for role in sorted(Role.ROLES.keys()): + wiki = "%s|| %10s" % (wiki, role) + wiki = "%s|| {{{%10s}}}" % (wiki, Role.ROLES[role].str_patterns) + for stage in sorted(Stage.STAGES.keys()): + try: + wiki = "%s || %s" % (wiki, Lifetime.LIFETIMES[stage][role].duration[0].upper()) + except KeyError: + wiki = "%s || -" % (wiki) + wiki = "%s ||\n" % wiki + print wiki + +if __name__ == '__main__': + import sys + if len(sys.argv) != 2: + sys.stderr.write(usage(sys.argv[0])) + sys.exit(1) + if sys.argv[1] == 'test': + import doctest + doctest.testmod() + elif sys.argv[1] == 'wiki': + import datetime + import os + generate_wiki_documentation(str(datetime.datetime.now()), + " ".join(sys.argv), + os.getcwd()) + else: + print Template.classify(sys.argv[1]) Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/CreateTablesTemplate.sql (revision 37068) @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS Files_@HOST@_@DIRECTORY@; + +CREATE TABLE Files_@HOST@_@DIRECTORY@ ( + filename CHAR(255) PRIMARY KEY, + neb_ins_id BIGINT(20) NOT NULL, + subdirname CHAR(3) NOT NULL, + INDEX(subdirname), + nebkey CHAR(255), + INDEX(nebkey), + size INT DEFAULT -1, + md5sum CHAR(33) DEFAULT NULL, + role ENUM('undefined', 'unknown', 'deleted', 'b1fits', 'b1jpg', 'b2fits', + 'b2jpg', 'bttable', 'bundle', 'catalog', + 'fits', 'compfits', 'kernel', 'log', 'logupdate', 'mask', 'mdc', 'mdl', + 'miscfits', 'png', 'psf', 'ptn', 'skycell', 'stats', + 'trace', 'weight', 'other') + NOT NULL DEFAULT 'undefined', + INDEX(role), + stage ENUM( 'undefined', 'unknown', 'raw', 'chip', 'cam', 'fake', 'detrend', + 'warp', 'stack', 'diff', 'magic', 'destreak', + 'dist', 'pub', 'staticsky', 'chipbackground', + 'other' ) + NOT NULL DEFAULT 'undefined', + INDEX(stage), + lifetime ENUM('undefined', 'permanent', 'ephemeral', 'raw', 'unknown') NOT NULL DEFAULT 'undefined', + creation_date TIMESTAMP DEFAULT 0, + last_check TIMESTAMP DEFAULT 0 +); Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/Makefile.am (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/Makefile.am (revision 37068) @@ -0,0 +1,5 @@ +pkgdatadir = $(datadir)/filemon/sql/ + +pkgdata_DATA = \ + CreateTablesTemplate.sql \ + insert_template.sql Index: branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/insert_template.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/insert_template.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/FilesMonitoring/share/sql/insert_template.sql (revision 37068) @@ -0,0 +1,7 @@ +INSERT INTO Files_%s_%s ( + filename, + neb_ins_id, + subdirname, + nebkey, + creation_date) +VALUES ('%s', %s, '%s', '%s', NOW()); Index: branches/eam_branches/ipp-ops-20130712/ippm/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/Makefile.am (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/Makefile.am (revision 37068) @@ -0,0 +1,9 @@ +SUBDIRS = \ + FilesMonitoring \ + FilesMonitoring/scripts \ + FilesMonitoring/share \ + FilesMonitoring/share/sql + + + + Index: branches/eam_branches/ipp-ops-20130712/ippm/autogen.sh =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/autogen.sh (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/autogen.sh (revision 37068) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippm +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOLIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: branches/eam_branches/ipp-ops-20130712/ippm/configure.ac =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippm/configure.ac (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippm/configure.ac (revision 37068) @@ -0,0 +1,17 @@ +AC_PREREQ(2.59) + +AC_INIT([ippM], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([autogen.sh]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL +AC_CONFIG_FILES([ + Makefile + FilesMonitoring/Makefile + FilesMonitoring/scripts/Makefile + FilesMonitoring/share/Makefile + FilesMonitoring/share/sql/Makefile +]) +AC_OUTPUT