Index: trunk/tools/log_parsing/nebulous_activity.py =================================================================== --- trunk/tools/log_parsing/nebulous_activity.py (revision 30662) +++ trunk/tools/log_parsing/nebulous_activity.py (revision 30662) @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +# +# Plot the apache error activity versus the nebulous activity +# + +import sys +import os +import gzip +import parse_apache_log + +def getFilenamesByPrefix(directoryName, prefix): + """ + Get a list of file names in directory starting by prefix + """ + filenames = [] + for entry in os.listdir(directoryName): + fullpath = directoryName + os.sep + entry + if entry.startswith(prefix): + if not os.path.isdir(fullpath): + filenames.append(fullpath) + return filenames + +def processAccessFile(filename): + """ + Process a Nebulous apache access_log file (that can be gzipped or not) + """ + isGZipFile = False + try: + inFile = gzip.open(filename) + inFile.read(1) + inFile.close() + isGZipFile = True + except IOError: + pass + if isGZipFile: + inFile = gzip.open(filename) + else: + inFile = open(filename) + sys.stderr.write('File %s is %sa gzip file\n' % (filename, + '' if isGZipFile else 'not ')) + counts = parse_apache_log.process(inFile) + #parse_apache_log.show_results(counts, sys.stdout) + return counts + +def processErrorFile(filename): + """ + Process a Nebulous apache error_log file (that can be gzipped or not) + """ + isGZipFile = False + try: + inFile = gzip.open(filename) + inFile.read(1) + inFile.close() + isGZipFile = True + except IOError: + pass + if isGZipFile: + inFile = gzip.open(filename) + else: + inFile = open(filename) + sys.stderr.write('File %s is %sa gzip file\n' % (filename, + '' if isGZipFile else 'not ')) + counts = parse_apache_log.processErrorLog(inFile) + #parse_apache_log.show_errors(counts, sys.stdout) + return counts + +if __name__ == '__main__': + """ + Usage: python sys.argv[0] + + Application log is written to /dev/stderr + Results are written to /dev/stdout + """ + # All logs are apache2 logs and can be found in one directory + # which happens to be the first script argument + logDirectoryName = sys.argv[1] + + sys.stderr.write('### Building access log filenames list\n') + accessFilenames = getFilenamesByPrefix(logDirectoryName, + 'access_log') + sys.stderr.write(' %d access_log files\n' % (len(accessFilenames))) + sys.stderr.write('### Building error log filenames list\n') + errorFilenames = getFilenamesByPrefix(logDirectoryName, + 'error_log') + sys.stderr.write(' %d error_log files\n' % (len(errorFilenames))) + accessCounts = dict() + for accessFilename in accessFilenames: + accessCounts.update(processAccessFile(accessFilename)) + errorCounts = dict() + for errorFilename in errorFilenames: + errorCounts.update(processErrorFile(errorFilename)) + + # Now show results + keys = set() + keys.update(accessCounts.keys()) + keys.update(errorCounts.keys()) + for key in keys: + try: + nebulousAccessCount = 0 + for index in range(5): + nebulousAccessCount += accessCounts[key][index] + createCount = accessCounts[key][0] + findInsCount = accessCounts[key][1] + deleteCount = accessCounts[key][2] + failStatCount = accessCounts[key][3] + statCount = accessCounts[key][4] + otherCount = accessCounts[key][5] + unknownCount = accessCounts[key][6] + except KeyError: + nebulousAccessCount = -1 + createCount = -1 + findInsCount = -1 + deleteCount = -1 + failStatCount = -1 + statCount = -1 + otherCount = -1 + unknownCount = -1 + try: + errorCount = errorCounts[key] + except KeyError: + errorCount = -1 + sys.stdout.write('%s %d %d %d %d %d %d %d %d %d\n' % (key, nebulousAccessCount, errorCount, + createCount, findInsCount, deleteCount, + failStatCount, statCount, otherCount, + unknownCount)) Index: trunk/tools/log_parsing/parse_apache_log.py =================================================================== --- trunk/tools/log_parsing/parse_apache_log.py (revision 30651) +++ trunk/tools/log_parsing/parse_apache_log.py (revision 30662) @@ -4,10 +4,16 @@ import re import datetime +import fileinput nebulousPattern = re.compile('POST /nebulous HTTP/1.1') +errorPattern = re.compile('exit signal Segmentation fault') + # Resolution: 10 minutes of 60 seconds of 1000000 microseconds datetime.resolution = 10*60*1000000 def message(size): + """ + Guess the message type according to the message size + """ if size == 488: # delete == 488 @@ -34,4 +40,7 @@ def show_results(counts, output=sys.stderr): + """ + Display results + """ output.write('# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb\n') for count in sorted(counts.iterkeys()): @@ -45,12 +54,18 @@ counts[count][6])) -if __name__ == '__main__': +def process(linesInFile = None): + """ + Process Nebulous entries in apache2 log file + """ counts = dict() - linesCount = 1 - for line in sys.stdin: - if linesCount % 10000 == 0: - sys.stderr.write('Analyzed lines: %d\n' % linesCount) - if linesCount % 100000 == 0: + if linesInFile == None: + input = fileinput.input() + else: + input = linesInFile + for line in input: + if linesCount % 100000 == 0: + sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount) + if linesCount % 500000 == 0: show_results(counts) linesCount += 1 @@ -85,4 +100,85 @@ counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0] counts[dt.isoformat()][6] += 1 + return counts - show_results(counts, sys.stdout) +def show_errors(counts, output=sys.stderr): + """ + Display error results + """ + output.write('# Epoch | Errors\n') + for count in sorted(counts.iterkeys()): + output.write('%19s | %6d\n' % (count, counts[count])) + +def processErrorLog(linesInFile = None): + """ + Process errors in apache2 error log file + """ + counts = dict() + linesCount = 1 + if linesInFile == None: + input = fileinput.input() + else: + input = linesInFile + for line in input: + if linesCount % 100000 == 0: + sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount) + if linesCount % 500000 == 0: + show_errors(counts) + linesCount += 1 + if errorPattern.search(line): + elements = line.split(']') + date = elements[0] + date = date[1:] + # print '[%s]' % (date) + dt = datetime.datetime.strptime(date, + '%a %b %d %H:%M:%S %Y') + dt = dt.replace(minute = int(dt.minute/10)*10, + second = 0) + # print dt.isoformat() + try: + counts[dt.isoformat()] += 1 + except KeyError: + counts[dt.isoformat()] = 1 + return counts + +def usage(): + sys.stderr.write(main.__doc__) + +def main(arguments): + """ +Usage: parse_apache_log [-a ] [-e ] +""" + mainName = arguments.pop() + if len(arguments) == 0: + sys.stderr.write('Reading from /dev/stdin\n') + counts = process() + show_results(counts, sys.stdout) + elif len(arguments) % 2 != 0: + sys.stderr.write('Arguments length: %d\n' % (len(arguments))) + usage() + sys.exit(1) + else: + accessFilenames = [] + errorFilenames = [] + while len(arguments) != 0: + flag = arguments.pop() + filename = arguments.pop() + if flag == '-a': + accessFilenames.append(filename) + elif flag == '-e': + errorFilenames.append(filename) + else: + sys.stderr.write('Unknown flag: [%s]' % (flag)) + usage() + sys.exit(1) + for accessFilename in accessFilenames: + sys.stderr.write('Processing: [%s]' % (accessFilename)) + inFile = open(accessFilename) + counts = process(inFile) + show_results(counts, sys.stdout) + for errorFilename in errorFilenames: + counts = processErrorLog(errorFilename) + show_errors(counts, sys.stdout) + +if __name__ == '__main__': + main(sys.argv)