Index: trunk/tools/log_parsing/README
===================================================================
--- trunk/tools/log_parsing/README	(revision 30522)
+++ trunk/tools/log_parsing/README	(revision 30522)
@@ -0,0 +1,16 @@
+Nebulous apache log analysis.
+
+Both scripts (parse_apache_log.pl and parse_apache_log.py) do the same thing: parse an apache log file
+and give some stat about the requests. The Python script is about 5 times faster
+
+Usually, you will use it like (bash):
+	cat access-log-20100204 | ./parse_apache_log.py > status.20100204 2> status.20100204.errors
+or:
+	zcat access-log-20100204.gz | ./parse_apache_log.py > status.20100204 2> status.20100204.errors
+
+You can 'tail -f status.20100204.errors' to see the progression.
+
+Unit tests can be run using runUnitTests.sh
+
+Note: parsing can take a LONG time. For instance, on ippc01, parsing 100 million lines requires about 1 hour.
+
Index: trunk/tools/log_parsing/data/unittests/expectations/log.create.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.create.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.create.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	create	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.create.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.create.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.create.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      1 |      0 |      0 |      0 |      0 |      0 |      0
Index: trunk/tools/log_parsing/data/unittests/expectations/log.delete.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.delete.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.delete.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	delete	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.delete.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.delete.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.delete.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      0 |      1 |      0 |      0 |      0 |      0
Index: trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	failure_stat	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.failure_stat.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      0 |      0 |      1 |      0 |      0 |      0
Index: trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	find_instance	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.find_instance.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      1 |      0 |      0 |      0 |      0 |      0
Index: trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl	(revision 30522)
@@ -0,0 +1,1 @@
+#epoch	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      0 |      0 |      0 |      0 |      0 |      1
Index: trunk/tools/log_parsing/data/unittests/expectations/log.stat.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.stat.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.stat.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	stat	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.stat.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.stat.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.stat.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      0 |      0 |      0 |      1 |      0 |      0
Index: trunk/tools/log_parsing/data/unittests/expectations/log.unknown.pl
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.unknown.pl	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.unknown.pl	(revision 30522)
@@ -0,0 +1,2 @@
+#epoch	unknown	
+1282187400	1	
Index: trunk/tools/log_parsing/data/unittests/expectations/log.unknown.py
===================================================================
--- trunk/tools/log_parsing/data/unittests/expectations/log.unknown.py	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/expectations/log.unknown.py	(revision 30522)
@@ -0,0 +1,2 @@
+#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb
+2010-08-19T03:10:00 |      0 |      0 |      0 |      0 |      0 |      1 |      0
Index: trunk/tools/log_parsing/data/unittests/input/log.create
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.create	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.create	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.32 - - [19/Aug/2010:03:12:45 -1000] "POST /nebulous HTTP/1.1" 200 631
Index: trunk/tools/log_parsing/data/unittests/input/log.delete
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.delete	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.delete	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.68 - - [19/Aug/2010:03:12:27 -1000] "POST /nebulous HTTP/1.1" 200 488
Index: trunk/tools/log_parsing/data/unittests/input/log.failure_stat
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.failure_stat	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.failure_stat	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.26 - - [19/Aug/2010:03:12:27 -1000] "POST /nebulous HTTP/1.1" 200 2092
Index: trunk/tools/log_parsing/data/unittests/input/log.find_instance
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.find_instance	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.find_instance	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.67 - - [19/Aug/2010:03:12:45 -1000] "POST /nebulous HTTP/1.1" 200 655
Index: trunk/tools/log_parsing/data/unittests/input/log.non_nebulous
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.non_nebulous	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.non_nebulous	(revision 30522)
@@ -0,0 +1,1 @@
+::1 - - [19/Aug/2010:03:12:27 -1000] "OPTIONS * HTTP/1.0" 200 -
Index: trunk/tools/log_parsing/data/unittests/input/log.stat
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.stat	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.stat	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.72 - - [19/Aug/2010:03:12:49 -1000] "POST /nebulous HTTP/1.1" 200 859
Index: trunk/tools/log_parsing/data/unittests/input/log.unknown
===================================================================
--- trunk/tools/log_parsing/data/unittests/input/log.unknown	(revision 30522)
+++ trunk/tools/log_parsing/data/unittests/input/log.unknown	(revision 30522)
@@ -0,0 +1,1 @@
+166.122.172.47 - - [19/Aug/2010:03:13:47 -1000] "POST /nebulous HTTP/1.1" 200 420
Index: trunk/tools/log_parsing/parse_apache_log.pl
===================================================================
--- trunk/tools/log_parsing/parse_apache_log.pl	(revision 30522)
+++ trunk/tools/log_parsing/parse_apache_log.pl	(revision 30522)
@@ -0,0 +1,81 @@
+#!/usr/bin/env perl
+
+use warnings;
+use DateTime;
+
+%months = ('Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4,
+	   'May' => 5, 'Jun' => 6, 'Jul' => 7, 'Aug' => 8,
+	   'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12);
+
+my ($ip,$day,$month,$year,$hour,$min,$sec,$bytes);
+
+while (<>) {
+    chomp;
+#	print STDERR "$i\n";
+    if ($_ =~ /POST \/nebulous HTTP\/1.1/) { # This is a nebulous query
+	s/(\d+?\.\d+?\.\d+?\.\d+?) - - \[(\d\d)\/(\w\w\w)\/(\d\d\d\d)\:(\d\d)\:(\d\d)\:(\d\d).*\] "POST \/nebulous HTTP\/1.1" 200 (\d+?)/$1 $2 $3 $4 $5 $6 $7 $8/;
+	($ip,$day,$month,$year,$hour,$min,$sec,$bytes) = split / /;
+	$month = $months{$month};
+	$type  = size_to_type($bytes);
+	count_it($type,$year,$month,$day,$hour,$min,$sec);
+    }
+}
+
+print "#epoch\t";
+foreach $tag (keys %tags) {
+    print "$tag\t";
+}
+print "\n";
+foreach $time (sort {$a <=> $b} (keys %data)) {
+    print "$time\t";
+    foreach $tag (sort (keys %tags)) {
+        unless (exists($data{$time}{$tag})) {
+            $data{$time}{$tag} = 0;
+        }
+        print "$data{$time}{$tag}\t";
+    }
+    print "\n";
+}
+
+sub size_to_type {
+    my $bytes = shift;
+    if ($bytes == 488) { # == 488
+	return("delete");
+    }
+    elsif ($bytes > 1800) { # 1577 + 3*ext_key for stat, 1597 + 3*ext_key for delete
+	return("failure_stat");
+    }
+    elsif ($bytes > 850) {  # 819 + ext_key for stat
+	return("stat");
+    }
+    elsif ($bytes > 650) {  # 600 + ext_key
+	return("find_instance");
+    }
+    elsif ($bytes > 575) { # 526 + ext_key
+	return("create");
+    }
+#    else {
+#	print "$bytes\n";
+#    }
+    return("unknown");
+}
+
+sub count_it {
+    my $tag = shift;
+    my $year = shift;
+    my $month = shift;
+    my $day = shift;
+    my $hour = shift;
+    my $min = shift;
+    my $sec = shift;
+    
+    # Truncate to hours;
+    $sec = 0;
+    $min = int($min / 10) * 10;
+#    $min = 0;
+    my $dt = DateTime->new( year => $year, month => $month, day => $day, hour => $hour, minute => $min, second => $sec);
+    my $time = $dt->epoch;
+
+    $data{$time}{$tag} += 1;
+    $tags{$tag} = 1;
+}
Index: trunk/tools/log_parsing/parse_apache_log.py
===================================================================
--- trunk/tools/log_parsing/parse_apache_log.py	(revision 30522)
+++ trunk/tools/log_parsing/parse_apache_log.py	(revision 30522)
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+import sys
+import re
+import datetime
+
+nebulousPattern = re.compile('POST /nebulous HTTP/1.1')
+# Resolution: 10 minutes of 60 seconds of 1000000 microseconds
+datetime.resolution = 10*60*1000000
+
+def message(size):
+    if size == 488:
+        # delete == 488
+        return 2
+    elif size > 1800:
+        # failure_stat
+        # 1577 + 3*ext_key for stat, 1597 + 3*ext_key for delete
+        return 3
+    elif size > 850:
+        # stat
+        # 819 + ext_key for stat
+        return 4
+    elif size > 650:
+        # find_instance
+        # 600 + ext_key
+        return 1
+    elif size > 575:
+        # create
+        # 526 + ext_key
+        return 0
+    else:
+        # unknown
+        return 5
+
+def show_results(counts, output=sys.stderr):
+    output.write('#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb\n')
+    for count in sorted(counts.iterkeys()):
+        output.write('%19s | %6d | %6d | %6d | %6d | %6d | %6d | %6d\n' % (count, 
+                                                                           counts[count][0],
+                                                                           counts[count][1],
+                                                                           counts[count][2],
+                                                                           counts[count][3],
+                                                                           counts[count][4],
+                                                                           counts[count][5],
+                                                                           counts[count][6]))
+
+if __name__ == '__main__':
+    counts = dict()
+
+    linesCount = 1
+    for line in sys.stdin:
+        if linesCount % 10000 == 0:
+            sys.stderr.write('Analyzed lines: %d\n' % linesCount)
+            if linesCount % 100000 == 0:
+                show_results(counts)
+        linesCount += 1
+        if nebulousPattern.search(line):
+            elements = line.split(' ')
+            date = elements[3]
+            date = date[1:]
+            size = elements[-1]
+            # print '[%s] %s' % (date, size)
+            dt = datetime.datetime.strptime(date, 
+                                            '%d/%b/%Y:%H:%M:%S')
+            dt = dt.replace(minute = int(dt.minute/10)*10,
+                            second = 0)
+            try:
+                counts[dt.isoformat()][message(int(size))] += 1
+            except KeyError:
+                counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0]
+                counts[dt.isoformat()][message(int(size))] += 1
+        else:
+            elements = line.split(' ')
+            date = elements[3]
+            date = date[1:]
+            size = elements[-1]
+            # print '[%s] %s' % (date, size)
+            dt = datetime.datetime.strptime(date, 
+                                            '%d/%b/%Y:%H:%M:%S')
+            dt = dt.replace(minute = int(dt.minute/10)*10,
+                            second = 0)
+            try:
+                counts[dt.isoformat()][6] += 1
+            except KeyError:
+                counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0]
+                counts[dt.isoformat()][6] += 1
+
+    show_results(counts, sys.stdout)
Index: trunk/tools/log_parsing/runUnitTests.sh
===================================================================
--- trunk/tools/log_parsing/runUnitTests.sh	(revision 30522)
+++ trunk/tools/log_parsing/runUnitTests.sh	(revision 30522)
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# 1) Input is processed from INPUT_DIRECTORY
+# 2) Output is saved to ACTUAL_DIRECTORY
+# 3) Output is compared to expectation from EXPECTATION_DIRECTORY
+INPUT_DIRECTORY=data/unittests/input
+ACTUAL_DIRECTORY=data/unittests/actual
+EXPECTATION_DIRECTORY=data/unittests/expectations
+
+/bin/mkdir -p $ACTUAL_DIRECTORY
+/bin/rm -f $ACTUAL_DIRECTORY/*
+DIFF="/usr/bin/diff --brief"
+
+function performUnitTest() {
+    LOGFILE=$1
+    TESTNAME=$2
+    TESTRESULT="PASS"
+    cat $INPUT_DIRECTORY/$LOGFILE | ./parse_apache_log.pl > $ACTUAL_DIRECTORY/$LOGFILE.pl
+    $DIFF $EXPECTATION_DIRECTORY/$LOGFILE.pl $ACTUAL_DIRECTORY/$LOGFILE.pl
+    if [ "$?" != "0" ]; then
+	TESTRESULT="FAIL"
+    fi
+    cat $INPUT_DIRECTORY/$LOGFILE | ./parse_apache_log.py > $ACTUAL_DIRECTORY/$LOGFILE.py
+    $DIFF $EXPECTATION_DIRECTORY/$LOGFILE.py $ACTUAL_DIRECTORY/$LOGFILE.py
+    if [ "$?" != "0" ]; then
+	TESTRESULT="FAIL"
+    fi
+    echo "Test [$TESTNAME] Result: $TESTRESULT"
+}
+
+LOGFILE=log.create
+performUnitTest $LOGFILE "CREATE"
+
+LOGFILE=log.delete
+performUnitTest $LOGFILE "DELETE"
+
+LOGFILE=log.failure_stat
+performUnitTest $LOGFILE "FAILURE_STAT"
+
+LOGFILE=log.find_instance
+performUnitTest $LOGFILE "FIND_INSTANCE"
+
+LOGFILE=log.stat
+performUnitTest $LOGFILE "STAT"
+
+LOGFILE=log.unknown
+performUnitTest $LOGFILE "Nebulous (unknown)"
+
+LOGFILE=log.non_nebulous
+performUnitTest $LOGFILE "Non Nebulous"