Index: trunk/ippMonitor/czartool/czarclean.pl
===================================================================
--- trunk/ippMonitor/czartool/czarclean.pl	(revision 32093)
+++ trunk/ippMonitor/czartool/czarclean.pl	(revision 32093)
@@ -0,0 +1,65 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+# local classes
+use czartool::Config;
+use czartool::CzarDb;
+
+my $czarDbName = undef;
+my $begin = undef;
+my $end = undef;
+my $interval = undef;
+my $verbose = undef;
+my $optimize = undef;
+my $config = new czartool::Config();
+
+GetOptions (
+        "dbname|d=s" => \$czarDbName,
+        "interval|i=s" => \$interval,
+        "begin|b=s" => \$begin,
+        "end|e=s" => \$end,
+        "optimize|o" => \$optimize,
+        "verbose|v" => \$verbose,
+        );
+
+print "\n*******************************************************************************\n";
+print "* \n";
+my $quit = 0;
+
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!$begin) {
+    $quit=1;
+    print "* REQUIRED: choose a begin date                -b <datetime>\n";
+}
+if (!$end) {
+    if($begin) {$end=$begin;} else {$end="NULL";}
+    print "* OPTIONAL: choose an end date date            -e <datetime>                   (default=$end)\n";
+}
+if (!$czarDbName) {
+    $czarDbName = "czardb";
+    print "* OPTIONAL: choose czar Db name                -d <name>                       (default=$czarDbName\n";
+}
+if (!$interval) {
+    print "* OPTIONAL: choose time interval               -i <'30 MINUTE'|'1 hour'|etc>   (default=".$config->getCzarCleanupInterval().")\n";
+}
+if (!$optimize) {
+    $optimize = 0;
+    print "* OPTIONAL: optimize database after cleanup    -o                              (default=$optimize)\n";
+}
+print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+my $czarDb = $config->getCzarDbInstance();
+
+my $labels = undef;
+
+
+$czarDb->cleanupDateRange($begin, $end, $config->getCzarCleanupInterval());
+if ($optimize) {$czarDb->optimize();}
Index: trunk/ippMonitor/czartool/czarmetrics.pl
===================================================================
--- trunk/ippMonitor/czartool/czarmetrics.pl	(revision 32093)
+++ trunk/ippMonitor/czartool/czarmetrics.pl	(revision 32093)
@@ -0,0 +1,97 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::Config;
+use czartool::DayMetrics;
+use czartool::MultiDayMetrics;
+use czartool::MetricsIndex;
+
+my $czarDbName = "czardb";
+my $day = undef;
+my $begin = undef;
+my $end = undef;
+my $cumulative = undef;
+my $index = undef;
+my $verbose = undef;
+my $save_temps = undef;
+
+GetOptions (
+        "dbname=s" => \$czarDbName,
+        "begin|b=s" => \$begin,
+        "end|e=s" => \$end,
+        "day|d=s" => \$day,
+        "index|i" => \$index,
+        "cumulative|c" => \$cumulative,
+        "verbose|v" => \$verbose,
+        );
+
+print "\n*******************************************************************************\n";
+print "* \n";
+my $quit = 0;
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!$day) {
+    print "* OPTIONAL: choose a single day                -d <date>               (default=today)\n";
+}
+if (!$begin) { 
+    print "* OPTIONAL: choose a begin date                -b <date>               (default=today)\n";
+}
+if (!$end) {
+    print "* OPTIONAL: choose an end date                 -e <date>               (default=today)\n";
+}
+if (!$cumulative) {
+    print "* OPTIONAL: cumulative metrics for date range  -c                      (default=off)\n";
+}
+if (!$index) {
+    print "* OPTIONAL: create index file for all metrics  -i                      (default=off)\n";
+}
+
+print "*\n*******************************************************************************\n";
+
+if ($quit) {exit;}
+
+my $config = new czartool::Config();
+
+if ($index) {
+
+    my $metricsIndex = new czartool::MetricsIndex($config, 1, 0);
+    $metricsIndex->writeHTML();
+    exit;
+}
+
+if (!$day && !$begin && !$end) {
+    
+    $day = strftime('%Y-%m-%d', localtime);
+}
+if ($day) {$begin = $end = $day;}
+
+
+my $thisDay = $begin;
+
+if ($cumulative) {
+
+        my $multiDayMetrics = new czartool::MultiDayMetrics($config, 1, 0, $begin, $end);
+        $multiDayMetrics->writeHTML();
+}
+else {
+
+    my $czarDb = $config->getCzarDbInstance();
+
+    while (1) {
+
+        my $dayMetrics = new czartool::DayMetrics($config, 1, 0, $thisDay);
+        $dayMetrics->writeHTML();
+
+        $thisDay = $czarDb->addInterval($thisDay, "1 DAY");
+        if ($czarDb->isBefore($end, $thisDay)) {last;}
+    }
+}
+
Index: trunk/ippMonitor/czartool/czarplot.pl
===================================================================
--- trunk/ippMonitor/czartool/czarplot.pl	(revision 32093)
+++ trunk/ippMonitor/czartool/czarplot.pl	(revision 32093)
@@ -0,0 +1,188 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+use czartool::Config;
+use czartool::Plotter;
+use czartool::StageMetrics;
+
+my $label = undef;
+my $stage = undef;
+my $save_temps = undef;
+my $timeinpast = undef;
+my $begin = undef;
+my $end = undef;
+my $day = undef;
+my $path = undef;
+my $verbose = undef;
+my $histogram = undef;
+my $timeSeries = undef;
+my $rate = undef;
+my $magicMask = undef;
+my $rateInterval = undef;
+my $deriv = undef;
+my $nebulous = undef;
+my $savingToFile = undef;
+my $analysis = undef;
+my $exposureId = undef;
+my $log = undef;
+
+GetOptions (
+        "label|l=s" => \$label,
+        "stage|s=s" => \$stage,
+        "timeinpast|p=s" => \$timeinpast,
+        "rateinterval|i=s" => \$rateInterval,
+        "exposureid|x=s" => \$exposureId,
+        "mask|m" => \$magicMask,
+        "begin|b=s" => \$begin,
+        "end|e=s" => \$end,
+        "day|d=s" => \$day,
+        "output|o=s" => \$path,
+        "histogram|h" => \$histogram,
+        "nebulous|n" => \$nebulous,
+        "rate|r" => \$rate,
+        "deriv|f" => \$deriv,
+        "analysis|a" => \$analysis,
+        "timeseries|t" => \$timeSeries,
+        "verbose|v" => \$verbose,
+        "log|g" => \$log,
+        );
+
+print "\n*******************************************************************************\n";
+print "* \n";
+my $quit = 0;
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if ($analysis && !$stage) {
+    $quit = 1;
+    print "* REQUIRED: choose a stage for analsis      -s <chip|cam|warp|etc>      (default=none)\n";}
+if (!$histogram) {
+    print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
+if (!$timeSeries) {
+    print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";} 
+if (!$magicMask) {
+    print "* OPTIONAL: plot magic mask for these times -m                          (default=off)\n";} 
+if (!$exposureId) {
+    print "* OPTIONAL: set exposure ID for magic mask  -x                          (default=none)\n";} 
+if (!$rate) {
+    print "* OPTIONAL: plot histogram of rate          -r                          (default=off)\n";} 
+if (!$deriv) {
+    $deriv = 0;
+    print "* OPTIONAL: plot first derivative           -f                          (default=$deriv)\n";} 
+if (!$analysis) {
+    $analysis = 0;
+    print "* OPTIONAL: include analysis                -a                          (default=$analysis)\n";} 
+if (!$log) {
+    $log = 0;
+    print "* OPTIONAL: use log plots                   -g                          (default=$log)\n";}
+if (!$nebulous) {
+    print "* OPTIONAL: plot nebulous disk space        -n                          (default=off)\n";} 
+if (!$label) {
+    $label = "all_stdscience_labels";
+    print "* OPTIONAL: choose a label                  -l <labellName>             (default=$label)\n";}
+if (!$stage) {
+    print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
+if (!$timeinpast) {
+    print "* OPTIONAL: choose time interval in past    -p <'1 hour'|'1 day'|etc>   (default=none\n";} 
+if (!$rateInterval) {
+    print "* OPTIONAL: time interval for rate plot     -i <'1 hour'|'1 day'|etc>   (default=depends on time frame)\n";} 
+if (!$begin) {
+    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=6:35am this morning)\n";} 
+if (!$end) {
+    print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";} 
+if (!$day) {
+    print "* OPTIONAL: choose a single day to plot     -d <date>                   (default=today)\n";} 
+if (!$path) {
+    print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
+} 
+print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+my $config = new czartool::Config();
+
+# default values
+if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;}
+if (!$verbose) {$verbose = 0;}
+if (!$save_temps) {$save_temps = 0;}
+if (!$path) {$savingToFile = 0; $path = ".";}
+else {$savingToFile = 1;}
+
+my $czarDb = $config->getCzarDbInstance();
+
+my $newDayTime =  $config->getMetricsStartTime();
+
+my $plotter = undef; 
+if ($savingToFile) {
+
+    $plotter = czartool::Plotter->new_file($config, $path, $save_temps);
+}
+else {
+
+    $plotter = czartool::Plotter->new_display($config, $save_temps);
+}
+
+# if a single day has been chosen 
+if($day) {
+
+    if ($magicMask) {
+
+        $begin = $day;
+        $end = $day;
+    }
+    else {
+
+        # plots should span 24 housr from previous day
+        my $dayBefore = $czarDb->subtractInterval($day, "1 DAY");
+                
+        $begin = "$dayBefore $newDayTime";
+        $end = "$day $newDayTime";
+    }
+}
+else {
+
+    if (!$end) {$end = $czarDb->getNowTimestamp();}
+    if (!$begin) {
+
+        my $today = strftime('%Y-%m-%d', localtime);
+        my $yesterday = $czarDb->subtractInterval($today, "1 DAY");
+
+        
+        if ($timeinpast) {$begin = $czarDb->subtractInterval($end, $timeinpast);}
+        elsif ($czarDb->isBefore($end, "$today $newDayTime")) {$begin = "$yesterday $newDayTime";}
+        else {$begin = "$today $newDayTime";}
+    }
+}
+
+if ($rate and !$timeSeries) {
+
+        $plotter->createRateHistogram($label, $stage, $begin, $end, $rateInterval);
+        exit;
+}
+
+if (!$histogram && !$nebulous) {
+    $plotter->createTimeSeries($label, $stage, $begin, $end, $timeSeries, $log, $rate);
+    exit;
+}
+
+if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
+if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
+elsif ($nebulous) {$plotter->plotDiskUsageHistogram();}
+if ($magicMask) {
+
+    if ($exposureId) {$plotter->plotMagicMaskFractionForThisExposure($exposureId);}
+    else {$plotter->plotMagicMaskFraction($begin, $end);}
+
+}
+
+if($analysis) {
+
+    my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end);
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
+}
Index: trunk/ippMonitor/czartool/czarpoll.pl
===================================================================
--- trunk/ippMonitor/czartool/czarpoll.pl	(revision 32093)
+++ trunk/ippMonitor/czartool/czarpoll.pl	(revision 32093)
@@ -0,0 +1,342 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# Main polling program for czartool. Grabs stuff from various sources every few minutes.
+#
+###########################################################################
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+# local classes
+use czartool::Config;
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+use czartool::Nebulous;
+use czartool::Plotter;
+use czartool::DayMetrics;
+use czartool::MetricsIndex;
+
+my $period = 60;
+my $save_temps = 0;
+
+GetOptions (
+        "period|p=s" => \$period, # TODO more Db args
+        );
+
+my $config = new czartool::Config();
+my $czarDb = $config->getCzarDbInstance(); 
+my $gpc1Db = $config->getGpc1Instance;
+my $nebulous = new czartool::Nebulous($czarDb);
+my $pantasks = new czartool::Pantasks();
+my $plotter = czartool::Plotter->new_file($config, "/tmp", $save_temps); 
+
+
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub");
+
+
+timePoll($period);
+
+###########################################################################
+#
+# Updates the dates from pantasks for all interested servers 
+#
+###########################################################################
+sub updateDates {
+
+    print "* Updating dates\n";
+    my @servers = ("stdscience", "registration");
+
+    my $server = undef;
+    $czarDb->emptyServerDates();
+    foreach $server (@servers) {
+
+        my @dates = @{$pantasks->getDates($server)};
+        if (@dates) { 
+
+            $czarDb->updateServerDates($server, \@dates);
+        }
+        else {
+
+            print "WARNING: No dates to update for '$server'\n";
+        }
+    }
+}
+
+###########################################################################
+#
+# Updates the labels from pantasks for all interested servers 
+#
+###########################################################################
+sub updateLabels {
+
+    print "* Updating labels\n";
+    my @servers = ("stdscience", "distribution", "publishing", "update");
+
+    my $server = undef;
+    foreach $server (@servers) {
+
+        my @labels = @{$pantasks->getLabels($server)};
+        if (@labels) { 
+
+            $czarDb->updateCurrentLabels($server, \@labels);
+        }
+        else {
+
+            print "WARNING: No labels to update for '$server'\n";
+        }
+    }
+}
+
+###########################################################################
+#
+# Updates pantasks server status TODO should really get info for all servers at once 
+#
+###########################################################################
+sub updateServerStatus {
+    print "* Checking all pantasks servers\n";
+
+    my $servers = $pantasks->getServerList();
+
+    my $server = undef;
+    my $alive = undef;
+    my $running = undef;
+    foreach $server (@{$servers}) {
+
+        $pantasks->getServerStatus($server, \$alive, \$running);
+        $czarDb->updateServerStatus($server, $alive, $running);
+    }
+}
+
+###########################################################################
+#
+# Polls with provided period (seconds) 
+#
+###########################################################################
+sub timePoll {
+    my ($period) = @_;
+
+    my $label;
+    my $new;
+    my $full;
+    my $faults;
+    my $stage;
+    my $query = undef;
+    my $str = undef;
+    my $labels = undef;
+    my $updateLabels = undef;
+    my $row = undef;
+    my $begin = undef;
+    my $end = undef;
+    my $priority = undef;
+    my $newState = undef;
+    my $nsStatus = undef;
+    my $today = undef;
+    my $yesterday = undef;
+    my $newDayTime = $config->getMetricsStartTime();
+    my $lastDayDailyTasks = "2010-01-01";
+
+    # main polling loop
+    while (1) {
+
+        # sort out times
+        $today = strftime('%Y-%m-%d', localtime);
+        $yesterday = $czarDb->subtractInterval($today, "1 DAY");
+        $end = $czarDb->getNowTimestamp();
+
+        # if before 18:00 today, then start plots from 18:00 yesterday
+        if ($czarDb->isBefore($end, "$today $newDayTime")) {$begin = "$yesterday $newDayTime";}
+        # if after 18:00 today, then perform some daily tasks and start plots from 18:00 today
+        else {
+
+            $begin = "$today $newDayTime";
+
+            # check whether yesterday was cleaned. if not, cleanup tables and optimize
+            if ($lastDayDailyTasks ne $yesterday) {
+
+                print "* performing daily tasks (clean-up, metrics)\n";
+
+                # create metrics for last 24 hours
+                print "* Creating metrics for last 24 hours\n";
+                my $dayMetrics = new czartool::DayMetrics($config, 1, 0, $today); 
+                $dayMetrics->writeHTML();
+
+                # now update metrics index page
+                my $metricsIndex = new czartool::MetricsIndex($config, 1, 0); 
+                $metricsIndex->writeHTML();
+
+                # now cleanup tables from yesterday and optimize
+                print "* Performing database cleanup\n";
+                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
+                print "* Optimizing the database\n";
+                $czarDb->optimize();
+                $lastDayDailyTasks = $yesterday;
+            }
+        }
+
+        # check nightly science status
+        print "* Checking nightly science status\n";
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus);
+
+        # check nebulous
+        print "* Checking Nebulous\n";
+        $nebulous->updateClusterSpaceInfo();
+        $plotter->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end);
+        $plotter->plotDiskUsageHistogram();
+        updateServerStatus();
+
+        # check pantasks dates
+        updateDates();
+
+        # check labels
+        updateLabels();
+
+        # servers to check
+        my @serversToCheck = ("stdscience", "update");
+
+        my $thisServer = undef;
+        foreach $thisServer (@serversToCheck) {
+
+            if ($thisServer eq "update") {$newState = "update";}
+            else {$newState = "new";}
+
+            # deal with stdscience labels
+            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
+            my $size = @{$labels};
+            if($size > 0) {
+
+                # get priority
+                foreach $row ( @{$labels} ) {
+                    my ($label) = @{$row};
+                    $priority = $gpc1Db->getPriority($label);
+                    $czarDb->setLabelPriority($label, $priority);
+                }
+
+                updateAllStages($thisServer, $newState, $labels, $begin, $end);
+                createPlots($thisServer, $labels, $begin, $end);
+            }
+            else { print "* WARNING: no $thisServer labels found in Db\n";}
+        }
+
+        print "--------------------------------------------------------------------------\n";
+        #print "* Going to sleep\n";
+        #sleep($period);
+        #print "* Waking up\n";
+    };
+}
+
+###########################################################################
+#
+# Loops through labels and creates time series and histogram plots
+#
+###########################################################################
+sub createPlots {
+    my ($server, $rows, $begin, $end) = @_;
+
+    my $stage = undef;
+    my $row = undef;
+
+    print "* Generating plots\n";
+
+    # create plots for each label for each stage
+    foreach $stage (@stages) {
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+            $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 1, 1);
+            $plotter->createRateHistogram($label, $stage, $begin, $end, undef);
+        }
+    }
+
+    my $allServerLabels = undef;
+
+    # create plots for each label for all stages
+    foreach $row ( @{$rows} ) {
+        my ($label) = @{$row};
+
+        $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
+        $plotter->createRateHistogram($label, undef, $begin, $end, undef);
+        $plotter->createHistogram($label, $begin, $end);
+    }
+
+    $allServerLabels = "all_".$server."_labels";
+
+    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1);
+    $plotter->createRateHistogram($allServerLabels, undef, $begin, $end, undef);
+    $plotter->createHistogram($allServerLabels, $begin, $end);
+
+    foreach $stage (@stages) {
+
+        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 1, 1); # TODO must be a neater way...
+        $plotter->createRateHistogram($allServerLabels, $stage, $begin, $end, undef);
+    }
+}
+
+###########################################################################
+#
+# Loops through some labels and updates processed/pending/faults in the Db
+#
+###########################################################################
+sub updateAllStages {
+    my ($labelServer, $newState, $rows, $begin, $end) = @_;
+
+    print "* Updating stage data\n";
+    my $totalNew = undef;
+    my $totalFaults = undef;
+    my $totalFull = undef;
+    my $stage = undef;
+    my $reverting = 0;
+    my $row = undef;
+    my $new = undef;
+    my $full = undef;
+    my $faults = undef;
+    my $server = undef;
+    my $state = undef;
+    my $today = strftime('%Y-%m-%d', localtime);
+
+    foreach $stage (@stages) {
+
+        $server = $pantasks->getServerForThisStage($stage);
+        $pantasks->getRevertStatus($stage, \$reverting);
+        $czarDb->updateRevertStatus($stage, $reverting);
+
+        print "* Checking labels for $stage stage\n";
+
+        $totalNew=$totalFaults=$totalFull=0;
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+
+            if ($stage eq "burntool") {
+
+                $new = $full = $faults = 0;
+                if ($labelServer eq "stdscience") {
+
+                    $full = $gpc1Db->countRegisteredExposures($label, $today);
+                }
+            }
+            else {
+
+                $new = $gpc1Db->countExposures($label, $stage, $newState);
+                $full = $gpc1Db->countExposures($label, $stage, "full");
+                $faults = $gpc1Db->countFaults($label, $stage, $newState);
+            }
+            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
+            $totalNew += $new;
+            $totalFull += $full;
+            $totalFaults += $faults;
+
+            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
+        }
+
+        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
+    }
+}
+
Index: trunk/ippMonitor/czartool/czartool.pl
===================================================================
--- trunk/ippMonitor/czartool/czartool.pl	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool.pl	(revision 32093)
@@ -0,0 +1,257 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+# local classes
+use czartool::Config;
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+
+
+my $czarDbName = "czardb";
+my $save_temps = 1;
+my $interval = undef;
+
+GetOptions ( 
+        "dbname|d=s" => \$czarDbName,
+        "interval|i=s" => \$interval,
+        );
+
+
+my @states = ("full", "new", "drop", "wait");
+my $config = new czartool::Config();
+my $czarDb = $config->getCzarDbInstance();
+my $gpc1Db = $config->getGpc1Instance();
+my $pantasks = new czartool::Pantasks();
+
+my @stdscienceLabels = undef;
+my @distributionLabels = undef;
+my @publishingLabels = undef;
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); 
+promptPoll();
+
+
+###########################################################################
+#
+# Prints simple instructions
+#
+###########################################################################
+sub printInstructions {
+
+    print "| Usage: (s)ervers, (l)abels, (q)uit, or label number from table above: ";
+}
+
+###########################################################################
+#
+# Polls waiting for input from user 
+#
+###########################################################################
+sub promptPoll {
+
+    @stdscienceLabels = @{$pantasks->getLabels("stdscience")};
+    @distributionLabels = @{$pantasks->getLabels("distribution")};
+    @publishingLabels = @{$pantasks->getLabels("publishing")};
+    checkAllLabels("new");
+    printInstructions();
+
+    my $key;
+    system "stty cbreak < /dev/tty > /dev/tty 2>&1";
+    while (($key=getc) ) {
+        last if $key eq "q";
+
+        if ($key eq "s") {checkServers();}
+        elsif ($key eq "l") {checkAllLabels("new");}
+        elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience");}
+        elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1]);}
+        printInstructions();
+
+    };
+    system "stty -cbreak < /dev/tty > /dev/tty 2>&1";
+}
+
+###########################################################################
+#
+# Compares two arrays and stores the differences
+#
+###########################################################################
+sub getArrayDifferences {
+
+    my @array1 = @{$_[0]};
+    my @array2 = @{$_[1]};
+
+    my @isect;
+    my @diff;
+    my %count;
+
+    my $item;
+    foreach $item (@array1, @array2) {$count{$item}++;}
+
+    foreach $item (keys %count) {
+        if ($count{$item} == 2) {
+            push @isect, $item;
+        } else {
+            push @diff, $item;
+        }
+    }
+
+    return \@diff;
+}
+
+###########################################################################
+#
+# Checks all servers to see if they are alive and running
+#
+###########################################################################
+sub checkServers {
+
+    my @servers = ("addstar", "cleanup", "detrend", "distribution", "pstamp", "update", "publishing", "registration", "replication", "stdscience", "summitcopy");
+    printf("\n+-----------------------------------------------+\n");
+    printf("|                      Servers                  |\n");
+    printf("+--------------+---------+----------------------+\n");
+    printf("|    server    |  alive? |  Scheduler running?  |\n");
+    printf("+--------------+---------+----------------------+\n");
+    my $server;
+    foreach $server (@servers) {
+
+        my @results = `czartool_checkServer.pl -s $server`;
+
+        printf("| %12s ", $server);
+        chomp($results[0]);
+        chomp($results[1]); 
+        printf("|   %3s   |         %3s          |\n", $results[0], $results[1]);
+    }
+
+    printf("+--------------+---------+----------------------+\n");
+}
+
+###########################################################################
+#
+# Takes label name or number
+#
+###########################################################################
+sub labelDetails {
+
+    print "\nPlease enter the label name, or number from table above: ";
+    my $input = <STDIN>;
+    chomp($input);
+    if ($input =~ m/^([0-9]|[1-9][0-9]|[1-9][0-9][0-9])$/) {
+        return checkOneLabel($stdscienceLabels[$input]);
+    }
+
+    checkOneLabel($input);
+}
+
+###########################################################################
+#
+# Checks one label and prints results in a table
+#
+###########################################################################
+sub checkOneLabel {
+    my ($label) = @_;
+
+    chomp($label);
+
+    printf("\n+----------------------------------------------------------------------------------------------------------------------------------------------+\n");
+    printf("|                                                         %32s                                                     |\n", $label);
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+    printf("|   state    |    chip    |    cam     |    fake    |    warp    |   stack    |    diff    |   magic    |  destreak  |    dist    |   publish  |\n");
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+
+    my $state;
+    my $stage;
+    my $i=0;
+    foreach $state (@states) {
+
+        chomp($label);
+        printf("| %10s ", $state);
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($label, $state, $stage));
+        }
+
+        printf("|\n");
+        $i++;
+    }
+
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+
+}
+
+###########################################################################
+#
+# Checks all labels for a particular state and prints results in a table
+#
+###########################################################################
+sub checkAllLabels {
+    my ($state) = @_;
+#print time, $/;
+    printf("\n+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n");
+    printf("|                                                                                         %10s (any faults are shown in parentheses)                                                                 |\n", $state);
+    printf("+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+\n");
+    printf("| no  |              label               | distributing? |  publishing?  |    chip    |    cam     |    fake    |    warp    |   stack    |    diff    |   magic    |  destreak  |    dist    |   publish  |\n");
+    printf("+-----+----------------------------------+---------------+---------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+
+    my $stdsLabel;
+    my $distLabel;
+    my $pubLabel;
+    my $distributing;
+    my $publishing;
+    my $stage;
+    my $i=1;
+    foreach $stdsLabel (@stdscienceLabels) {
+
+        $distributing = 0;
+        chomp($stdsLabel);
+        foreach $distLabel (@distributionLabels) {
+            chomp($distLabel);
+            if ($stdsLabel eq $distLabel) { $distributing = 1; last;}
+        }
+
+        foreach $pubLabel (@publishingLabels) {
+            chomp($pubLabel);
+            if ($stdsLabel eq $pubLabel) { $publishing = 1; last;}
+        }
+
+        printf("| %3d ", $i);
+        printf("| %32s ", $stdsLabel);
+        printf("| %10s    ", $distributing ? "yes" : "NO" );
+        printf("| %10s    ", $publishing ? "yes" : "NO" );
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($stdsLabel, $state, $stage));
+        }
+
+        printf("|\n");
+        $i++;
+    }
+
+    printf("+-----+----------------------------------+---------------+---------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+
+}
+
+###########################################################################
+#
+# Returns state and fault-count (if new) as a string TODO untidy
+#
+###########################################################################
+sub getStateAndFaultsString {
+    my ($label, $state, $stage) = @_;
+
+    my $new = $gpc1Db->countExposures($label, $stage, $state);
+
+    #if ($state ne "new") {return $new;}
+
+    my $faults = $gpc1Db->countFaults($label, $stage, $state);
+
+    if ($faults < 1) {return $new;}
+
+    return $new."(".$faults.")";
+}
+
+
Index: trunk/ippMonitor/czartool/czartool/Burntool.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Burntool.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Burntool.pm	(revision 32093)
@@ -0,0 +1,68 @@
+#!/usr/bin/perl -w
+package czartool::Burntool;
+
+use warnings;
+use strict;
+
+use IPC::Cmd 0.36 qw( can_run run );
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {};
+    $self->{_verbose} = 0;
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Get pending and processed for this label
+#
+###########################################################################
+sub getPendingAndProcessed {
+    my ($self, $label, $pending, $processed) = @_;
+
+    ${$pending} = ${$processed} = 0;
+
+    my $target = undef;
+
+    if ($label =~ m/(.*)\.nightlyscience/) {
+
+        $target = $1;
+    }
+    else {return 0;}
+
+    # get today's date UTC
+    my ($day, $month, $year) = (gmtime)[3,4,5];
+    my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day);
+
+    if ($self->{_verbose}) {print "Checking $target for $self->{_today}\n";}
+    my $command = "automate_stacks.pl --check_chips --burntool_stats --this_target_only $target --date $today";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $self->{_verbose});
+
+    if (!$success) {return 0;}
+
+    my $line;
+    foreach $line (@{$stdout_buf}) {
+
+        chomp($line);
+        if($line =~ m/BTSTATS: $today $target ([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) {
+            if ($self->{_verbose}) {print "Output =  $1 $2 $3 $4\n";}
+            ${$pending} = ($2 - $3)/60;
+            ${$processed} = $3/60;
+            return 1;
+        }
+    }
+
+    return 0;
+}
+1;
+
+
Index: trunk/ippMonitor/czartool/czartool/Config.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Config.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Config.pm	(revision 32093)
@@ -0,0 +1,267 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+use XML::LibXML;
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+
+package czartool::Config;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = {};
+
+    bless $self, $class;
+    $self->parseConfigFile();
+    return $self;
+}
+
+###########################################################################
+#
+# Parses the XML config file 
+#
+###########################################################################
+sub parseConfigFile {
+    my ($self) = @_;
+
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file("czartool/czarconfig.xml");
+    my $xc = XML::LibXML::XPathContext->new($doc);
+    # $xc->registerNs('czarconfig', 'ippczar');
+
+    # gnuplot stuff
+    $self->{gnuplotfont} = $xc->findvalue('//gnuplot/font');
+    $self->{gnuplotfontsize} = $xc->findvalue('//gnuplot/size');
+
+    # metrics
+    $self->{metricssavelocation} = $xc->findvalue('//metrics/savelocation');
+    $self->{metricsstarttime} = $xc->findvalue('//metrics/starttime');
+
+    # gpc1 db
+    $self->{gpc1name} = $xc->findvalue('//gpc1database/name');
+    $self->{gpc1host} = $xc->findvalue('//gpc1database/host');
+    $self->{gpc1user} = $xc->findvalue('//gpc1database/user');
+    $self->{gpc1password} = $xc->findvalue('//gpc1database/password');
+
+    # czar db
+    $self->{czarname} = $xc->findvalue('//czardatabase/name');
+    $self->{czarhost} = $xc->findvalue('//czardatabase/host');
+    $self->{czaruser} = $xc->findvalue('//czardatabase/user');
+    $self->{czarpassword} = $xc->findvalue('//czardatabase/password');
+    $self->{czardbcleanupinterval} = $xc->findvalue('//czardatabase/cleanupinterval');
+
+    # roboczar
+    $self->{roboczaremail} = $xc->findvalue('//roboczar/email');
+    $self->{roboczarserverinterval} = $xc->findvalue('//roboczar/serverinterval');
+    $self->{roboczarinterestedservers} = $xc->findvalue('//roboczar/interestedservers');
+}
+
+###########################################################################
+#
+# Returns an instance of the czardb database 
+#
+###########################################################################
+sub getCzarDbInstance {
+    my ($self) = @_;
+
+    my $czarDb = new czartool::CzarDb(
+            $self->{czarname}, 
+            $self->{czarhost}, 
+            $self->{czaruser},
+            $self->{czarpassword}
+            );
+    $czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+    return $czarDb;
+}
+
+###########################################################################
+#
+# Returns an instance of the gpc1 database 
+#
+###########################################################################
+sub getGpc1Instance {
+    my ($self) = @_;
+
+    return new czartool::Gpc1Db(
+            $self->{gpc1name}, 
+            $self->{gpc1host}, 
+            $self->{gpc1user},
+            $self->{gpc1password});
+}
+
+###########################################################################
+#
+# Returns the interval used in czarDB cleanup
+#
+###########################################################################
+sub getCzarCleanupInterval {
+    my ($self) = @_;
+    return $self->{czardbcleanupinterval};
+}
+
+###########################################################################
+#
+# Returns the interval after which we consider a server to be down/stopped
+#
+###########################################################################
+sub getRoboczarServerInterval {
+    my ($self) = @_;
+    return $self->{roboczarserverinterval};
+}
+
+###########################################################################
+#
+# Returns array of servers we are interested in for roboczar
+#
+###########################################################################
+sub getRoboczarInterestedServers {
+    my ($self) = @_;
+
+    my @servers = split('\s+', $self->{roboczarinterestedservers});
+
+    return @servers;
+}
+
+###########################################################################
+#
+# Returns the email to which roboczar should send its warnings
+#
+###########################################################################
+sub getRoboczarEmail {
+    my ($self) = @_;
+    return $self->{roboczaremail};
+}
+
+###########################################################################
+#
+# Returns font for gnuplot
+#
+###########################################################################
+sub getGnuplotFont {
+    my ($self) = @_;
+    return $self->{gnuplotfont};
+}
+
+###########################################################################
+#
+# Returns font size for gnuplot
+#
+###########################################################################
+sub getGnuplotFontSize {
+    my ($self) = @_;
+    return $self->{gnuplotfontsize};
+}
+
+###########################################################################
+#
+# Returns save location for metrics
+#
+###########################################################################
+sub getMetricsSaveLocation {
+    my ($self) = @_;
+    return $self->{metricssavelocation};
+}
+
+###########################################################################
+#
+# Returns daily start time for metrics
+#
+###########################################################################
+sub getMetricsStartTime {
+    my ($self) = @_;
+    return $self->{metricsstarttime};
+}
+
+###########################################################################
+#
+# Returns name for gpc1 Db
+#
+###########################################################################
+sub getGpc1Name {
+    my ($self) = @_;
+    return $self->{gpc1name};
+}
+
+###########################################################################
+#
+# Returns host for gpc1 Db
+#
+###########################################################################
+sub getGpc1Host {
+    my ($self) = @_;
+    return $self->{gpc1host};
+}
+
+###########################################################################
+#
+# Returns user for gpc1 Db
+#
+###########################################################################
+sub getGpc1User {
+    my ($self) = @_;
+    return $self->{gpc1user};
+}
+
+###########################################################################
+#
+# Returns password for gpc1 Db
+#
+###########################################################################
+sub getGpc1Password {
+    my ($self) = @_;
+    return $self->{gpc1password};
+}
+
+###########################################################################
+#
+# Returns czar Db name
+#
+###########################################################################
+sub getCzarName {
+    my ($self) = @_;
+    return $self->{czarname};
+}
+
+###########################################################################
+#
+# Returns host for czar Db
+#
+###########################################################################
+sub getCzarHost {
+    my ($self) = @_;
+    return $self->{czarhost};
+}
+
+###########################################################################
+#
+# Returns username for czar Db
+#
+###########################################################################
+sub getCzarUser {
+    my ($self) = @_;
+    return $self->{czaruser};
+}
+
+###########################################################################
+#
+# Returns password for czar Db 
+#
+###########################################################################
+sub getCzarPassword {
+    my ($self) = @_;
+    return $self->{czarpassword};
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/CzarDb.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/CzarDb.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/CzarDb.pm	(revision 32093)
@@ -0,0 +1,1773 @@
+#!/usr/bin/perl -w
+
+package czartool::CzarDb;
+
+use warnings;
+use strict;
+
+use File::Temp;
+
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); # TODO put elsewhere
+
+use base 'czartool::MySQLDb';
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4], $_[5], $_[6]);
+
+    bless $self, $class;
+
+    $self->update();    
+    return $self;
+}
+
+###########################################################################
+#
+# Gets current_labels table
+#
+###########################################################################
+sub getCurrentLabels {
+    my ($self, $server, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT label
+        FROM current_labels
+        WHERE server LIKE '$server';
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
+
+###########################################################################
+#
+# Updates server table
+#
+###########################################################################
+sub updateServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO servers
+        (server, alive, running)
+        VALUES
+        ('$server', $alive, $running);
+SQL
+
+       $query->execute;
+}
+
+###########################################################################
+#
+# Gets revert status for this stage
+#
+###########################################################################
+sub getRevertStatus {
+    my ($self, $stage, $reverting) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT reverting 
+        FROM reverts
+         WHERE stage LIKE '$stage'; 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Updates revert status for this stage
+#
+###########################################################################
+sub updateRevertStatus {
+    my ($self, $stage, $reverting) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+     UPDATE reverts 
+         SET reverting = $reverting 
+         WHERE stage LIKE '$stage';
+SQL
+
+    $query->execute;
+}
+
+###########################################################################
+#
+# Sets priority for this label
+#
+###########################################################################
+sub setLabelPriority {
+    my ($self, $label, $priority) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+     UPDATE current_labels 
+         SET priority = $priority 
+         WHERE label LIKE '$label' 
+         AND server LIKE 'stdscience';
+SQL
+
+    $query->execute;
+}
+
+###########################################################################
+#
+# Cleans out server_dates table
+#
+###########################################################################
+sub emptyServerDates {
+    my ($self, $server, $dates) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM server_dates;
+SQL
+
+    $query->execute;
+}
+
+###########################################################################
+#
+# Updates current_labels table
+#
+###########################################################################
+sub updateServerDates {
+    my ($self, $server, $dates) = @_;
+
+    my $size = scalar @{$dates};
+    if ($size < 1) { return; }
+
+    my $date = undef;
+
+    foreach $date (@{$dates}) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        INSERT INTO server_dates
+            (server, date)
+            VALUES
+            ('$server', '$date');
+SQL
+
+       $query->execute;
+    }
+}
+
+###########################################################################
+#
+# Updates nightlyscience table
+#
+###########################################################################
+sub updateNightlyScience {
+    my ($self, $status) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM nightlyscience;
+SQL
+
+    $query->execute;
+
+    $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO nightlyscience
+        (status)
+        VALUES
+        ('$status');
+SQL
+
+       $query->execute;
+}
+
+###########################################################################
+#
+# Updates current_labels table
+#
+###########################################################################
+sub updateCurrentLabels {
+    my ($self, $server, $labels) = @_;
+
+    my $size = scalar @{$labels};
+    if ($size < 1) { return; }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM current_labels WHERE server LIKE '$server';
+SQL
+
+    $query->execute;
+
+    my $label = undef;
+
+    foreach $label (@{$labels}) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        INSERT INTO current_labels
+            (server, label)
+            VALUES
+            ('$server', '$label');
+SQL
+
+       $query->execute;
+    }
+}
+
+###########################################################################
+#
+# Updates hosts table
+#
+###########################################################################
+sub updateHost {
+    my ($self, $host, $total, $available, $used, $readable, $writable) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM hosts WHERE host LIKE '$host';
+SQL
+
+    $query->execute;
+    $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO hosts
+        (host, total, available, used, readable, writable)
+        VALUES
+        ('$host', $total, $available, $used, $readable, $writable);
+SQL
+    $query->execute;
+
+}
+
+###########################################################################
+#
+# Inserts new cluster space info
+#
+###########################################################################
+sub insertNewClusterSpace {
+    my ($self, $total, $available, $used, $hostsOver98) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO cluster_space
+        (total, available, used, hostsOver98)
+        VALUES
+        ( $total, $available, $used, $hostsOver98);
+SQL
+
+        $query->execute;
+}
+
+###########################################################################
+#
+# Inserts new time data into relevant table for a given label
+#
+###########################################################################
+sub insertNewTimeData {
+    my ($self, $stage, $label, $pending, $processed, $faults) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO $stage
+        (label, pending, processed, faults)
+        VALUES
+        ('$label', $pending, $processed, $faults);
+SQL
+
+        $query->execute;
+}
+
+###########################################################################
+#
+# Gets time min/max/diff for a given period
+#
+###########################################################################
+sub getTimeMinMaxDiff {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
+        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
+            FROM $stage 
+            WHERE label LIKE '$label'
+            AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    my ($_maxX, $_minX, $_timeDiff) = $query->fetchrow_array();
+
+    # no values - get out of here
+    if (!$_maxX || !$_minX || !$_timeDiff) {return 0;}
+    if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
+
+    # set new min/max
+    if (!$self->isBefore(${$maxX}, $_maxX)) {${$maxX} = $_maxX;}
+    if (!$self->isBefore($_minX, ${$minX})) {${$minX} = $_minX;}
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
+
+###########################################################################
+#
+# Analysis this stage between these times 
+#
+###########################################################################
+sub runAnalysis {
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults, processed 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    my $lastProcessed = -1;
+    my $lastPending = -1;
+    my $lastFaults = -1;
+    my $lastTimestamp = undef;
+    my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
+    my ($derivProcessed, $derivPending, $derivFaults);
+    my $notProcessing = undef;
+    my $howLongNotProcessing = undef;
+    my $notProcessingLongerThanInterval = 0;
+    my $stuck = undef;
+    my $howLongStuck = undef;
+    my $stuckLongerThanInterval = 0;
+    my $stuffToDo = undef;
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
+    $linearProcessed = 0;
+    $lastProcessed = -1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed);
+        }
+        $linearPending = $thisPending;
+        $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
+
+        # get first derivative values
+        if (defined $lastTimestamp) {
+
+            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp) + 0.001;
+                
+            if ($thisProcessed >= $lastProcessed){
+                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+            }
+            else {$derivProcessed = 0;}
+            $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
+            $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
+        }
+        else {
+        
+            $derivProcessed = 0;
+            $derivPending = 0;
+            $derivFaults = 0;
+        }
+
+        # analysis
+        if (defined $lastTimestamp) {
+
+            # anything to do?
+            $stuffToDo = $linearPending - $linearFaults;
+
+            # we are not processing
+            if ($derivProcessed < 0.001 ) {
+
+                if (!defined $notProcessing) {$notProcessing = $lastTimestamp;}
+                if ($stuffToDo && !defined $stuck) {$stuck = $lastTimestamp;}
+            }
+
+            # we are processing
+            else {
+                if (defined $notProcessing) {
+                    
+                    $notProcessing = undef; 
+                    $stuck = undef; 
+                }
+                if (!defined $started) {
+
+                    $started = $lastTimestamp;
+                    $finished = undef; 
+                }
+            }
+
+            # how long stuck?
+            if (defined $stuck) {
+
+                $howLongStuck = $self->diffTimes($lastTimestamp, $stuck);
+                $stuckLongerThanInterval = $self->isBefore($interval, $howLongStuck)
+            }
+            else {
+
+                $howLongStuck = 0;
+                $stuckLongerThanInterval = 0;
+            }
+
+            # how long not processing?
+            if (defined $notProcessing) {
+                
+                $howLongNotProcessing = $self->diffTimes($lastTimestamp, $notProcessing);
+                $notProcessingLongerThanInterval = $self->isBefore($interval, $howLongNotProcessing);
+            }
+            else {
+            
+                $howLongNotProcessing = 0;
+                $notProcessingLongerThanInterval = 0;
+            }
+
+            if ($stuffToDo) {$finished = undef;}
+            elsif (!defined $finished && $started && $notProcessingLongerThanInterval) {
+
+                $finished = $notProcessing;
+            }
+        }
+
+        $lastProcessed = $thisProcessed;
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $lastTimestamp = $thisTimestamp;
+    }
+
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
+
+    if (defined $finished) {
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
+    if (!defined $started){
+    
+        $stageMetrics->setStarted(undef);
+    }
+    else {
+    
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
+    }
+    if ($stuck && $stuckLongerThanInterval && !defined $finished) {
+
+        $stageMetrics->setStuck($stuck);
+    }
+    else {
+    
+        $stageMetrics->setStuck(undef);
+    }
+
+    return 1;
+}
+###########################################################################
+#
+# Gets time series data and stores it to temp files
+#
+###########################################################################
+sub createTimeSeriesData {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $linDataFile, $logDataFile, $ratDataFile) = @_;
+
+    # get total number of data points
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT  COUNT(*)
+        FROM $stage 
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    if (!$query->execute) {return 0;}
+    my ($count) = $query->fetchrow_array();
+
+    # decide on a number for the running mean for rate plots - use 10% of total data points
+    my $range = int($count/10);
+    if ($range < 1) {$range = 1;}
+
+    $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
+    # grab all the data for the provided time range
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults, processed 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    $query->execute;
+
+    # set up Gnuplot datafiles
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$linDataFile} = $tmpFile->filename;
+    open (LINDAT, ">${$linDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$logDataFile} = $tmpFile->filename;
+    open (LOGDAT, ">${$logDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$ratDataFile} = $tmpFile->filename;
+    open (RATDAT, ">${$ratDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    my $lastProcessed = -1;
+    my $lastPending = -1;
+    my $lastFaults = -1;
+    my $lastTimestamp = undef;
+    my ($linearProcessed, $linearPending, $linearFaults);
+    my ($logProcessed, $logPending, $logFaults);
+    my ($derivProcessed, $derivPending, $derivFaults);
+    my (@rateProcessed, @ratePending, @rateFaults);
+    my $formattedTimestamp = undef;
+    my $middleTime = undef;
+    my $formattedMiddleTimestamp = undef;
+    my $positiveJump = 0;
+    my @rateTimes;
+    my $diffPending = 0;
+    my $diffProcessed = 0;
+    my $diffFaults = 0;
+    my $lastDiffProcessed = 0;
+    $derivProcessed = 0;
+
+    $linearProcessed = 0;
+    my $someData = 0;
+    my $timeSep;
+    while (my @row = $query->fetchrow_array()) {
+
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        $formattedTimestamp = $self->getFormattedDate($thisTimestamp);
+
+        # if not first time in....
+        if (defined $lastTimestamp) {
+
+            # times
+            $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+            $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp);
+            $formattedMiddleTimestamp = $self->getFormattedDate($middleTime);
+
+            $diffPending = $thisPending - $lastPending;
+            $diffFaults = $thisFaults - $lastFaults;
+
+            # First, look for large positive jumps in processed, most likely a label 
+            # added, and store value to subtract from linear value below. If > 1000
+            # processed per hour, then it must be wrong
+            if (($thisProcessed - $lastProcessed)/($timeSep/3600) > 1000) {
+                $positiveJump = $thisProcessed - $lastProcessed;
+            }
+            else {
+
+                $positiveJump = 0;
+            }
+            # we only count increases in processing - drops are due to cleanup etc
+            if ($thisProcessed > $lastProcessed) {
+
+                $diffProcessed = ($thisProcessed - $lastProcessed) - $positiveJump;
+                $linearProcessed = $linearProcessed + $diffProcessed;
+            }
+            else {
+            
+                $diffProcessed = 0;
+                $linearProcessed = $linearProcessed;
+            }
+
+            # calculate first derivative in units of images per hour
+            $derivPending = $diffPending/($timeSep/3600); 
+            $derivFaults = $diffFaults/($timeSep/3600); 
+            $derivProcessed = $diffProcessed/($timeSep/3600); 
+
+        }
+        # first time in
+        else {
+        
+            $derivPending = 0;
+            $derivFaults = 0;
+            $derivProcessed = 0;
+            $formattedMiddleTimestamp = $formattedTimestamp;
+        }
+
+        # deal with rate plot
+        push(@ratePending, $derivPending);
+        push(@rateFaults, $derivFaults);
+        push(@rateProcessed, $derivProcessed);
+        push(@rateTimes, $middleTime);
+
+        # get last few values
+        my $size = scalar (@rateProcessed);
+        if ($size > $range) {
+
+            my $counter = 0;
+            my $oldTime;
+            my $newTime = $rateTimes[$size-1];
+            my $totalProcessed = 0;
+            my $totalPending = 0;
+            my $totalFaults = 0;
+            for (my $i=$size-1; $i>=($size - $range); $i--) { 
+               
+                $totalPending = $totalPending + $ratePending[$i];
+                $totalFaults = $totalFaults + $rateFaults[$i];
+                $totalProcessed = $totalProcessed + $rateProcessed[$i];
+                $oldTime = $rateTimes[$i];
+                $counter++;
+            }
+
+            my $meanPending = $totalPending / $counter;
+            my $meanFaults = $totalFaults / $counter;
+            my $meanProcessed = $totalProcessed / $counter;
+
+            my $formattedTimestamp = $self->getFormattedDate($self->getMiddleTime($oldTime, $newTime));
+            print RATDAT "$formattedTimestamp $meanPending $meanFaults $meanProcessed\n";
+        }
+
+        $linearPending = $thisPending;
+        $linearFaults = $thisFaults;
+
+        # get log values
+        $logProcessed = ($linearProcessed < 0) ? 0 : $linearProcessed;
+        $logPending =  ($linearPending < 0) ? 0 : $linearPending;
+        $logFaults =  ($linearFaults < 0) ? 0 : $linearFaults;
+        $logProcessed = log($logProcessed + 1)/log(10);
+        $logPending = log($logPending + 1)/log(10);
+        $logFaults = log($logFaults + 1)/log(10);
+
+        # print to data file
+        print LOGDAT "$formattedTimestamp $logPending $logFaults $logProcessed\n";
+        print LINDAT "$formattedTimestamp $linearPending $linearFaults $linearProcessed\n";
+
+        if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
+
+        $lastProcessed = $thisProcessed;
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $lastTimestamp = $thisTimestamp;
+        $lastDiffProcessed = $diffProcessed;
+    }
+
+    close(LINDAT) or print "* Problem closing linear gnuplot data file for plot for '$label' '$stage'\n";
+    close(LOGDAT) or print "* Problem closing log gnuplot data file for plot for '$label' '$stage'\n";
+    close(RATDAT) or print "* Problem closing rate gnuplot data file for plot for '$label' '$stage'\n";
+}
+
+###########################################################################
+#
+# Returns number of faults at a particular time in the past for a given label and stage
+#
+###########################################################################
+sub countFaultsInPast {
+    my ($self, $label, $stage, $interval) = @_; # TODO use time not interval
+
+    # get earliest time for this label
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval) 
+        FROM $stage
+        WHERE label LIKE '$label'; 
+SQL
+    $query->execute;
+
+my $timestamp = $query->fetchrow_array(); 
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT faults 
+        FROM $stage
+        WHERE label LIKE '$label' 
+        AND timestamp <= '$timestamp' LIMIT 1;
+SQL
+    $query->execute;
+
+return $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures that are pending right now for a given stage and label
+#
+###########################################################################
+sub countPendingNow {
+    my ($self, $label, $stage) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT pending - faults 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+        $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Creates histogram data
+#
+###########################################################################
+sub countProcessedPendingAndFaults {
+    my ($self, $label, $stage, $fromTime, $toTime, $processed, $pending, $faults) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        pending, faults 
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    (${$pending}, ${$faults}) = $query->fetchrow_array();    
+
+    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
+
+    if (!${$pending}) {${$pending} = 0;}
+    if (!${$faults}) {${$faults} = 0;}
+    if (!${$processed}) {${$processed} = 0;}
+
+    return 1;
+}
+
+###########################################################################
+#
+# Gets host data and stores it to temp file
+#
+###########################################################################
+sub createHostsData {
+    my ($self, $limit, $tmpFile) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        host, used, available, writable, readable 
+        FROM hosts
+        ORDER BY host;
+SQL
+
+    $query->execute;
+
+    my $underLimit;
+
+    # loop round results
+    while (my @row = $query->fetchrow_array()) {
+
+        my ($host, $used, $available, $writable, $readable) = @row;
+   
+        if (($used/($used+$available))*100 > $limit) {$underLimit = 0;}
+        else {$underLimit = 1;}
+
+        # Col 1: host
+        print GNUDAT "$host";
+        
+        # Col 2: available, readable used space under limit
+        if ($readable && $underLimit) {print GNUDAT " $used";}
+        else {print GNUDAT " 0";}
+
+        # Col 3: available, readable used space OVER limit
+        if ($readable && !$underLimit) {print GNUDAT " $used";}
+        else {print GNUDAT " 0";}
+
+        # Col 4: NOT available used space
+        if (!$readable) {print GNUDAT " $used";}
+        else {print GNUDAT " 0";}
+
+        # Col 5: available writable space
+        if($writable) {print GNUDAT " $available";}
+        else {print GNUDAT " 0";}
+
+        # Col 6: NOT available writable space
+        if(!$writable) {print GNUDAT " $available";}
+        else {print GNUDAT " 0";}
+
+        print GNUDAT "\n";
+    }
+
+    close(GNUDAT);
+}
+
+###########################################################################
+#
+# Gets total used cluster space as a percentage
+#
+############################################################################
+sub getTotalClusterStorageAsPercentage {
+    my ($self) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT used/total*100 
+        FROM cluster_space 
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Gets time series data for cluster storage and saves it to file 
+#
+###########################################################################
+sub createStorageTimeSeriesData {
+    my ($self, $tmpFile, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MAX(available), MIN(available),  
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
+        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
+            FROM cluster_space 
+            WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    if (!$query->execute) {return undef;}
+
+    (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
+
+
+    if (!${$maxY} || !${$minY} || !${$maxX} || !${$minX} || !${$timeDiff}) {return 0;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available 
+        FROM cluster_space 
+        WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    $query->execute;
+
+    # loop round results
+    while (my @row = $query->fetchrow_array()) {
+
+        my ($timestamp, $available) = @row;
+        print GNUDAT "$timestamp $available\n";
+    }
+
+    close(GNUDAT);
+
+    return 1;
+}
+
+###########################################################################
+#
+# Creates data for processing rate plots
+#
+###########################################################################
+sub createProcessingRateData {
+    my ($self, $stage, $label, $begin, $end, $interval, $dataFile, $minX, $maxX, $timeDiff) = @_;
+
+    my $startTime = $begin;
+    my $endTime;
+    my $quit = 0;
+    my $processed;
+    my $pending;
+    my $faults;
+    my $timestamp;
+
+    $self->getTimeMinMaxDiff($label, $stage, $begin, $end, $minX, $maxX, $timeDiff);
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy(0);
+    ${$dataFile} = $tmpFile->filename;
+
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+    my $cleanupCarry = 0;
+    my $someData = 0;
+    while(1) {
+
+        if (!$self->isBefore($startTime, $end)) {last;}
+        $endTime = $self->addInterval($startTime, $interval);
+        if (!$self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults)) {}
+        $timestamp = $self->getFormattedDate($endTime);
+        if (!$processed) {$processed = 0;}
+        print GNUDAT "$timestamp $pending $faults $processed\n";
+
+        if ($processed > 0 ) {$someData = 1;}
+
+        $startTime = $endTime;
+    }
+
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n";
+}
+
+###########################################################################
+#
+# Gets count of processed stuff in a given time period 
+#
+###########################################################################
+sub countProcessed {
+    my ($self, $label, $stage, $begin, $end) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        processed 
+        FROM $stage
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$begin' 
+        AND timestamp < '$end' 
+SQL
+        $query->execute;
+
+    my $processedArray = $query->fetchall_arrayref();
+
+    my $processed;
+    my $thisCount = -1;
+    my $lastCount = -1;
+    my $count = 0;
+    foreach $processed ( @{$processedArray} ){
+
+        ($thisCount) = @{$processed};
+
+        if ($thisCount > $lastCount && $lastCount != -1) {$count = $count + ($thisCount -  $lastCount);}
+        $lastCount = $thisCount;
+    }
+
+    return $count;
+}
+
+###########################################################################
+#
+# Deletes all but one row per interval from all stage tables for all labels between the two dates
+#
+###########################################################################
+sub cleanupDateRange {
+    my ($self, $startDay, $endDay, $interval) = @_;
+
+    my $thisDay = $startDay;
+    my $quit = 0;
+    while(!$quit) {
+
+        if (!$self->isBefore($thisDay, $endDay)) {
+
+            $quit = 1;
+        }
+
+        print "* Running cleanup for $thisDay with an interval of $interval\n";
+        $self->cleanupADay($thisDay, $interval);
+        $thisDay = $self->addInterval($thisDay, "1 DAY");
+
+    }
+}
+###########################################################################
+#
+# Deletes all but one row per interval from all tables for the provided date range
+# TODO this is very clumsy, I just don't have time to think of something more elegant
+#
+###########################################################################
+sub cleanupADay {
+    my ($self, $startDay, $interval) = @_;
+
+    my $endDay =  $self->addInterval($startDay, "1 DAY");
+
+    my $labels = undef;
+    my $fromTime = $startDay;
+    my $toTime = undef;
+    my $totalDeleted = undef;
+    my $quit = 0;
+    my $query = undef;
+    while(!$quit) {
+
+        $toTime = $self->addInterval($fromTime, $interval);
+        if (!$self->isBefore($toTime, $endDay)) {
+
+            $toTime = $endDay;
+            $quit = 1;
+        }
+
+        my $stage = undef;
+        $totalDeleted = 0;
+        foreach $stage (@stages) {
+
+            if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;}
+
+            my $label = undef;
+            my $row = undef;
+            foreach $row ( @{$labels} ) {
+                my ($label) = @{$row};
+
+                $query = $self->{_db}->prepare(<<SQL);
+                SELECT COUNT(*) 
+                    FROM $stage 
+                    WHERE timestamp > '$fromTime'
+                    AND timestamp <= '$toTime'
+                    AND label = '$label' 
+SQL
+
+                    $query->execute;
+
+                my $toDelete = scalar $query->fetchrow_array() - 1;
+                if ($toDelete < 1) {next;}
+
+                $query = $self->{_db}->prepare(<<SQL);
+                DELETE FROM $stage 
+                    WHERE timestamp > '$fromTime' 
+                    AND timestamp <= '$toTime' 
+                    AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                    $query->execute;
+
+                $totalDeleted += $toDelete;
+            }
+
+        }
+
+        # servers table
+        my $servers = undef;
+        if ($self->getServers(\$servers)) {
+
+            my $server = undef;
+            my $row = undef;
+            foreach $row ( @{$servers} ) {
+                my ($server) = @{$row};
+
+                $query = $self->{_db}->prepare(<<SQL);
+                SELECT COUNT(*) 
+                    FROM servers 
+                    WHERE timestamp > '$fromTime'
+                    AND timestamp <= '$toTime'
+                    AND server = '$server' 
+SQL
+                    $query->execute;
+
+                my $toDelete = scalar $query->fetchrow_array() - 1;
+                if ($toDelete > 0) {
+
+                    $query = $self->{_db}->prepare(<<SQL);
+                    DELETE FROM servers 
+                        WHERE timestamp > '$fromTime' 
+                        AND timestamp <= '$toTime' 
+                        AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                        $query->execute;
+
+                    $totalDeleted += $toDelete;
+                }
+            }
+        }
+
+        # now deal with cluster_space table
+        $query = $self->{_db}->prepare(<<SQL);
+        SELECT COUNT(*) 
+            FROM cluster_space 
+            WHERE timestamp > '$fromTime'
+            AND timestamp <= '$toTime'
+SQL
+            $query->execute;
+
+        my $toDelete = scalar $query->fetchrow_array() - 1;
+        if ($toDelete > 0) {
+
+            $query = $self->{_db}->prepare(<<SQL);
+            DELETE FROM cluster_space 
+                WHERE timestamp > '$fromTime' 
+                AND timestamp <= '$toTime' 
+                ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                $query->execute;
+
+            $totalDeleted += $toDelete;
+        }
+
+        print "   * Deleted $totalDeleted between $fromTime and  $toTime\n";
+        $fromTime = $toTime;
+    }
+}
+
+###########################################################################
+#
+# Optimizes all tables that need to be optimized 
+#
+###########################################################################
+sub optimize {
+    my ($self) = @_;
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        $self->optimizeTable($stage);
+    }
+
+    $self->optimizeTable("cluster_space");
+    $self->optimizeTable("servers");
+}
+
+###########################################################################
+#
+# Returns if a particular server has been stopped for a certain interval 
+#
+###########################################################################
+sub isServerStopped {
+    my ($self, $server, $interval, $since) = @_;
+
+    return $self->getServerStatus($server, $interval, $since, "running");
+}
+
+###########################################################################
+#
+# Returns if a particular server has been down for a certain interval 
+#
+###########################################################################
+sub isServerDown {
+    my ($self, $server, $interval, $since) = @_;
+
+    return $self->getServerStatus($server, $interval, $since, "alive");
+}
+
+###########################################################################
+#
+# Returns if a particular server has been down for a certain interval 
+#
+###########################################################################
+sub getServerStatus {
+    my ($self, $server, $interval, $since, $mode) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM servers
+        WHERE server = '$server' 
+        AND timestamp > now() - INTERVAL $interval
+SQL
+
+        $query->execute;
+    my $numOfReadings = scalar $query->fetchrow_array();
+
+    if ($numOfReadings == 0) {return 0;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM servers
+        WHERE server = '$server' 
+        AND timestamp > now() - INTERVAL $interval
+        AND !$mode
+SQL
+
+        $query->execute;
+    my $numNotRunning = scalar $query->fetchrow_array();
+
+    if ($numOfReadings != $numNotRunning) {return 0;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp
+        FROM servers
+        WHERE server = '$server'
+        AND $mode 
+        ORDER BY timestamp DESC 
+        LIMIT 1
+
+SQL
+    $query->execute;
+    ${$since} = scalar $query->fetchrow_array();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns an array of servers 
+#
+###########################################################################
+sub getServers {
+    my ($self, $servers) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT server
+        FROM servers
+SQL
+
+        if (!$query->execute) {
+
+            return 0;
+        }
+
+    ${$servers} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns an array of labels present during the provided time frame 
+#
+###########################################################################
+sub getLabelsInThisTimePeriod {
+    my ($self, $stage, $fromTime, $toTime, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT label 
+        FROM $stage 
+        WHERE timestamp > '$fromTime' 
+        AND timestamp <= '$toTime';
+SQL
+
+        if (!$query->execute) {
+
+            return 0;
+        }
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Update Db to newest version 
+#
+###########################################################################
+sub update { 
+    my ($self) = @_;
+
+    my $currentRevision = -1;
+
+    while (1) {
+
+        $currentRevision = $self->getRevision();
+        if ($self->{_verbose}) {print "* Current Db revision = $currentRevision\n";}
+
+        if ($currentRevision == 0) {$self->createRevision_1();}
+        elsif ($currentRevision == 1) {$self->createRevision_2();}
+        elsif ($currentRevision == 2) {$self->createRevision_3();}
+        elsif ($currentRevision == 3) {$self->createRevision_4();}
+        elsif ($currentRevision == 4) {$self->createRevision_5();}
+        elsif ($currentRevision == 5) {$self->createRevision_6();}
+        elsif ($currentRevision == 6) {$self->createRevision_7();}
+        elsif ($currentRevision == 7) {$self->createRevision_8();}
+        elsif ($currentRevision == 8) {$self->createRevision_9();}
+        elsif ($currentRevision == 9) {$self->createRevision_10();}
+        elsif ($currentRevision == 10) {$self->createRevision_11();}
+        elsif ($currentRevision == 11) {$self->createRevision_12();}
+        elsif ($currentRevision == 12) {$self->createRevision_13();}
+        else {last;}
+    }
+}
+
+#######################################################################################
+# 
+# Sets current revision of ippToPsps database
+#
+#######################################################################################
+sub setRevision {
+    my ($self, $revision) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO revision (revision) VALUES ($revision);
+SQL
+        $query->execute;
+}
+
+#######################################################################################
+# 
+# Gets current revision of ippToPsps database
+#
+#######################################################################################
+sub getRevision {
+    my ($self) = @_;
+
+    if (!$self->SUPER::doesTableExist("revision")) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT revision
+        FROM revision
+        ORDER BY revision DESC LIMIT 1;
+SQL
+
+        $query->execute;
+    my @row = $query->fetchrow_array();
+
+    return $row[0];
+}
+
+#######################################################################################
+# 
+# Create revision 1 of the database 
+#
+#######################################################################################
+sub createRevision_1 {
+    my ($self) = @_;
+
+    print "* Creating revision 1 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE revision (
+            revision INT, 
+            created TIMESTAMP DEFAULT NOW(),
+            primary key (revision)
+            );
+SQL
+        $query->execute;
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE TABLE $stage (
+                timestamp TIMESTAMP DEFAULT NOW(),
+                label VARCHAR(128) DEFAULT "NONE", 
+                pending BIGINT NOT NULL,
+                faults BIGINT NOT NULL
+                );
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(1);
+}
+
+#######################################################################################
+# 
+# Create revision 2 of the database 
+#
+#######################################################################################
+sub createRevision_2 {
+    my ($self) = @_;
+
+    print "* Creating revision 2 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(2);
+}
+
+#######################################################################################
+# 
+# Create revision 3 of the database 
+#
+#######################################################################################
+sub createRevision_3 {
+    my ($self) = @_;
+
+    print "* Creating revision 3 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
+SQL
+
+            $query->execute;
+    }
+
+    $self->setRevision(3);
+}
+
+#######################################################################################
+# 
+# Create revision 4 of the database 
+#
+#######################################################################################
+sub createRevision_4 {
+    my ($self) = @_;
+
+    print "* Creating revision 4 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE current_labels (server VARCHAR(128), label VARCHAR(128));
+SQL
+
+        $query->execute;
+
+    $self->setRevision(4);
+}
+
+#######################################################################################
+# 
+# Create revision 5 of the database 
+#
+#######################################################################################
+sub createRevision_5 {
+    my ($self) = @_;
+
+    print "* Creating revision 5 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE servers (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            server VARCHAR(128), 
+            alive TINYINT, running TINYINT);
+SQL
+
+        $query->execute;
+
+    $self->setRevision(5);
+}
+
+#######################################################################################
+# 
+# Create revision 6 of the database 
+#
+#######################################################################################
+sub createRevision_6 {
+    my ($self) = @_;
+
+    print "* Creating revision 6 of '$self->{_dbName}'\n";
+
+    my $stage = undef;
+    my $index = undef;
+    foreach $stage (@stages) {
+
+        $index = $stage . "Index";
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX $index ON $stage (timestamp, label);
+SQL
+
+            $query->execute;
+    }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX serverIndex ON servers (timestamp, server);
+SQL
+
+        $query->execute;
+
+
+    $self->setRevision(6);
+}
+
+#######################################################################################
+# 
+# Create revision 7 of the database 
+#
+#######################################################################################
+sub createRevision_7 {
+    my ($self) = @_;
+
+    print "* Creating revision 7 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE current_labels 
+        ADD COLUMN priority INT NOT NULL DEFAULT 0;
+SQL
+
+        $query->execute;
+
+    $self->setRevision(7);
+}
+
+#######################################################################################
+# 
+# Create revision 8 of the database 
+#
+#######################################################################################
+sub createRevision_8 {
+    my ($self) = @_;
+
+    print "* Creating revision 8 of '$self->{_dbName}'\n";
+
+    # drop reverting column from all stages tables
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            DROP COLUMN reverting;
+SQL
+
+            $query->execute;
+    }
+
+    # create new 'revert' table
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE reverts (
+            stage VARCHAR(128), 
+            reverting TINYINT);
+SQL
+
+        $query->execute;
+
+    # insert stages into revert table
+    foreach $stage (@stages) {
+        my $query = $self->{_db}->prepare(<<SQL);
+        INSERT INTO reverts
+            (stage, reverting)
+            VALUES
+            ('$stage', 0);
+SQL
+            $query->execute;
+    }
+
+    $self->setRevision(8);
+}
+
+#######################################################################################
+# 
+# Create revision 9 of the database 
+#
+#######################################################################################
+sub createRevision_9 {
+    my ($self) = @_;
+
+    print "* Creating revision 9 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE cluster_space (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            total FLOAT, 
+            available FLOAT, 
+            used FLOAT, 
+            hostsOver98 SMALLINT);
+SQL
+
+        $query->execute;
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE hosts (
+            host VARCHAR(128), 
+            total FLOAT, 
+            available FLOAT, 
+            used FLOAT); 
+SQL
+
+        $query->execute;
+
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
+SQL
+
+        $query->execute;
+
+    $self->setRevision(9);
+}
+
+#######################################################################################
+# 
+# Create revision 10 of the database 
+#
+#######################################################################################
+sub createRevision_10 {
+    my ($self) = @_;
+
+    print "* Creating revision 10 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE hosts 
+        ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
+            ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
+SQL
+
+        $query->execute;
+
+    $self->setRevision(10);
+}
+
+#######################################################################################
+# 
+# Create revision 11 of the database 
+#
+#######################################################################################
+sub createRevision_11 {
+    my ($self) = @_;
+
+    print "* Creating revision 11 of '$self->{_dbName}'\n";
+
+    # same shape as other stage tables to enable easy update
+    #my $query = $self->{_db}->prepare(<<SQL);
+    #CREATE TABLE burntool (
+    #        timestamp TIMESTAMP DEFAULT NOW(),
+    #        label VARCHAR(128) DEFAULT "NONE", 
+    #        pending BIGINT NOT NULL,
+    #        processed BIGINT NOT NULL,
+    #        faults BIGINT NOT NULL);
+#SQL
+
+#        $query->execute;
+#    $query = $self->{_db}->prepare(<<SQL);
+#    CREATE INDEX burntoolIndex ON burntool (timestamp, label);
+#SQL
+
+#        $query->execute;
+
+    $self->setRevision(11);
+}
+
+#######################################################################################
+# 
+# Create revision 12 of the database 
+#
+# - adding nightlyscience table
+#
+#######################################################################################
+sub createRevision_12 {
+    my ($self) = @_;
+
+    print "* Creating revision 12 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE nightlyscience (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            status VARCHAR(128) DEFAULT "NONE"); 
+SQL
+
+        $query->execute;
+
+    $self->setRevision(12);
+}
+#######################################################################################
+# 
+# Create revision 13 of the database 
+#
+# adding server_dates table
+#
+#######################################################################################
+sub createRevision_13 {
+    my ($self) = @_;
+
+    print "* Creating revision 13 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE server_dates (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            server VARCHAR(64) DEFAULT "NONE", 
+            date VARCHAR(64) DEFAULT "NONE"); 
+SQL
+
+        $query->execute;
+
+    $self->setRevision(13);
+}
+1;
+
Index: trunk/ippMonitor/czartool/czartool/DayMetrics.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/DayMetrics.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/DayMetrics.pm	(revision 32093)
@@ -0,0 +1,324 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+use czartool::Metrics;
+
+
+package czartool::DayMetrics;
+our @ISA = qw(czartool::Metrics);    # inherits from Metrics class
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new(
+            $_[1],  # config object 
+            $_[2],  # verbose
+            $_[3]   # save_temps
+            ); 
+
+    $self->{day} = $_[4];
+
+    my $yesterday =  $self->{czarDb}->subtractInterval($self->{day}, "1 DAY");
+
+    # sort out times
+    $self->{begin} =  "$yesterday " . $self->{config}->getMetricsStartTime();
+    $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY");
+    $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR");
+
+    # create path, dir and html file
+    $self->{path} = "$self->{baseDir}/$self->{day}";
+    rmdir($self->{path});
+    mkdir($self->{path}, 0777);
+    $self->{plotter}->setOutputPath($self->{path});
+
+    if ($self->{verbose}) {print "* Creating metrics for $self->{day}\n";}
+
+    bless $self, $class;
+
+    return $self;
+}
+
+###########################################################################
+#
+# Writes HTML document
+#
+###########################################################################
+sub writeHTML {
+    my ($self) = @_;
+
+    # create HTML file and write header stuff
+    $self->createHtml("IPP Metrics for $self->{day}");
+    my $htmlFile = $self->{htmlFile};
+
+    # summit and burntool exposures
+    my $summitExposures = $self->{gpc1Db}->countScienceExposuresFromLastNight($self->{day});
+    my $burntoolMetrics = new czartool::StageMetrics("burntool", "all_stdscience_labels", $self->{begin}, $self->{burntoolEnd});
+    $self->{czarDb}->runAnalysis($burntoolMetrics);
+
+    my $previousDay = $self->{czarDb}->subtractInterval($self->{day}, "1 DAY");
+    my $nextDay = $self->{czarDb}->addInterval($self->{day}, "1 DAY");
+
+    print $htmlFile "<h5  align=\"middle\">";
+    print $htmlFile "<a href=\"../$previousDay/index.html\"> \< previous day</a> | <a href=\"../index.html\">all</a> | <a href=\"../$nextDay/index.html\">next day \></a><br>\n";
+    print $htmlFile "Measured from $self->{begin} to $self->{end} HST<br>\n";
+    printf ( $htmlFile "%d exposures taken on summit last night, %d through burntool today</h5>\n", 
+            $summitExposures, $burntoolMetrics->getProcessed() ? $burntoolMetrics->getProcessed() : 0 );
+    print $htmlFile "</head>\n";
+    print $htmlFile "<body>\n";
+
+
+
+    my $labels = undef;
+    my $label;
+    if (!$self->{czarDb}->getStdscienceLabelsInTimePeriod($self->{begin}, $self->{end}, \$labels)) {next;}
+
+    my $size = @{$labels};
+    my $table;
+    my $haveLabelData;
+    my %initialPendingTotals;
+    my %processedTotals;
+    my %finalPendingTotals;
+    my %faultTotals;
+    my %labelTables;
+
+    my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+# loop through all labels and create tables for each that have content
+    foreach my $row ( @{$labels} ) {
+        my ($label) = @{$row};
+
+
+        $table = "<table border='1'>";
+        $table .= "  <tr>\n";
+        $table .= "    <th>Stage</th>\n";
+        $table .= "    <th>Initial Pending</th>\n";
+        $table .= "    <th>Started</th>\n";
+        $table .= "    <th>25% complete</th>\n";
+        $table .= "    <th>50% complete</th>\n";
+        $table .= "    <th>75% complete</th>\n";
+        $table .= "    <th>90% complete</th>\n";
+        $table .= "    <th>95% complete</th>\n";
+        $table .= "    <th>Finished</th>\n";
+        $table .= "    <th>Time taken</th>\n";
+        $table .= "    <th>Processed</th>\n";
+        $table .= "    <th>Exp per hour</th>\n";
+        $table .= "    <th>Final Pending</th>\n";
+        $table .= "  </tr>\n";
+
+        $haveLabelData = 0;
+        foreach my $stage ( @stages ){
+
+            my $stageMetrics = new czartool::StageMetrics($stage, $label, $self->{begin}, ($stage eq "burntool") ? $self->{burntoolEnd} : $self->{end});
+            if (!$self->{czarDb}->runAnalysis($stageMetrics)) {next;}
+
+            if (!$stageMetrics->anythingProcessed()) {next;}
+
+            if ($stageMetrics->getProcessed() > 0) {$haveLabelData = 1;};
+
+            $table .= "  <tr>\n";
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getStage() );
+            $table .= sprintf( "    <td>%d</td>\n", $stageMetrics->getInitialPending() ? $stageMetrics->getInitialPending() : "0" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getStarted() ? $stageMetrics->getStarted() : "");
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished25() ? $stageMetrics->getFinished25() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished50() ? $stageMetrics->getFinished50() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished75() ? $stageMetrics->getFinished75() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished90() ? $stageMetrics->getFinished90() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished95() ? $stageMetrics->getFinished95() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished100() ? $stageMetrics->getFinished100() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getTotalTime() ? $stageMetrics->getTotalTime() : "-" );
+            $table .= sprintf( "    <td>%d</td>\n", $stageMetrics->getProcessed() ? $stageMetrics->getProcessed() : "0" );
+            $table .= sprintf( "    <td>%.2f</td>\n", $stageMetrics->getRate() ? $stageMetrics->getRate() : "0" );
+            $table .= sprintf( "    <td>");
+            if ($stageMetrics->getFinalPending() > 0) {$table .= "<font color=\"red\">";}
+            $table .= sprintf( "%d\n", $stageMetrics->getFinalPending() );
+            if ($stageMetrics->getFinalPending() > 0) {$table .= "</font>\n";}
+            $table .= sprintf( "</td>");
+            $table .= "  </tr>\n";
+
+            if (!$initialPendingTotals{$stage}) {$initialPendingTotals{$stage} = 0;}
+            $initialPendingTotals{$stage} = $initialPendingTotals{$stage} + $stageMetrics->getInitialPending();
+            if (!$processedTotals{$stage}) {$processedTotals{$stage} = 0;}
+            $processedTotals{$stage} = $processedTotals{$stage} + $stageMetrics->getProcessed();
+            if (!$finalPendingTotals{$stage}) {$finalPendingTotals{$stage} = 0;}
+            $finalPendingTotals{$stage} = $finalPendingTotals{$stage} + $stageMetrics->getFinalPending();
+            if (!$faultTotals{$stage}) {$faultTotals{$stage} = 0;}
+            $faultTotals{$stage} = $faultTotals{$stage} + $stageMetrics->getFaults();
+        }
+
+        $table .= "</table>";
+        $table .= "<br>\n";
+
+        if ($haveLabelData) { 
+            $labelTables{$label} = $table; 
+            if ($self->{verbose}) {print " - $label\n";}
+        }
+    }
+
+    # create contents
+    print $htmlFile "<h2>Contents</h2>\n";
+    print $htmlFile "<a href=\"#surveys\">Survey timings</a><br>\n";
+    print $htmlFile "<a href=\"#summaryplot\">Summary plots</a><br>\n";
+    print $htmlFile "<a href=\"#stages\">Stage totals</a><br>\n";
+    print $htmlFile "<a href=\"#mask\">Magic mask fraction</a><br>\n";
+    foreach my $label (keys (%labelTables)) {
+
+        print $htmlFile "<a href=\"#$label\">$label</a><br>\n";
+    }
+
+    # survey table
+    if ($self->{verbose}) {print " * creating survey table\n";}
+    $self->printSurveyTable();
+
+    # some summary plots
+    if ($self->{verbose}) {print " * creating summary plots\n";}
+    $self->createSummaryPlots();
+
+    # magic mask plots
+    if ($self->{verbose}) {print " * creating mask plots\n";}
+    $self->createMaskPlots($self->{day}, $self->{day});
+
+    # table for stage totals
+    $table = "<a name=\"stages\"></a>\n";
+    $table .= "<h2  align=\"middle\">Stage totals <a href=\"#top\">(top)</a></h1>\n";
+    $table .= "<table border='1'>";
+    $table .= "  <tr>\n";
+    $table .= "    <th>Stage</th>\n";
+    $table .= "    <th>Initial Pending</th>\n";
+    $table .= "    <th>Processed</th>\n";
+    $table .= "    <th>Final Pending</th>\n";
+    $table .= "    <th>Remaining Faults</th>\n";
+    $table .= "  </tr>\n";
+    foreach my $stage ( @stages ){
+
+        $table .= "  <tr>\n";
+        $table .= sprintf( "    <td>%s</td>\n", $stage );
+        $table .= sprintf( "    <td>%d</td>\n", $initialPendingTotals{$stage} ? $initialPendingTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $processedTotals{$stage} ? $processedTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $finalPendingTotals{$stage} ? $finalPendingTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $faultTotals{$stage} ? $faultTotals{$stage} : 0);
+        $table .= "  </tr>\n";
+    }
+    $table .= "</table>";
+    $table .= "<br>\n";
+
+    print $htmlFile $table;
+
+    # print all label tables to page
+    foreach my $label (keys (%labelTables)) {
+
+        print $htmlFile "<a name=\"$label\"></a>\n";
+        print $htmlFile "<h2  align=\"middle\">$label <a href=\"#top\">(top)</a></h1>\n";
+        $self->{plotter}->createTimeSeries($label, undef, $self->{begin}, $self->{end}, 1, 1, 1);
+        print $htmlFile "<img src=\"czarplot_linear_".$label."_all_stages_t.png\" alt=\"All stages for $label for $self->{day}\" />\n";
+        $self->{plotter}->createHistogram($label, $self->{begin}, $self->{end}, 0, 0, 0);
+        print $htmlFile "<img src=\"czarplot_linear_".$label."_all_stages_h.png\" alt=\"All stages for $label for $self->{day}\" />\n";
+
+        print $htmlFile $labelTables{$label};
+    }
+
+
+    $self->finishHtml();
+}
+
+###########################################################################
+#
+# Creates a survey table
+#
+###########################################################################
+sub printSurveyTable() {
+    my ($self) = @_;
+
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<a name=\"surveys\"></a>\n";
+    print $htmlFile "<h2  align=\"middle\">Survey timings <a href=\"#top\">(top)</a></h1>\n";
+    print $htmlFile "<table border='1'>";
+    print $htmlFile "  <tr>\n";
+    print $htmlFile "    <th>Survey</th>\n";
+    print $htmlFile "    <th>Started burntool</th>\n";
+    print $htmlFile "    <th>Finished distribution</th>\n";
+    print $htmlFile "    <th>Time taken</th>\n";
+    print $htmlFile "  </tr>\n";
+
+    # OSS survey
+    my ($started, $finished, $timeTaken) = undef;
+    $self->{czarDb}->getDayTimings("OSS.nightlyscience", "burntool", "diff", $self->{begin}, \$started, \$finished, \$timeTaken);
+    $self->printSurveyDetails("OSS", $started, $finished, $timeTaken);
+
+    # MDF surveys
+    my @mdfs = ("M31.nightlyscience",
+            "MD01.nightlyscience", 
+            "MD02.nightlyscience", 
+            "MD03.nightlyscience", 
+            "MD04.nightlyscience", 
+            "MD05.nightlyscience", 
+            "MD06.nightlyscience", 
+            "MD07.nightlyscience", 
+            "MD08.nightlyscience", 
+            "MD09.nightlyscience", 
+            "MD10.nightlyscience",
+            "MD11.nightlyscience");
+
+    my $earliest = "2099-01-01 00:00:00";
+    my $latest = "2001-01-01 00:00:00";
+
+    my $mdf;
+    my $haveStart = 0;
+    my $haveEnd = 0;
+    foreach $mdf (@mdfs) {
+
+        if ($self->{czarDb}->getDayTimings($mdf, "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken)) {
+
+            if ($started && $self->{czarDb}->isBefore($started, $earliest)) {$earliest = $started; $haveStart = 1;}
+            if ($finished && $self->{czarDb}->isBefore($latest, $finished)) {$latest = $finished; $haveEnd = 1;}
+        }
+    }
+
+    if (!$haveStart) {$earliest = undef;}
+    if (!$haveEnd) {$latest = undef;}
+    if (!$haveStart || !$haveEnd) {$timeTaken = undef;}
+
+    else {$timeTaken = $self->{czarDb}->diffTimes($latest, $earliest);}
+    $self->printSurveyDetails("MDF", $earliest, $latest, $timeTaken);
+
+    $self->{czarDb}->getDayTimings("all_stdscience_labels", "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken);
+    $self->printSurveyDetails("All", $started, $finished, $timeTaken);
+
+    print $htmlFile "</table>\n";
+}
+
+###########################################################################
+#
+# Creates a row in the survey table
+#
+###########################################################################
+sub printSurveyDetails() {
+    my ($self, $survey, $started, $finished, $timeTaken, $processed, $pending) = @_;
+
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "  <tr>\n";
+    print $htmlFile "    <td>$survey</td>\n";
+    printf ($htmlFile "    <td>%s</td>\n", $started ? $started : "no");
+    printf ($htmlFile "    <td>%s</td>\n", $finished ? $finished : "no");
+    printf ($htmlFile "    <td>%s</td>\n", $timeTaken ? $timeTaken : "na");
+    print $htmlFile "  </tr>\n";
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/Gpc1Db.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Gpc1Db.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Gpc1Db.pm	(revision 32093)
@@ -0,0 +1,299 @@
+#!/usr/bin/perl i-w
+
+package czartool::Gpc1Db;
+
+use warnings;
+use strict;
+
+use czartool::MySQLDb;
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+###########################################################################
+#
+# Returns the right join table and ID to join on for for a given stage
+#
+###########################################################################
+sub getJoinTableAndId {
+    my ($self, $stage, $joinTable, $id) = @_;
+
+    ${$id} = $stage."_id";
+
+    if ($stage eq "chip") {${$joinTable}="chipProcessedImfile";}
+    elsif ($stage eq "cam") {${$joinTable}="camProcessedExp";}
+    elsif ($stage eq "fake") {${$joinTable}="fakeProcessedImfile";}
+    elsif ($stage eq "warp") {${$joinTable}="warpSkyfile";}
+    elsif ($stage eq "stack") {${$joinTable}="stackSumSkyfile";}
+    elsif ($stage eq "diff") {${$joinTable}="diffSkyfile";}
+    elsif ($stage eq "magic") {${$joinTable}="magicNodeResult";}
+    elsif ($stage eq "magicDS") {${$id} = "magic_ds_id"; ${$joinTable}="magicDSFile";}
+    elsif ($stage eq "dist") {${$joinTable}="distComponent";}
+    elsif ($stage eq "pub") {${$joinTable}="publishDone";}
+    else {
+
+        print "* ERROR: could not find joinTable and ID for '$stage' stage\n";
+        return 0;
+    }
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns the right table for a given stage
+#
+###########################################################################
+sub getTableForStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "chip" ) {return "chipRun";}
+    elsif ($stage eq "cam" ) {return "camRun";}
+    elsif ($stage eq "fake" ) {return "fakeRun";}
+    elsif ($stage eq "warp" ) {return "warpRun";}
+    elsif ($stage eq "stack" ) {return "stackRun";}
+    elsif ($stage eq "diff" ) {return "diffRun";}
+    elsif ($stage eq "magic" ) {return "magicRun";}
+    elsif ($stage eq "magicDS" ) {return "magicDSRun";}
+    elsif ($stage eq "dist" ) {return "distRun";}
+    elsif ($stage eq "pub" ) {return "publishRun";}
+
+    return "ERROR";
+}
+
+###########################################################################
+#
+# Returns the number of science exposures taken 'last night' for a given day
+# 
+# Notes:
+#
+# - 03:00 HST = 17:00 last night UTC
+# - 17:00 HST = 07:00 this morning UTC
+# - 'OBJECT' = science exposures
+#
+###########################################################################
+sub countScienceExposuresFromLastNight {
+    my ($self, $day) = @_;
+
+    my $begin = "$day 03"; 
+    my $end = "$day 17"; 
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*)
+        FROM summitExp 
+        WHERE dateobs > '$begin' 
+        AND dateobs < '$end' AND
+        exp_type = 'OBJECT';
+SQL
+
+    $query->execute;
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures between two dates that have been registered with chip
+# 
+# Notes:
+#
+# necessary to convert times as rawExp.dateobs is in UTC
+#
+###########################################################################
+sub getProcessedExposures {
+    my ($self, $beginDate, $endDate, $expIds) = @_;
+
+    $beginDate = "$beginDate 03";
+    $endDate = "$endDate 17";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        SELECT rawExp.exp_id 
+        FROM chipRun, rawExp 
+        WHERE chipRun.exp_id = rawExp.exp_id 
+        AND rawExp.dateobs >= '$beginDate' 
+        AND rawExp.dateobs <= '$endDate';
+SQL
+
+    $query->execute;
+    ${$expIds} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns count of faults for this stage and label
+#
+###########################################################################
+sub countFaults {
+    my ($self, $label, $stage, $state) = @_;
+
+    my $table = undef;
+    my $joinTable = undef;
+    my $id = undef;
+    $table = getTableForStage($self, $stage);
+    if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
+
+    my $faultCol =  $joinTable.".fault";
+    my $stateCol =  $table.".state";
+
+    my $query =  $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(DISTINCT $id) 
+        FROM $table
+        JOIN $joinTable USING ($id)
+        WHERE label LIKE '$label'
+        AND (($faultCol != 0 AND $stateCol = '$state') OR $stateCol = 'failed_revert')
+SQL
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns priority for this label
+#
+###########################################################################
+sub getPriority {
+    my ($self, $label) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT priority 
+        FROM Label 
+        WHERE label LIKE '$label' 
+SQL
+
+    $query->execute;
+    my $priority = scalar $query->fetchrow_array();
+    if (!$priority) {return 50000;} # assume labels not given priority in gpc1 Db have highest priority
+    return $priority;
+}
+
+###########################################################################
+#
+# Returns count of exposures that have been registered
+#
+# NB JOINing summitExp to newExp to rawExp because summit_id is indexed, whereas
+# a direct JOIN from summitExp to rawExp using exp_name is more direct, but slower
+# as exp_name is not indexed in rawExp
+#
+###########################################################################
+sub countRegisteredExposures {
+    my ($self, $label, $date) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM summitExp 
+        JOIN newExp USING (summit_id) 
+        JOIN rawExp USING(exp_id) 
+        JOIN chipRun USING(exp_id) 
+        WHERE summitExp.dateobs > '$date' 
+        AND summitExp.exp_type = 'OBJECT' 
+        AND chipRun.label = "$label";
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns count of exposures with this state for this label
+#
+###########################################################################
+sub countExposures {
+    my ($self, $label, $stage, $state) = @_;
+
+    my $table = getTableForStage($self, $stage);
+    my $query = undef;
+
+    if ($state eq "update") {
+
+        if ($stage eq "dist") {return 0;}
+        my $joinTable = undef;
+        my $id = undef;
+        if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
+
+        if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") {
+            $query = $self->{_db}->prepare(<<SQL);
+            SELECT COUNT(state)
+                FROM $table JOIN $joinTable 
+                USING($id) 
+                WHERE label LIKE '$label'
+                AND state = 'update' 
+                AND data_state = 'update';
+SQL
+        }
+        else {
+        
+            $query = $self->{_db}->prepare(<<SQL);
+            SELECT COUNT(state)
+                FROM $table JOIN $joinTable 
+                USING($id) 
+                WHERE label LIKE '$label'
+                AND state = 'update'; 
+SQL
+        }
+
+    }
+    else {
+
+        $query = $self->{_db}->prepare(<<SQL);
+        SELECT COUNT(state)  
+            FROM $table 
+            WHERE label LIKE '$label' 
+            AND state = '$state'
+SQL
+    }
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns magic mask fraction across all chips for a particular exposure
+#
+###########################################################################
+sub getMagicMaskFraction {
+        my ($self, $exp_id, $fracs) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT component, streak_frac 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%";
+SQL
+    $query->execute;
+    ${$fracs} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns average magic mask fraction, sum of mask fractions and chip count 
+# for a particular exposure
+#
+###########################################################################
+sub getMagicMaskStats {
+        my ($self, $exp_id, $mean, $sum, $chipCount) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT AVG(streak_frac), SUM(streak_frac), COUNT(*) 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%"
+                AND stage = 'chip';
+SQL
+    if (!$query->execute) {return 0;} # TODO do this everywhere
+
+    (${$mean}, ${$sum}, ${$chipCount}) = $query->fetchrow_array();
+
+    return 1;
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/Metrics.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Metrics.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Metrics.pm	(revision 32093)
@@ -0,0 +1,122 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+
+package czartool::Metrics;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = {
+        config => shift,
+        verbose => shift,
+        save_temps => shift,
+    };
+
+    $self->{gpc1Db} = $self->{config}->getGpc1Instance();
+    $self->{czarDb} = $self->{config}->getCzarDbInstance();
+    $self->{baseDir} = $self->{config}->getMetricsSaveLocation();
+
+    # create path, dir
+    mkdir($self->{baseDir}, 0777);
+
+    # instantiate a plotter object
+    $self->{plotter} = czartool::Plotter->new_file(
+            $self->{config},
+            ".",
+            $self->{save_temps});
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Writes HTML intro
+#
+###########################################################################
+sub createHtml {
+    my ($self, $title) = @_; 
+
+    open (FILE, ">$self->{path}/index.html");
+    $self->{htmlFile} = *FILE;
+
+    my $htmlFile = $self->{htmlFile};
+    print $htmlFile "<html>\n"; 
+    print $htmlFile "<head>\n";
+    print $htmlFile "<title>$title</title>\n";
+    print $htmlFile "<a name=\"top\"></a>\n";
+    print $htmlFile "<h1 align=\"middle\">$title</h1>\n";
+}
+
+###########################################################################
+#
+# Creates summary plots for period 
+#
+###########################################################################
+sub createSummaryPlots {
+    my ($self) = @_;
+
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<a name=\"summaryplot\"></a>\n";
+    print $htmlFile "<h2  align=\"middle\">Summary plots <a href=\"#top\">(top)</a></h1>\n";
+    $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 1, 1, 1);
+    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_t.png\" alt=\"timeseries\" />\n";
+    $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0);
+    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_h.png\" alt=\"histogram\" />\n";
+    $self->{plotter}->createRateHistogram("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef);
+    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_rh.png\" alt=\"rate\" />\n";
+    $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end});
+    print $htmlFile "<img src=\"czarplot_cluster.png\" alt=\"storage plot not available\" />\n";
+}
+
+###########################################################################
+#
+# Creates magic mask plots for provided period 
+#
+###########################################################################
+sub createMaskPlots {
+    my ($self, $begin, $end) = @_;
+
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<a name=\"mask\"></a>\n";
+    print $htmlFile "<h2  align=\"middle\">Magic mask fraction <a href=\"#top\">(top)</a></h1>\n";
+
+    $self->{plotter}->plotMagicMaskFraction($begin, $end);
+    print $htmlFile "<img src=\"czarplot_magic_mask_fraction_h.png\" alt=\"\" />\n";
+    print $htmlFile "<img src=\"czarplot_magic_mask_fraction_d.png\" alt=\"\" />\n";
+}
+
+###########################################################################
+#
+# Writes HTML intro
+#
+###########################################################################
+sub finishHtml {
+    my ($self) = @_;
+
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<br>\n";
+    print $htmlFile "</body>\n";
+    print $htmlFile "</html>\n";
+    close($htmlFile);
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/MetricsIndex.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/MetricsIndex.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/MetricsIndex.pm	(revision 32093)
@@ -0,0 +1,114 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+use czartool::Metrics;
+
+
+package czartool::MetricsIndex;
+our @ISA = qw(czartool::Metrics);    # inherits from Metrics class
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new(
+            $_[1],  # config object 
+            $_[2],  # verbose
+            $_[3]   # save_temps
+            );
+
+    # create path, dir and html file
+    $self->{path} = "$self->{baseDir}";
+
+    if ($self->{verbose}) {print "* Creating index file for metrics\n";}
+
+    bless $self, $class;
+
+    return $self;
+}
+
+###########################################################################
+#
+# Writes HTML document
+#
+###########################################################################
+sub writeHTML {
+    my ($self) = @_;
+
+    $self->createHtml("IPP Metrics");
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<img src=\"masterMask.png\" alt=\"master magic mask fraction plot\" />\n";
+    print $htmlFile "<img src=\"masterPro.png\" alt=\"master MD processing plot\" />\n";
+
+    opendir(DIR, $self->{path}) or die $!;
+
+    # read metrics base dir and store contents in array
+    my @days;
+    my @lunations;
+    while (my $dir = readdir(DIR)) {
+
+        next if ($dir =~ m/^\./);
+        next if (-f "$self->{path}/$dir");
+
+        if ($dir =~ m/[0-9]{4}-[0-9]{2}-[0-9]{2}_.*/) {push(@lunations, $dir);}
+        else { push(@days, $dir);}
+
+    }
+    closedir(DIR);
+
+    # sort array then write to HTML
+    print $htmlFile "<h2>Lunations</h2>\n";
+    @lunations = sort{$b cmp $a}(@lunations);
+    foreach my $lunation (@lunations) {
+
+        print "$lunation\n";
+        print $htmlFile "<a href=\"$lunation/index.html\">$lunation</a><br>\n";
+    }
+
+    # sort array then write to HTML
+    my @months = qw( NA Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec );
+    @days = sort{$b cmp $a}(@days);
+    my $lastMonth = "NA";
+    my $lastYear = "NA";
+    foreach my $day (@days) {
+
+        if ($day =~ m/^([0-9]{4})-([0-9]{2})-[0-9]{2}$/) {
+
+            # print year, if changes
+            if ($1 ne $lastYear) {
+      
+                print $htmlFile "<h2>$1</h2>";
+                $lastYear = $1;
+            }
+            # print month, if changes
+            if ($2 ne $lastMonth) {
+      
+                print $htmlFile "<h3>".$months[$2]."</h3>";
+                $lastMonth = $2;
+            }
+            print "$day\n";
+            print $htmlFile "<a href=\"$day/index.html\">$day</a><br>\n";
+        }
+    }
+
+    $self->finishHtml();
+
+    return 1;
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/MultiDayMetrics.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/MultiDayMetrics.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/MultiDayMetrics.pm	(revision 32093)
@@ -0,0 +1,76 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+use czartool::Metrics;
+
+
+package czartool::MultiDayMetrics;
+our @ISA = qw(czartool::Metrics);    # inherits from Metrics class
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new(
+            $_[1],  # config object 
+            $_[2],  # verbose
+            $_[3]   # save_temps
+            );
+
+    $self->{startDay} = $_[4];
+    $self->{endDay} = $_[5];
+
+    # sort out times
+    $self->{begin} =  "$self->{startDay} " . $self->{config}->getMetricsStartTime();
+    $self->{end} =  "$self->{endDay} " . $self->{config}->getMetricsStartTime();
+
+    # create path, dir and html file
+    $self->{path} = "$self->{baseDir}/$self->{startDay}_$self->{endDay}";
+    rmdir($self->{path});
+    mkdir($self->{path}, 0777);
+    $self->{plotter}->setOutputPath($self->{path});
+
+    if ($self->{verbose}) {print "* Creating metrics for $self->{startDay} to $self->{endDay}\n";}
+
+    bless $self, $class;
+
+    return $self;
+}
+
+###########################################################################
+#
+# Writes HTML document
+#
+###########################################################################
+sub writeHTML {
+    my ($self) = @_;
+
+    $self->createHtml("IPP Metrics for $self->{startDay} to $self->{endDay}");
+    my $htmlFile = $self->{htmlFile};
+
+    print $htmlFile "<h5  align=\"middle\"><a href=\"../index.html\">all</a><br> </h5>";
+
+    # summary plots
+    $self->createSummaryPlots();
+
+    # magic mask plots
+    $self->createMaskPlots($self->{startDay}, $self->{endDay});
+
+    $self->finishHtml();
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/MySQLDb.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/MySQLDb.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/MySQLDb.pm	(revision 32093)
@@ -0,0 +1,326 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use DBI;
+
+package czartool::MySQLDb;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = { 
+        _dbName => shift,
+        _dbHost => shift,
+        _dbUser => shift,
+        _dbPass => shift,
+        _verbose => shift,
+        _save_temps => shift,
+    };                              
+
+    my $dbname = $self->{_dbName};
+    my $dbhost = $self->{_dbHost};
+    my $dbuser = $self->{_dbUser};
+    my $dbpass = $self->{_dbPass};
+
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+
+     $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
+            "mysql_socket=" . DB_SOCKET(),
+            ${dbuser},${dbpass},
+            { RaiseError => 1, AutoCommit => 1}
+            );
+
+    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
+
+    bless $self, $class;                                
+    return $self;                                           
+}                     
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}
+
+sub setDbHost {
+    my ( $self, $dbHost ) = @_;                              
+    $self->{_dbHost} = $dbHost if defined($dbHost);        
+    return $self->{_dbHost};                            
+}                                                                       
+
+sub getDbHost {
+    my( $self ) = @_;                                                       
+    return $self->{_dbHost};                                                 
+}                                                                               
+
+###########################################################################
+#
+# Returns a date formatted wit hthe current formatting
+#
+###########################################################################
+sub getFormattedDate {
+    my ($self, $time) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DATE_FORMAT('$time','$self->{_dateFormat}'); 
+SQL
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns a full MySQL formatted date
+#
+###########################################################################
+sub getFullDate {
+    my ($self, $time) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DATE_FORMAT('$time','%Y-%m-%d %H:%i:%s'); 
+SQL
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Takes two times, time1 and time2 and finds the middle point
+#
+###########################################################################
+sub getMiddleTime {
+    my ($self, $time1, $time2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT '$time1' + INTERVAL (TIME_TO_SEC(TIMEDIFF('$time2', '$time1')) / 2) SECOND; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Adds the provided interval to the provided time
+#
+###########################################################################
+sub addInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' + INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Formats a date/time string in a way suitable for TIMEDIFF calls
+#
+###########################################################################
+sub formatTimeForDiff {
+    my ($self, $time) = @_;
+
+    if (${$time} =~ m/^[0-9]{4}-[0-9]{2}-[0-9]{2}.*/) {${$time} = $self->getFullDate(${$time});}
+    elsif(${$time} !~ m/^[0-9]+:[0-9]{2}:[0-9]{2}$/) {return 0;}
+
+    return 1;
+}
+
+###########################################################################
+#
+# Finds the difference of two times
+#
+# Should be able to handle most date/time formats
+#
+###########################################################################
+sub diffTimes {
+    my ($self, $time1, $time2) = @_;
+
+    if (!$time1 || !$time2) {return 0;}
+
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIMEDIFF('$time1','$time2'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the provided time in seconds
+#
+###########################################################################
+sub getTimeInSecs {
+    my ($self, $time) = @_;
+
+    if (!$time) {return 0;}
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC('$time'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Finds the difference of two times in seconds
+#
+###########################################################################
+sub diffTimesInSecs {
+    my ($self, $time1, $time2) = @_;
+
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2')); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns whether the first interval is larger than the second
+#
+###########################################################################
+sub isIntervalGreaterThan {
+    my ($self, $interval1, $interval2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT INTERVAL('$interval1', '$interval2'); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns whether time1 is before time2
+#
+###########################################################################
+sub isBefore {
+    my ($self, $time1, $time2) = @_;
+
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT TIME_TO_SEC(TIMEDIFF('$time1', '$time2'));
+SQL
+    $query->execute;
+
+    return (scalar $query->fetchrow_array() >= 0) ? 0 : 1;
+}
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getNowTimestamp {
+    my ($self) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now(); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Optimizes a table 
+#
+#######################################################################################
+sub optimizeTable {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    OPTIMIZE TABLE $table; 
+SQL
+
+    my $success = $query->execute;
+
+    print "* ";
+    if (!$success) {print "UN";}
+    print "successfully optimized '$table' table\n";
+
+    return $success;
+}
+
+#######################################################################################
+# 
+# Checks whether a certain table exists 
+#
+#######################################################################################
+sub doesTableExist {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM information_schema.tables 
+        WHERE table_schema = '$self->{_dbName}' 
+        AND table_name = '$table';
+SQL
+
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Destructor 
+#
+#######################################################################################
+sub DESTROY {
+    my($self) = @_;                                                       
+
+    if ($self->{_verbose}) {
+        print "* " . ref($self) . " is disconnecting from " . $self->{_dbName} . "@" . $self->{_dbHost} . "\n";
+    }
+    #$self->{_db}->disconnect();
+}
+1;                                        
+
Index: trunk/ippMonitor/czartool/czartool/Nebulous.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Nebulous.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Nebulous.pm	(revision 32093)
@@ -0,0 +1,162 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::Nebulous;
+
+###########################################################################
+#
+# Constructor TODO put in check that we can run nebulous
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {_czarDb => shift};
+
+    $self->{_totalAvail} = undef;
+    $self->{_totalUsed} = undef;
+    $self->{_totalUsedAsPercent} = undef;
+    $self->{_totalCluster} = undef;
+    $self->{_totalHosts} = undef;
+    $self->{_hostsOverNinetyEightPC} = undef;
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Converts Kb to Tb 
+# 
+###########################################################################
+sub convertKbToTb {
+    my ($self, $Kb) = @_;
+
+    return $Kb/1073741824;
+}
+
+###########################################################################
+#
+# Returns total cluster usage as percentage
+#
+###########################################################################
+sub getTotalUsedAsPercentage {
+    my ($self) = @_;
+
+    return $self->{_totalUsedAsPercent};
+}
+
+###########################################################################
+#
+# Returns total cluster usage in T 
+#
+###########################################################################
+sub getTotalUsedInTb {
+    my ($self) = @_;
+
+    return $self->convertKbToTb($self->{_totalUsed});
+}
+
+###########################################################################
+#
+# Returns total cluster space in T 
+#
+###########################################################################
+sub getTotalClusterInTb {
+    my ($self) = @_;
+
+    return $self->convertKbToTb($self->{_totalCluster});
+}
+
+###########################################################################
+#
+# Returns total cluster available in T 
+#
+###########################################################################
+sub getTotalAvailInTb {
+    my ($self) = @_;
+
+    return $self->convertKbToTb($self->{_totalAvail});
+}
+
+###########################################################################
+#
+# Runs neb-df and puts output in gnuplot ready format 
+#
+###########################################################################
+sub updateClusterSpaceInfo {
+    my ($self) = @_;
+
+    my @cmdOut = `neb-df`;
+    my $line;
+    $self->{_totalHosts} = 0;
+    $self->{_hostsOverNinetyEightPC} = 0;
+    my $readable;
+    my $writable;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/.*Filesystem.*/i) {next;}
+        elsif ($line =~ m/.*summmary.*/i) {next;}
+        elsif ($line =~ m/.*total.*/i) {next;}
+        elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) {
+
+            $self->{_totalUsed} = $self->convertKbToTb($1);
+            $self->{_totalAvail} = $self->convertKbToTb($2);
+            $self->{_totalCluster} = $self->convertKbToTb($1 + $2);
+            $self->{_totalUsedAsPercent} = ($self->{_totalUsed}/$self->{_totalCluster})*100;
+
+            next;
+        }
+        elsif (($line =~ m/(ipp[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
+	       ($line =~ m/(ippb[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
+        
+            $self->{_totalHosts}++;
+            if ($4 >= 98) {$self->{_hostsOverNinetyEightPC}++;}
+
+            $self->getReadableWritableStatus($1, \$readable, \$writable);
+
+            $self->{_czarDb}->updateHost($1, 
+                    $self->convertKbToTb($2+$3), 
+                    $self->convertKbToTb($3), 
+                    $self->convertKbToTb($2), 
+                    $readable, $writable);
+        }
+    }
+
+    $self->{_czarDb}->insertNewClusterSpace(
+            $self->{_totalCluster}, 
+            $self->{_totalAvail},  
+            $self->{_totalUsed}, 
+            $self->{_hostsOverNinetyEightPC});
+}
+
+###########################################################################
+#
+# Runs neb-host and readable/writable status for this host 
+#
+###########################################################################
+sub getReadableWritableStatus {
+    my ($self, $host, $readable, $writable) = @_;
+
+    ${$readable} = 0;
+    ${$writable} = 0;
+
+    my @cmdOut = `neb-host`;
+    my $line;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/$host\s+[0-9.A-Za-z]+\s+([01]+)\s+([01]+)\s+([01]+)\s+.*/i) {
+
+            # if mounted...
+            if ($1) {
+
+                ${$readable} = $3;
+                ${$writable} = $2;
+            }
+        }
+    }
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/Pantasks.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Pantasks.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Pantasks.pm	(revision 32093)
@@ -0,0 +1,243 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::Pantasks;
+
+my @servers = (
+        "addstar", 
+        "cleanup", 
+        "detrend", 
+        "distribution", 
+        "pstamp", 
+        "update", 
+        "publishing", 
+        "registration", 
+        "replication", 
+        "stdscience", 
+        "summitcopy"
+        );
+
+###########################################################################
+#
+# Constructor TODO put in check that we can run pantasks_client
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {};
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Returns the server list as an array 
+#
+###########################################################################
+sub getServerList {
+    my ($self) = @_;
+
+    return \@servers;
+}
+
+###########################################################################
+#
+# Returns the correct server for this processing stage 
+#
+###########################################################################
+sub getServerForThisStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "destreak" or $stage eq "dist") {return "distribution";}
+    return "stdscience";
+}
+
+###########################################################################
+#
+# Checks for a meaningful response from pantasks client 
+#
+###########################################################################
+sub outputOk {
+    my ($self, $output) = @_;
+
+    if (
+            $output =~ m/.*show\.labels.*/i ||
+            $output =~ m/.*Connection reset by peer.*/i ||
+            $output =~ m/.*server is busy.*/i ) { 
+
+        print "* WARNING: pantasks returned '$output'\n";
+        return 0;
+    }
+    return 1;
+}
+
+###########################################################################
+#
+# Gets nightlyscience status for today 
+#
+###########################################################################
+sub getNightlyScienceStatus {
+    my ($self, $status) = @_;
+
+    my @labels;
+
+    my ($day, $month, $year) = (localtime)[3,4,5];
+    my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day);
+
+    my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c ~ipp/stdscience/ptolemy.rc 2>&1`;
+    my $line;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if (!$self->outputOk($line)) {return 0;}
+        if ($line =~ m/$today\s+(.+)/) {
+            ${$status} = $1;
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+###########################################################################
+#
+# Gets labels for provided server 
+#
+###########################################################################
+sub getLabels {
+    my ($self, $server) = @_;
+
+    my @labels;
+
+    my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if (!$self->outputOk($line)) {return \@labels;}
+
+        # sometimes a label can be on the same line as the panstasks prompt
+        if ($line =~ m/pantasks: (.*)/) {
+            $line = $1;
+            $passedHeader=1;
+        }
+
+        if ($passedHeader && length($line) > 0){push(@labels, $line);}
+    }
+
+    return \@labels;
+}
+
+###########################################################################
+#
+# Returns whether this server is alive and running 
+#
+###########################################################################
+sub getServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    ${$alive} = 0;
+    ${$running} = 0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/Scheduler is stopped/) {${$running} = 0; ${$alive}=1;last;}
+        if ($line =~ m/Scheduler is running/) {${$running} = 1; ${$alive}=1;last;}
+        if ($line =~ m/Task Status/) {last;}
+    }
+}
+
+###########################################################################
+#
+# Gets server status 
+#
+###########################################################################
+sub getServerCurrentStatus {
+    my ($self, $server) = @_;
+
+    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    print "\n";
+    foreach $line (@cmdOut) {
+
+        if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;}
+        if ($passedHeader){print "$line";}
+    }
+}
+
+###########################################################################
+#
+# Gets revert status for this stage 
+#
+###########################################################################
+sub getRevertStatus {
+    my ($self, $stage, $reverting) = @_;
+
+    my $server = $self->getServerForThisStage($stage);
+
+    if ($stage eq "cam") {$stage = "camera";}
+
+    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/\s+([+-])[+-]\s+$stage\.revert.*/) {
+
+            if ($1 =~ m/-/) {${$reverting} = 0;}
+            elsif ($1 =~ m/\+/) {${$reverting}=1;}
+        }
+    }
+}
+
+###########################################################################
+#
+# Gets current dates for this server
+#
+###########################################################################
+sub getDates {
+    my ($self, $server) = @_;
+
+    my @dates;
+
+    my $prefix;
+
+    if ($server eq "stdscience") {$prefix = "ns";}
+    elsif ($server eq "registration") {$prefix = "register";}
+
+    my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if (!$self->outputOk($line)) {return \@dates;}
+
+        # sometimes content is on same line as pantasks prompt
+        if ($line =~ m/pantasks: (.*)/) {
+            $line =~ s/pantasks: //; 
+                $passedHeader=1;
+        }
+
+        if ($passedHeader && length($line) > 0){
+
+            # strip out only the date part
+            if ($line =~ m/([0-9]{4}-[0-9]{2}-[0-9]{2}).*/) {
+
+                push(@dates, $1);
+            }
+        }
+    }
+
+    return \@dates;
+}
+
+
+1;
Index: trunk/ippMonitor/czartool/czartool/Plotter.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/Plotter.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/Plotter.pm	(revision 32093)
@@ -0,0 +1,843 @@
+#!/usr/bin/perl -w
+
+package czartool::Plotter;
+
+use warnings;
+use strict;
+
+use File::Temp;
+
+use czartool::Config;
+
+my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub");
+
+###########################################################################
+#
+# Main constructor with all arguments
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {
+        _config => shift,
+        _dateFormat => shift,
+        _outputFormat => shift,
+        _outputPath => shift,
+        _save_temps => shift,
+    };
+
+    bless $self, $class;
+    $self->init();
+    return $self;
+}
+
+###########################################################################
+#
+# Constructor sending all plots to file
+#
+###########################################################################
+sub new_file {
+    my $class = shift;
+    my $self = {
+        _config => shift,
+        _outputPath => shift,
+        _save_temps => shift,
+                                                    };
+    $self->{_dateFormat} = "%Y%m%d-%H%M%S";
+    $self->{_outputFormat} = "png font \"".$self->{_config}->getGnuplotFont()."\" ".$self->{_config}->getGnuplotFontSize();
+
+    bless $self, $class;
+    $self->init();
+    return $self;
+}
+
+###########################################################################
+#
+# Constructor sending all plots to an X window
+#
+###########################################################################
+sub new_display {
+    my $class = shift;
+    my $self = {
+        _config => shift,
+        _save_temps => shift,
+    };
+
+    $self->{_dateFormat} = "%Y%m%d-%H%M%S";
+    $self->{_outputFormat} = "X11";
+    $self->{_outputPath} = undef;
+
+    bless $self, $class;
+    $self->init();
+    return $self;
+}
+
+###########################################################################
+#
+# Some initialisation code common to all constructors
+#
+###########################################################################
+sub init {
+    my ($self) = @_;
+
+    $self->{_gpc1Db} = $self->{_config}->getGpc1Instance();
+    $self->{_czarDb} = $self->{_config}->getCzarDbInstance();
+}
+
+###########################################################################
+#
+# Generates a suitable filename for this plot
+#
+###########################################################################
+sub createImageFileName {
+    my ($self, $label, $stage, $suffix, $isLog, $isRate) = @_;
+
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
+    my $stagePart = $stage ? $stage : "all_stages";
+    my $type = $isLog ? "log" : "linear";
+    return $prefix . "/czarplot_" . $type . "_" . $label . "_" . $stagePart . "_$suffix.png";
+}
+
+###########################################################################
+#
+# Figures out a suitable interval for rate plots
+# Could use a function here, but instead we want to choose meaningful intervals
+#
+###########################################################################
+sub getSuitableInterval {
+    my ($self, $begin, $end, $interval) = @_;
+
+    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);
+
+    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "26:00:00")) {${$interval} = "1 HOUR";} # under 1 day
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "72:00:00")) {${$interval} = "4 HOUR";} # under 3 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "96:00:00")) {${$interval} = "5 HOUR";} # under 4 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "144:00:00")) {${$interval} = "6 HOUR";} # under 6 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "168:00:00")) {${$interval} = "12 HOUR";} # under week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "672:00:00")) {${$interval} = "1 DAY";} # under 2 week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "2016:00:00")) {${$interval} = "1 WEEK";} # under 3 month
+    else {${$interval} = "1 MONTH";}
+
+# ${$interval} = "2 HOUR";
+
+    return 1;
+}
+
+###########################################################################
+#
+# Plots a time series of processing rate for all stages for this label TODO duplication below. combine
+#
+###########################################################################
+sub createRateHistogram {
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;
+
+    # stages
+    my $stages = undef;                 
+    if (!$selectedStage) {
+        $stages = \@allStages;
+        $selectedStage = 'all_stages';
+    }
+    else {$stages = ["$selectedStage"]};
+
+    # interval
+    if (!$interval && !$self->getSuitableInterval($beginTime, $endTime, \$interval)) {return 0;}
+
+    # set up files
+    my %gnuplotFiles;
+    my $stage = undef;
+    my $gnuplotFile = undef;
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = 0;
+
+    foreach $stage (@{$stages}) {
+
+        if ($self->{_czarDb}->createProcessingRateData(
+                    $stage, 
+                    $label, 
+                    $beginTime, 
+                    $endTime, 
+                    $interval,
+                    \$gnuplotFile,
+                    \$minX, \$maxX, \$timeDiff)) {
+
+            $gnuplotFiles{$stage} = $gnuplotFile;
+            #print "$minX, $maxX, $timeDiff\n";
+        }
+    }
+
+    my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0);
+    $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval);
+
+    # now delete temp dat files
+    foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}
+
+    return 1;
+}
+
+###########################################################################
+#
+# Plots a stacked histogram of rate data for one or more stages
+#
+###########################################################################
+sub plotRateStackedHistogram {
+    my ($self, $gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"'$label' for '$selectedStage'\\nFrom '$beginTime' to '$endTime' HST\";" .
+        "set boxwidth;" .
+        "set xtic rotate by -90 scale 0;" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures processed per $interval\";" .
+        "set boxwidth 0.75;" .
+        "plot ";
+
+    my $first = 1;
+    foreach my $stage (@allStages) {
+
+
+        if(!$gnuplotFiles->{$stage}) {next;}
+
+        if (!$first) { print GP ","; }
+        print GP "'" . $gnuplotFiles->{$stage}. "' using 4:xtic(1) title \"$stage\" ";
+        $first = 0;
+    }
+
+    print GP ";\n";
+    close GP;
+
+    return 1;
+}
+
+###########################################################################
+#
+# Plots a time series for all stages for this label
+#
+###########################################################################
+sub createTimeSeries {
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $linear, $log, $rate) = @_;
+
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = 0;
+
+    my $stages = undef;                 
+
+    if (!$selectedStage) {$stages = \@allStages;}
+    else {$stages = ["$selectedStage"]};        
+
+    my %linDataFiles;
+    my %logDataFiles;
+    my %ratDataFiles;
+    my $stage = undef;
+    my $linDataFile = undef;
+    my $logDataFile = undef;
+    my $ratDataFile = undef;
+
+    foreach $stage (@{$stages}) {
+
+        if($self->{_czarDb}->createTimeSeriesData(
+                    $label, 
+                    $stage, 
+                    $beginTime, 
+                    $endTime, 
+                    \$minX, 
+                    \$maxX, 
+                    \$timeDiff, 
+                    \$linDataFile,
+                    \$logDataFile,
+                    \$ratDataFile)) {
+
+            $linDataFiles{$stage} = $linDataFile;
+            $logDataFiles{$stage} = $logDataFile;
+            $ratDataFiles{$stage} = $ratDataFile;
+        }
+    }                                                           
+
+    my $numOfPlots =  keys(%linDataFiles);
+
+    if ($numOfPlots == 0 ) {
+
+#        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
+        return;
+    } 
+
+    if ($timeDiff == 0) {
+
+        print "WARNING: Zero time difference for '$label' during time period '$beginTime', '$endTime'\n";
+        return;
+    }
+
+    if ($linear) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 0);
+
+        $self->plotTimeSeries(
+                \%linDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Exposures");
+    }
+
+    if ($log) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 1);
+
+        $self->plotTimeSeries(
+                \%logDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Log(Exposures)");
+    }
+
+    if ($rate) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0);
+
+        $self->plotTimeSeries(
+                \%ratDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Exposures per hour");
+}    
+}                                                
+
+###########################################################################
+#
+# Plots a histogram of stuff processed in provided interval and label for all stages
+#
+###########################################################################
+sub createHistogram {
+    my ($self, $label, $beginTime, $endTime) = @_;
+
+    my $outputFile = createImageFileName($self, $label, undef, "h", 0);
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_h.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    my ($processed, $pending, $faults);
+    my $stage = undef;
+    my $pendingMinusFaults = undef;
+    foreach $stage (@allStages) {
+
+        if (!$self->{_czarDb}->countProcessedPendingAndFaults(
+                $label, 
+                $stage, 
+                $beginTime, 
+                $endTime, 
+                \$processed, 
+                \$pending, 
+                \$faults)) {next;}
+
+        $pendingMinusFaults = $pending - $faults; 
+        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
+    }
+
+    close(GNUDAT);
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"'$label'\\nFrom '$beginTime' to '$endTime' HST\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures\";" .
+        "set boxwidth 0.75;" .
+        "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
+        "\n";
+
+    close GP;
+}
+
+###########################################################################
+#
+# Sets the output path 
+#
+###########################################################################
+sub setOutputFormat {
+    my ($self, $outputFormat) = @_;
+    $self->{_outputFormat} = $outputFormat if defined($outputFormat);
+}           
+
+###########################################################################
+#
+# Sets the output type 
+#
+###########################################################################
+sub setOutputPath {
+    my ($self, $outputPath) = @_;
+    $self->{_outputPath} = $outputPath if defined($outputPath);
+}           
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}       
+
+###########################################################################
+#
+# Figures out suitable spacing for time tics on time-series plots
+#
+###########################################################################
+sub getTimeSpacing {
+    my ($self, $timeDiff, $divX, $timeFormat) = @_;
+
+    # if less than a few mins of data, show 1 minute tics
+    if ($timeDiff < 300) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 60;
+    }
+    # if less than a few mins of data, show 5 minute tics
+    elsif ($timeDiff < 1800) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 300;
+    }
+    # if less than an hour of data, show 15 minute tics
+    elsif ($timeDiff < 3600) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 900;
+    }
+    # if less than a couple of hour's data plotted, show 30 mins tics
+    elsif ($timeDiff < 7200) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 1800;
+    }
+    # if less than half a day's data plotted, show hourly tics
+    elsif ($timeDiff < 43200) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 3600;
+    }
+    # if less than one day's data plotted, show 2 hourly tics
+    elsif ($timeDiff < 86400) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 7200;
+    }
+    # if less than 1 week's data is data plotted, show daily tics
+    elsif ($timeDiff < 604800) {
+
+        ${$timeFormat} = "%m/%d";
+        ${$divX} = 86400;
+    }
+    # if less than 1 months's data is data plotted, show weekly ticks
+    elsif ($timeDiff < 2419200) {
+
+        ${$timeFormat} = "%m/%d";
+        ${$divX} = 604800;
+    }
+    # otherwise, show monthly ticks
+    else {
+
+        ${$timeFormat} = "%b";
+        ${$divX} = 2419200;
+    }
+}
+
+###########################################################################
+#
+# Plots a time-series of pending/faults
+#
+###########################################################################
+sub plotTimeSeries {
+    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $title, $yTitle) = @_;
+
+    my $timeFormat = undef;
+    my $divX = undef;
+
+    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
+
+    my $numOfPlots = keys %$gnuplotFiles;
+
+    # sort out plot title
+    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
+    else {$title .= "'all stages'"}
+
+    $title .= " for '$label'\\nFrom '$fromTime' to '$toTime' HST";
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"$title\";" .
+        "set key left top;" .
+        "set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+        "set format x \"$timeFormat\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+        "set grid xtics;" .
+        "set grid ytics;" .
+        "set xlabel \"Time\";" .
+        "set ylabel \"$yTitle\";" .
+        "plot ";
+
+    my $firstIn = 1;
+    # loop through stages array so that they are ordered properly (not maintained in hash)
+    foreach my $stage (@allStages) {
+
+        if(!$gnuplotFiles->{$stage}) {next;}
+        if (!$firstIn) {print GP ",";}
+
+        # for single stage plots, show pending/processed/faults
+        if ($numOfPlots == 1) {
+
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 1 lw 2";
+        }
+        # when plotting multiple stages, show only processed
+        else {
+
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines lw 2";
+        }
+        $firstIn = 0;
+    }
+
+    print GP "\n";
+    close GP;
+
+    # now delete temp dat files
+    foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
+}
+
+###########################################################################
+#
+# Plots a time-series of cluster storage
+#
+###########################################################################
+sub plotStorageTimeSeries {
+    my ($self, $fromTime, $toTime) = @_;
+
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
+    my $outputFile = "$prefix/czarplot_cluster.png";
+    my ($minX, $maxX, $minY, $maxY, $timeDiff);
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    if (!$self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff)) {return 0;}
+
+    my $timeFormat = undef;
+    my $divX = undef;
+
+    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    my $datFile = $tmpFile->filename;
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP <<PLOT;
+        set term $self->{_outputFormat}
+        set title "Total available cluster space over time\\nFrom $fromTime to $toTime HST"
+        set key left top
+        set xdata time
+        set timefmt "$self->{_dateFormat}"
+        set format x "$timeFormat"
+        set xtics "$minX", $divX, "$maxX"
+        set grid xtics
+        set xlabel "Time"
+        set ylabel "Available (TB)"
+        plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2
+PLOT
+
+    print GP "\n";
+    close GP;
+
+    return 1;
+}
+
+###########################################################################
+#
+# Plots disk usage across cluster as a stacked histogram
+#
+###########################################################################
+sub plotDiskUsageHistogram {
+    my ($self) = @_;
+
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $outputFile = "$prefix/czarplot_hosts_space.png";
+    my $limit = 97.0;
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $self->{_czarDb}->createHostsData($limit, $tmpFile);
+
+    my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();
+
+    my $totalPercent = sprintf("%.1f%%", $totalUsed);
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
+        "set style fill solid 1.00 border -1;" .
+        "set style histogram rowstacked;" .
+        "set style data histograms;" .
+        "set ylabel \"Space (TB)\";" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot '".$tmpFile->filename."' " .
+        "using 2:xtic(1) t \"Used\" lt 7," .
+        "'' using 3 t \"Over $limit% used\" lt 1," . 
+        "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," . 
+        "'' using 5 t \"Free\" lt 2," . 
+        "'' using 6 notitle fs solid 0.50 lt -1" . 
+        ";" .
+
+        "\n";
+
+    close GP;
+}
+
+###########################################################################
+#
+# Generates 2D plot of magic mask fraction for provided exposure ID
+#
+###########################################################################
+sub plotMagicMaskFractionForThisExposure {
+    my ($self, $exp_id) = @_;
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $outputFile = "$prefix/czarplot_single_exp_magic_mask_fraction.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_single_exp_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => '.dat');
+
+    my $fracs;
+    if (!$self->{_gpc1Db}->getMagicMaskFraction($exp_id, \$fracs)) {return 0;}
+
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    # missing corner chip
+    print GNUDAT "0 0 0.0\n";
+    foreach my $row ( @{$fracs} ) {
+
+        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
+            print GNUDAT "$1 $2 @{$row}[1]\n";
+            # missing corner chips
+            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
+            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
+        }
+    }
+    # missing corner chip
+    print GNUDAT "7 7 0.0\n";
+
+
+    close(GNUDAT);
+
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    my $datFile = $tmpFile->filename;
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP <<PLOT;
+
+    set term $self->{_outputFormat}
+    set title "Magic mask fraction for exposure $exp_id"
+        set grid
+        set size square
+        set xrange [-0.5:7.5] reverse 
+        set yrange [-0.5:7.5]  
+        unset key
+        set palette rgb 22,13,10
+        set ylabel "Y"
+        set xlabel "X"
+        set cbrange [0:1]
+        set datafile missing "NaN"
+        plot "$datFile" using 1:2:3  with image
+PLOT
+
+    close GP;
+
+    return 1;
+}
+
+
+###########################################################################
+#
+# Plots mask fraction histogram for exposures taken between these dates
+#
+###########################################################################
+sub plotMagicMaskFraction {
+    my ($self, $begin, $end) = @_;
+
+    # get exposures between these dates
+    my $exp_ids = undef;
+    if (!$self->{_gpc1Db}->getProcessedExposures($begin, $end, \$exp_ids)) {return 0;}
+    my $totalExp = @{$exp_ids};
+    if ($totalExp < 1) {return 0;}
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $histoOutputFile = "$prefix/czarplot_magic_mask_fraction_h.png";
+    my $distOutputFile = "$prefix/czarplot_magic_mask_fraction_d.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+
+    # set up bins
+    my %histogram; 
+    my @bins;
+    my $interval = 0.05;
+    my $min = 1.0;
+    my $max = 1.0;
+    for (my $n=0.0; $n<$max; $n += $interval){
+
+        push(@bins, $n);
+        $histogram{$n} = 0;
+    }
+
+    my $meanMask = undef;
+    my $sumMask = undef;
+    my $chipCount = undef;
+    my $expCount = 0;
+    my $totalChipCount = 0;
+    my $totalMask = 0;
+    my $exp_id = undef;
+
+    # get mask for each exposure, and bin
+    foreach my $row ( @{$exp_ids} ) {
+
+        $exp_id = @{$row}[0];
+
+        if (!$self->{_gpc1Db}->getMagicMaskStats($exp_id, \$meanMask, \$sumMask, \$chipCount )) {next;}
+
+
+        if (!$meanMask) {next;}
+        #print "expId=$exp_id meanMask=$meanMask sumMask=$sumMask nChips=$chipCount\n";
+        $expCount++;
+        $totalMask += $sumMask;
+        $totalChipCount += $chipCount;
+
+        foreach my $bin (@bins) {
+            if ($meanMask <= ($bin+$interval)) {
+
+                $histogram{$bin}++;
+                last;
+            }
+        }
+    }
+
+    my $totalMaskFrac;
+    
+    if ($totalChipCount > 0) {
+
+        $totalMaskFrac = sprintf( "%.1f", ($totalMask/$totalChipCount) * 100.0);
+    }
+    else {
+
+        $totalMaskFrac = "NA";
+    }
+
+    # write data to GNUPLOT data file
+    print "* Found mask values for $expCount exposures out of $totalExp\n";
+    open (GNUDAT, ">".$tmpFile->filename);
+    my $accum = 0;
+    my $maxBin = 0;
+    foreach my $bin (@bins) {
+
+        $accum += $histogram{$bin};
+        if ($histogram{$bin} > $maxBin) {$maxBin = $histogram{$bin};}
+        print GNUDAT "$bin $histogram{$bin} $accum\n";
+    }
+    close(GNUDAT);
+
+
+    $maxBin = $maxBin * 1.1;
+    $accum = $accum * 1.1;
+
+    my $title = "Magic Mask Fraction for $expCount exposures\\nData taken between '$begin' and '$end'\\nTotal masked fraction is $totalMaskFrac%";
+
+    # make histogram
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"$title\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set yrange [0:$maxBin];" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set boxwidth 0.75;" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot '".$tmpFile->filename."' using 2:xtic(1) notitle lt 1;" . 
+        "\n";
+
+    close GP;
+
+    # make cumulative distribution
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Cumulative distribution of $title\";" .
+        "set yrange [0:$accum];" .
+        "set key left top;" .
+        "set grid xtics;" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot " . 
+        "'".$tmpFile->filename."' using 1:3 notitle with lines lt 2 lw 2\n";
+
+    print GP "\n";
+    close GP;
+
+    return 1;
+}
+1;
Index: trunk/ippMonitor/czartool/czartool/StageMetrics.pm
===================================================================
--- trunk/ippMonitor/czartool/czartool/StageMetrics.pm	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/StageMetrics.pm	(revision 32093)
@@ -0,0 +1,129 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::StageMetrics;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {
+
+        stage => shift,
+        label => shift,
+        startTime => shift,
+        endTime => shift
+
+    };
+
+    $self->{started} = undef;
+    $self->{stuck} = undef;
+    $self->{processed} = undef;
+    $self->{initialPending} = undef;
+    $self->{finalPending} = undef;
+    $self->{faults} = undef;
+    $self->{totalTime} = undef;
+    $self->{finished25} = undef;
+    $self->{finished50} = undef;
+    $self->{finished75} = undef;
+    $self->{finished90} = undef;
+    $self->{finished95} = undef;
+    $self->{finished100} = undef;
+    $self->{rate} = undef;
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Returns whether anything has been processed or not
+#
+###########################################################################
+sub anythingProcessed() {
+    my $self = shift; 
+
+    if (!$self->{processed}) {return 0;}
+    if ($self->{processed} < 1) {return 0;}
+    return 1;
+}
+
+###########################################################################
+#
+# Prints all metrics
+#
+###########################################################################
+sub printMe() {
+    my $self = shift;
+
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+    print "|     stage     |       started       |       finished      | time taken | processed | pending |  rate | faults | stuck? |\n";
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+    printf( "| %10s    ", $self->{stage} ? $self->{stage} : "no");
+    printf( "| %20s", $self->{started} ? $self->{started} : "no");
+    #printf( "| %s", $self->{finished25} ? $self->{finished25} : "no");
+    #printf( "| %s", $self->{finished50} ? $self->{finished50} : "no");
+    #printf( "| %s", $self->{finished75} ? $self->{finished75} : "no");
+    #printf( "| %s", $self->{finished90} ? $self->{finished90} : "no");
+    #printf( "| %s", $self->{finished95} ? $self->{finished95} : "no");
+    printf( "| %20s", $self->{finished100} ? $self->{finished100} : "no");
+    printf( "| %10s ", $self->{totalTime} ? $self->{totalTime} : "na");
+    printf( "| %5d     ", $self->{processed} ? $self->{processed} : 0);
+    printf( "| %5d   ", $self->{pending} ? $self->{pending} : 0);
+    printf( "| %4.2f ", $self->{rate} ? $self->{rate} : 0);
+    printf( "| %5d  ", $self->{faults} ? $self->{faults} : 0);
+    printf( "| %4s   ", $self->{stuck} ? "YES" : "no");
+    print "|\n";
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+}
+
+###########################################################################
+#
+# Setters for all class variables
+#
+###########################################################################
+sub setStarted() {my $self = shift; $self->{started} = shift;}
+sub setStuck() {my $self = shift; $self->{stuck} = shift;}
+sub setProcessed() {my $self = shift; $self->{processed} = shift;}
+sub setInitialPending() {my $self = shift; $self->{initialPending} = shift;}
+sub setFinalPending() {my $self = shift; $self->{finalPending} = shift;}
+sub setFaults() {my $self = shift; $self->{faults} = shift;}
+sub setTotalTime() {my $self = shift; $self->{totalTime} = shift;}
+sub setFinished25() {my $self = shift; $self->{finished25} = shift;}
+sub setFinished50() {my $self = shift; $self->{finished50} = shift;}
+sub setFinished75() {my $self = shift; $self->{finished75} = shift;}
+sub setFinished90() {my $self = shift; $self->{finished90} = shift;}
+sub setFinished95() {my $self = shift; $self->{finished95} = shift;}
+sub setFinished100() {my $self = shift; $self->{finished100} = shift;}
+sub setRate() {my $self = shift; $self->{rate} = shift;}
+
+###########################################################################
+#
+# Getters for all class variables
+#
+###########################################################################
+sub getStartTime() {my $self = shift; return $self->{startTime};}
+sub getEndTime() {my $self = shift; return $self->{endTime};}
+sub getLabel() {my $self = shift; return $self->{label};}
+sub getStage() {my $self = shift; return $self->{stage};}
+sub getStarted() {my $self = shift; return $self->{started};}
+sub getStuck() {my $self = shift; return $self->{stuck};}
+sub getProcessed() {my $self = shift; return $self->{processed};}
+sub getInitialPending() {my $self = shift; return $self->{initialPending};}
+sub getFinalPending() {my $self = shift; return $self->{finalPending};}
+sub getFaults() {my $self = shift; return $self->{faults};}
+sub getTotalTime() {my $self = shift; return $self->{totalTime};}
+sub getFinished25() {my $self = shift; return $self->{finished25};}
+sub getFinished50() {my $self = shift; return $self->{finished50};}
+sub getFinished75() {my $self = shift; return $self->{finished75};}
+sub getFinished90() {my $self = shift; return $self->{finished90};}
+sub getFinished95() {my $self = shift; return $self->{finished95};}
+sub getFinished100() {my $self = shift; return $self->{finished100};}
+sub getRate() {my $self = shift; return $self->{rate};}
+
+1;
Index: trunk/ippMonitor/czartool/czartool/czarconfig.xml
===================================================================
--- trunk/ippMonitor/czartool/czartool/czarconfig.xml	(revision 32093)
+++ trunk/ippMonitor/czartool/czartool/czarconfig.xml	(revision 32093)
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Global config for all czartool stuff -->
+
+<czarconfig>
+
+  <!-- Gnuplot config section -->
+  <gnuplot>
+    <font>/usr/share/fonts/corefonts/arial.ttf</font>
+    <size>8</size>
+  </gnuplot>
+
+  <!-- Metrics section -->
+  <metrics>
+    <savelocation>/data/ipp004.0/ipp/ippMetrics</savelocation>
+    <starttime>18:00</starttime>
+  </metrics>
+
+  <!-- gpc1 Db section -->
+  <gpc1database>
+    <name>gpc1</name>
+    <host>ippdb01</host>
+    <user>ippuser</user>
+    <password>ippuser</password>
+  </gpc1database>
+
+  <!-- Czar Db section -->
+  <czardatabase>
+    <name>czardb</name>
+    <host>ippdb01</host>
+    <user>ipp</user>
+    <password>ipp</password>
+    <cleanupinterval>30 MINUTE</cleanupinterval>
+  </czardatabase>
+
+  <!-- Roboczar section -->
+  <roboczar>
+    <email>ps-ipp-ops@ifa.hawaii.edu</email>
+    <!-- How often should ropbczar check status? (MySQL 'interval' format please, eg 20 MINUTE, 1 HOUR etc) check-->
+    <serverinterval>20 MINUTE</serverinterval>
+    <!-- whitespace-separated server list for servers we want roboczar to check --> 
+    <interestedservers>stdscience distribution summitcopy registration pstamp</interestedservers>
+    <!-- <interestedservers>pstamp</interestedservers> -->
+  </roboczar>
+
+</czarconfig>