Index: trunk/tools/czarplot.pl
===================================================================
--- trunk/tools/czarplot.pl	(revision 28704)
+++ trunk/tools/czarplot.pl	(revision 28704)
@@ -0,0 +1,126 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use File::Temp qw(tempfile);
+use POSIX qw/strftime/;
+
+use czartool::CzarDb;
+use czartool::Gnuplot;
+
+
+my $czarDbName = "czardb";
+my $label = undef;
+my $stage = undef;
+my $save_temps = undef;
+my $interval = undef;
+my $verbose = undef;
+my $histogram = undef;
+my $timeSeries = undef;
+
+GetOptions (
+        "dbname|d=s" => \$czarDbName,
+        "label|l=s" => \$label,
+        "stage|s=s" => \$stage,
+        "interval|i=s" => \$interval,
+        "histogram|h" => \$histogram,
+        "timeseries|t" => \$timeSeries,
+        "verbose|v" => \$verbose,
+        );
+
+print "\n";
+if (!$histogram) {
+    print "* OPTIONAL: plot histogram         -h                          (default=on)\n";
+}
+if (!$timeSeries) {
+    print "* OPTIONAL: plot timeseries        -t                          (default=on)\n";
+} 
+if (!$label) {
+    print "* OPTIONAL: choose a label         -l <labellName>             (default='all_labels')\n";
+}
+if (!$stage) {
+    print "* OPTIONAL: choose a stage         -s <chip|cam|warp|etc>      (default=none)\n";
+}
+
+if (!$interval) {
+    print "* OPTIONAL: choose an interval     -i <'1 hour'|'1 day'|etc>   (default=interval since 7am this morning)\n";
+} 
+
+print "\n";
+
+# default values
+if (!$label) {$label = "all_labels";}
+if (!$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 1;}
+if (!$verbose) {$verbose = 0;}
+if (!$save_temps) {$save_temps = 0;}
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose);
+my $gnuplot = new czartool::Gnuplot("%Y%m%d-%H%M%S", "X11", "histo.pbm");
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+my $toTime = $czarDb->getNowTimestamp();
+my $fromTime = undef;
+
+if ($interval) {$fromTime = $czarDb->getIntervalInPast($interval);}
+else {$fromTime =  strftime('%Y-%m-%d 07:00',localtime);}
+
+if ($histogram) {plotHistogram($label, $fromTime, $toTime);}
+if ($timeSeries) {plotTimeSeries($label, $stage, $fromTime, $toTime); }
+
+###########################################################################
+#
+# Plots a time series for all stages for this label
+#
+###########################################################################
+sub plotTimeSeries {
+    my ($label, $selectedStage, $fromTime, $toTime) = @_;
+
+    my ($minX, $maxX, $minY, $maxY, $timeDiff);
+    $minX = 999999999;
+    $maxX = -9999999999;
+    $minY = 999999999;
+    $maxY = -99999999;
+
+    my $stages = undef;
+
+    if (!$selectedStage) {$stages = \@allStages;}
+    else {$stages = ["$selectedStage"]};
+
+    my %gnuplotFiles;
+    foreach $stage (@{$stages}) {
+        $gnuplotFiles{$stage} = $czarDb->createTimeSeriesData($save_temps, $label, $stage, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
+    }
+
+    if (!$selectedStage) {$gnuplot->plotAllStagesTimeSeries($label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, \%gnuplotFiles);}
+    else {$gnuplot->plotSingleTimeSeries($gnuplotFiles{$selectedStage}, $selectedStage, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff);}
+
+}
+
+###########################################################################
+#
+# Plots a histogram of stuff processed in provided interval and label for all stages
+#
+###########################################################################
+sub plotHistogram {
+    my ($label, $fromTime, $toTime) = @_;
+
+    my ($tempFile, $filePath) = tempfile( "/tmp/czartool_gnuplot_histogram.XXXX", UNLINK => !$save_temps);
+    my ($processed, $pending, $faults);
+
+    foreach $stage (@allStages) {
+
+        $czarDb->countProcessedPendingAndFaults($label, $stage, $fromTime, $toTime, \$processed, \$pending, \$faults);
+        print $tempFile "$stage $processed, $pending, $faults\n";
+    }
+
+    close($tempFile);
+
+    $gnuplot->plotHistogram($filePath, $label, $fromTime, $toTime);
+}
+
+
+
Index: trunk/tools/czartool.pl
===================================================================
--- trunk/tools/czartool.pl	(revision 28701)
+++ trunk/tools/czartool.pl	(revision 28704)
@@ -3,26 +3,35 @@
 use warnings;
 use strict;
-use PS::IPP::Config 1.01 qw( :standard );
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-use IPC::Cmd 0.36 qw( can_run run );
-use DBI;
-use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
-use File::Temp qw(tempfile);
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+
+
+my $czarDbName = "czardb";
+my $save_temps = 1;
+my $interval = undef;
+
+GetOptions ( 
+        "dbname|d=s" => \$czarDbName,
+        "interval|i=s" => \$interval,
+        );
+
 
 my @states = ("full", "new", "drop", "wait");
-my $db = connectToDb();
-
-if (!$db) {die;}
-
-my @stdscienceLabels = `czartool_getLabels.pl -s stdscience`;
-my @distributionLabels = `czartool_getLabels.pl -s distribution`;
-my @publishingLabels = `czartool_getLabels.pl -s publishing`;
-
-checkAllLabels("new");
-printInstructions();
-
-poll();
-$db->disconnect();
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp");
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $pantasks = new czartool::Pantasks();
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @stdscienceLabels = undef;
+my @distributionLabels = undef;
+my @publishingLabels = undef;
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); 
+promptPoll();
+
 
 ###########################################################################
@@ -41,5 +50,11 @@
 #
 ###########################################################################
-sub poll {
+sub promptPoll {
+
+    @stdscienceLabels = @{$pantasks->getLabels("stdscience")};
+    @distributionLabels = @{$pantasks->getLabels("distribution")};
+    @publishingLabels = @{$pantasks->getLabels("publishing")};
+    checkAllLabels("new");
+    printInstructions();
 
     my $key;
@@ -60,25 +75,4 @@
 ###########################################################################
 #
-# Connects to the database
-#
-###########################################################################
-sub connectToDb {
-
-    my $dbname = 'gpc1';
-    my $dbserver = 'ippdb01';
-    my $dbuser = 'ippuser';
-    my $dbpass = 'ippuser';
-    my $db = DBI->connect(
-            "DBI:mysql:database=${dbname};host=${dbserver};mysql_socket=" .
-            DB_SOCKET(),${dbuser},${dbpass},
-            {
-            RaiseError => 1, AutoCommit => 1}
-            ) or die "Unable to connect to database $DBI::errstr\n";
-
-    return $db;
-}
-
-###########################################################################
-#
 # Compares two arrays and stores the differences
 #
@@ -120,5 +114,5 @@
 
         chomp($line);
-    
+
         print "$line\n";
     }
@@ -163,5 +157,5 @@
     chomp($input);
     if ($input =~ m/^([0-9]|[1-9][0-9]|[1-9][0-9][0-9])$/) {
-       return checkOneLabel($stdscienceLabels[$input]);
+        return checkOneLabel($stdscienceLabels[$input]);
     }
 
@@ -179,11 +173,12 @@
     chomp($label);
 
-    printf("\n+-------------------------------------------------------------------------------------+\n");
-    printf("|                        %32s                             |\n", $label);
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
-    printf("|   state    |  chip |  cam  |  fake | warp  | stack |  diff | magic |destreak|  dist |\n");
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
+    printf("\n+---------------------------------------------------------------------------------------------------------------------------------+\n");
+    printf("|                                                  %32s                                               |\n", $label);
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+    printf("|   state    |    chip    |    cam     |    fake    |    warp    |   stack    |    diff    |   magic    |  destreak  |    dist    |\n");
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
 
     my $state;
+    my $stage;
     my $i=0;
     foreach $state (@states) {
@@ -191,18 +186,15 @@
         chomp($label);
         printf("| %10s ", $state);
-        printf("| %5s ", getStateAndFaults($label, "chipRun", $state, "chip"));
-        printf("| %5s ", getStateAndFaults($label, "camRun", $state, "cam"));
-        printf("| %5s ", getStateAndFaults($label, "fakeRun", $state, "fake"));
-        printf("| %5s ", getStateAndFaults($label, "warpRun", $state, "warp"));
-        printf("| %5s ", getStateAndFaults($label, "stackRun", $state, "stack"));
-        printf("| %5s ", getStateAndFaults($label, "diffRun", $state, "diff"));
-        printf("| %5s ", getStateAndFaults($label, "magicRun", $state, "magic"));
-        printf("| %6s ", getStateAndFaults($label, "magicDSRun", $state, "magicDS"));
-        printf("| %5s ", getStateAndFaults($label, "distRun", $state, "dist"));
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($label, $state, $stage));
+        }
+
         printf("|\n");
         $i++;
     }
 
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
 
 }
@@ -225,7 +217,8 @@
     my $distLabel;
     my $pubLabel;
-    my $i=1;
     my $distributing;
     my $publishing;
+    my $stage;
+    my $i=1;
     foreach $stdsLabel (@stdscienceLabels) {
 
@@ -246,13 +239,10 @@
         printf("| %10s    ", $distributing ? "yes" : "NO" );
         printf("| %10s    ", $publishing ? "yes" : "NO" );
-        printf("| %10s ", getStateAndFaults($stdsLabel, "chipRun", $state, "chip"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "camRun", $state, "cam"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "fakeRun", $state, "fake"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "warpRun", $state, "warp"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "stackRun", $state, "stack"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "diffRun", $state, "diff"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "magicRun", $state, "magic"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "magicDSRun", $state, "magicDS"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "distRun", $state, "dist"));
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($stdsLabel, $state, $stage));
+        }
+
         printf("|\n");
         $i++;
@@ -265,15 +255,15 @@
 ###########################################################################
 #
-# Returns state and fault-count (if new) as a string
-#
-###########################################################################
-sub getStateAndFaults {
-    my ($label, $table, $state, $stage) = @_;
-
-    my $new = checkLabel($label, $table, $state, $stage);
+# Returns state and fault-count (if new) as a string TODO untidy
+#
+###########################################################################
+sub getStateAndFaultsString {
+    my ($label, $state, $stage) = @_;
+
+    my $new = $gpc1Db->countExposures($label, $stage, $state);
 
     if ($state ne "new") {return $new;}
 
-    my $faults = countFaults($label,$table,$stage);
+    my $faults = $gpc1Db->countFaults($label, $stage);
 
     if ($faults < 1) {return $new;}
@@ -282,63 +272,3 @@
 }
 
-###########################################################################
-#
-# Returns count of exposures with this state for this label
-#
-###########################################################################
-sub checkLabel {
-    my ($label, $table, $state, $stage) = @_;
-
-    if ($state eq "fault") {return countFaults($label, $table, $stage);}
-
-    my $query = $db->prepare(<<SQL);
-    SELECT count(state)  
-        FROM $table 
-        WHERE label LIKE '$label' 
-        AND state = '$state'
-
-SQL
-
-        $query->execute;
-    return scalar $query->fetchrow_array();
-}
-
-###########################################################################
-#
-# Returns count of faults for this stage and label
-#
-###########################################################################
-sub countFaults {
-    my ($label, $table, $stage) = @_;
-
-    my $joinTable;
-    my $id = $stage."_id";
-
-    if ($stage eq "chip") {$joinTable="chipProcessedImfile";}
-    elsif ($stage eq "cam") {$joinTable="camProcessedExp";}
-    elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";}
-    elsif ($stage eq "warp") {$joinTable="warpSkyfile";}
-    elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";}
-    elsif ($stage eq "diff") {$joinTable="diffSkyfile";}
-    elsif ($stage eq "magic") {$joinTable="magicNodeResult";}
-    elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
-    elsif ($stage eq "dist") {$joinTable="distComponent";}
-    else {return -1;}
-
-    my $faultCol =  $joinTable.".fault";
-    my $stateCol =  $table.".state";
-
-    my $query = $db->prepare(<<SQL);
-    SELECT COUNT(DISTINCT $id) 
-        FROM $table
-        JOIN $joinTable USING ($id)
-        WHERE label LIKE '$label'
-        AND $faultCol != 0
-        AND $stateCol = 'new'
-
-SQL
-
-        $query->execute;
-    return scalar $query->fetchrow_array();
-}
-
+
Index: trunk/tools/czartool/CzarDb.pm
===================================================================
--- trunk/tools/czartool/CzarDb.pm	(revision 28704)
+++ trunk/tools/czartool/CzarDb.pm	(revision 28704)
@@ -0,0 +1,506 @@
+#!/usr/bin/perl i-w
+
+package czartool::CzarDb;
+
+
+use warnings;
+use strict;
+
+use File::Temp qw(tempfile);
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
+
+
+use base 'czartool::MySQLDb';
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+# Override constructor
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class, Person.
+    my $self = $class->SUPER::new( $_[1], $_[2], $_[3], $_[4],  $_[5]);
+
+    bless $self, $class;
+
+    $self->update();    
+    return $self;
+}
+
+###########################################################################
+#
+# Gets current_labels table
+#
+###########################################################################
+sub getCurrentLabels {
+    my ($self, $server) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT label
+        FROM current_labels
+        WHERE server LIKE '$server';
+SQL
+
+    $query->execute;
+
+    return $query->fetchall_arrayref();
+}
+
+###########################################################################
+#
+# Updates server table
+#
+###########################################################################
+sub updateServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO servers
+        (server, alive, running)
+        VALUES
+        ('$server', $alive, $running);
+SQL
+
+       $query->execute;
+}
+###########################################################################
+#
+# Updates current_labels table
+#
+###########################################################################
+sub updateCurrentLabels {
+    my ($self, $server, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM current_labels WHERE server LIKE '$server';
+SQL
+
+    $query->execute;
+
+    my $label = undef;
+
+    foreach $label (@{$labels}) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        INSERT INTO current_labels
+            (server, label)
+            VALUES
+            ('$server', '$label');
+SQL
+
+       $query->execute;
+    }
+}
+###########################################################################
+#
+# Inserts new time data into relevant table for a given label
+#
+###########################################################################
+sub insertNewTimeData {
+    my ($self, $table, $label, $pending, $processed, $faults, $reverting) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO $table
+        (label, pending, processed, faults, reverting)
+        VALUES
+        ('$label', $pending, $processed, $faults, $reverting);
+SQL
+
+        $query->execute;
+}
+
+
+###########################################################################
+#
+# Gets timedifference in seconds for these two times 
+#
+###########################################################################
+sub getTimeDifference { # TODO fix this
+    my ($self, $save_temps, $fromTime, $toTime) = @_; # TODO save_temps should be constructor arg
+
+    my ($tempFile, $tempName) = tempfile( "/tmp/czartool_gnuplot_timeseries.XXXX", UNLINK => !$save_temps);
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        TIME_TO_SEC(TIMEDIFF($fromTime, $toTime))
+SQL
+
+    $query->execute;
+
+    return $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Gets time series data and store to temp file
+#
+###########################################################################
+sub createTimeSeriesData {
+    my ($self, $save_temps, $label, $table, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_; # TODO save_temps should be constructor arg
+
+    my ($tempFile, $tempName) = tempfile( "/tmp/czartool_gnuplot_timeseries.XXXX", UNLINK => !$save_temps);
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MIN(processed), GREATEST(MAX(pending), MAX(faults), max(processed)-min(processed)) AS maxY, 
+        LEAST(MIN(pending), MIN(faults), max(processed)-min(processed)) AS minY,
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
+        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
+            FROM $table 
+            WHERE label LIKE '$label'
+            AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    $query->execute;
+    my $minProcessed;
+
+    ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed-$minProcessed 
+        FROM $table 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    $query->execute;
+
+    # loop round results
+    while (my @row = $query->fetchrow_array()) {
+
+        my ($timestamp, $pending, $faults, $processed) = @row;
+        print $tempFile "$timestamp $pending $faults $processed\n";
+    }
+    close($tempFile);
+
+    return $tempName;
+}
+
+###########################################################################
+#
+# Returns number of faults at a particular time in the past for a given label and stage
+#
+###########################################################################
+sub countFaultsInPast {
+    my ($self, $label, $table, $interval) = @_; # TODO use time not interval
+
+    # get earliest time for this label
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval) 
+        FROM $table
+        WHERE label LIKE '$label'; 
+SQL
+    $query->execute;
+
+my $timestamp = $query->fetchrow_array(); 
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT faults 
+        FROM $table
+        WHERE label LIKE '$label' 
+        AND timestamp <= '$timestamp' LIMIT 1;
+SQL
+    $query->execute;
+
+return $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures that are pending right now for a given stage and label
+#
+###########################################################################
+sub countPendingNow {
+    my ($self, $label, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT pending - faults 
+        FROM $table 
+        WHERE label LIKE '$label' 
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+        $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Creates histogram data
+#
+###########################################################################
+sub countProcessedPendingAndFaults {
+    my ($self, $label, $table, $fromTime, $toTime, $processed, $pending, $faults) = @_;
+
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        pending, faults 
+        FROM $table
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+
+    $query->execute;
+    (${$pending}, ${$faults}) = $query->fetchrow_array();    
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MAX(processed) - MIN(processed) 
+        FROM $table 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+    $query->execute;
+    (${$processed}) = $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Determines how much has been processed in the provided interval of time 
+# (format: 1 HOUR, 1 MINUTE 1 DAY etc)
+#
+###########################################################################
+sub countProcessed { # TODO use time not interval
+    my ($self, $label, $table, $interval) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MAX(processed) - MIN(processed) 
+        FROM $table 
+        WHERE label LIKE '$label' 
+        AND timestamp > (now() - INTERVAL $interval); 
+SQL
+        $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Update Db to newest version 
+#
+###########################################################################
+sub update { 
+    my ($self) = @_;
+
+    my $currentRevision = -1;
+    my $latestRevision = 6;
+
+    while ($currentRevision != $latestRevision) {
+
+        $currentRevision = $self->getRevision();
+        if ($self->{_verbose}) {print "* Current Db revision = $currentRevision\n";}
+
+        if ($currentRevision == 0) {$self->createRevision_1();}
+        elsif ($currentRevision == 1) {$self->createRevision_2();}
+        elsif ($currentRevision == 2) {$self->createRevision_3();}
+        elsif ($currentRevision == 3) {$self->createRevision_4();}
+        elsif ($currentRevision == 4) {$self->createRevision_5();}
+        elsif ($currentRevision == 5) {$self->createRevision_6();}
+    }
+}
+
+#######################################################################################
+# 
+# Create revision 1 of the database 
+#
+#######################################################################################
+sub createRevision_1 {
+    my ($self) = @_;
+
+    print "* Creating revision 1 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE revision (
+            revision INT, 
+            created TIMESTAMP DEFAULT NOW(),
+            primary key (revision)
+            );
+SQL
+        $query->execute;
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        CREATE TABLE $stage (
+                timestamp TIMESTAMP DEFAULT NOW(),
+                label VARCHAR(128) DEFAULT "NONE", 
+                pending BIGINT NOT NULL,
+                faults BIGINT NOT NULL
+                );
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(1);
+}
+
+#######################################################################################
+# 
+# Create revision 2 of the database 
+#
+#######################################################################################
+sub createRevision_2 {
+    my ($self) = @_;
+
+    print "* Creating revision 2 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(2);
+}
+
+#######################################################################################
+# 
+# Create revision 3 of the database 
+#
+#######################################################################################
+sub createRevision_3 {
+    my ($self) = @_;
+
+    print "* Creating revision 3 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
+SQL
+
+      $query->execute;
+    }
+
+    $self->setRevision(3);
+}
+
+#######################################################################################
+# 
+# Create revision 4 of the database 
+#
+#######################################################################################
+sub createRevision_4 {
+    my ($self) = @_;
+
+    print "* Creating revision 4 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE current_labels (server VARCHAR(128), label VARCHAR(128));
+SQL
+
+      $query->execute;
+
+    $self->setRevision(4);
+}
+
+#######################################################################################
+# 
+# Create revision 5 of the database 
+#
+#######################################################################################
+sub createRevision_5 {
+    my ($self) = @_;
+
+    print "* Creating revision 5 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE servers (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            server VARCHAR(128), 
+            alive TINYINT, running TINYINT);
+SQL
+
+      $query->execute;
+
+    $self->setRevision(5);
+}
+
+#######################################################################################
+# 
+# Create revision 6 of the database 
+#
+#######################################################################################
+sub createRevision_6 {
+    my ($self) = @_;
+
+    print "* Creating revision 6 of '$self->{_dbName}'\n";
+
+    my $stage = undef;
+    my $index = undef;
+    foreach $stage (@stages) {
+
+        $index = $stage . "Index";
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX $index ON $stage (timestamp, label);
+SQL
+
+      $query->execute;
+    }
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX serverIndex ON servers (timestamp, server);
+SQL
+
+      $query->execute;
+
+
+    $self->setRevision(6);
+}
+
+#######################################################################################
+# 
+# Sets current revision of ippToPsps database
+#
+#######################################################################################
+sub setRevision {
+    my ($self, $revision) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO revision (revision) VALUES ($revision);
+SQL
+    $query->execute;
+}
+
+#######################################################################################
+# 
+# Gets current revision of ippToPsps database
+#
+#######################################################################################
+sub getRevision {
+    my ($self) = @_;
+
+    if (!$self->SUPER::doesTableExist("revision")) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT revision
+        FROM revision
+        ORDER BY revision DESC LIMIT 1;
+SQL
+
+    $query->execute;
+    my @row = $query->fetchrow_array();
+
+    return $row[0];
+}
+
+1;
+
Index: trunk/tools/czartool/Gnuplot.pm
===================================================================
--- trunk/tools/czartool/Gnuplot.pm	(revision 28704)
+++ trunk/tools/czartool/Gnuplot.pm	(revision 28704)
@@ -0,0 +1,163 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::Gnuplot;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {
+        _dateFormat => shift,
+        _outputFormat => shift,
+        _outputPath => shift,
+    };
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Sets the ouput path to be X11 
+#
+###########################################################################
+sub setOutputFormat {
+    my ($self, $outputFormat, $outputPath) = @_;
+    $self->{_outputFormat} = $outputFormat if defined($outputFormat);
+    $self->{_outputPath} = $outputPath if defined($outputPath);
+}           
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}       
+
+###########################################################################
+#
+# Plots a time-series of pending/faults
+#
+###########################################################################
+sub plotAllStagesTimeSeries {
+    my ($self, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, $gnuplotFiles) = @_;
+
+    $maxY = $maxY + ($maxY/10);
+    $minY = $minY - ($minY/10);
+    if ($maxY == 0) {$maxY = 1;}
+    if ($maxY == 0) {$maxY = 1;}
+    if ($minY == 0) {$minY = -1;}
+
+    my $divX = $timeDiff/2;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$self->{_outputPath}\";" .
+        "set title \"All stages for '$label' since $fromTime\";" .
+        "set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+        #"set xrange [\"$minX\":\"$maxX\"];" .
+        #"set yrange [$minY:$maxY];" .
+        "set format x \"%m/%d %H:%M\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+        "set grid xtics;" .
+        "set xlabel \"Time\";" .
+        "set ylabel \"Exposures\";" .
+        "plot ";
+
+    my $firstIn = 1;
+    foreach my $stage (keys %$gnuplotFiles) {
+        if (!$firstIn) {print GP ",";}
+        print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines";
+        $firstIn = 0;
+    }
+
+    print GP "\n";
+    close GP;
+}
+
+###########################################################################
+#
+# Plots a time-series of pending/faults
+#
+###########################################################################
+sub plotSingleTimeSeries {
+    my ($self, $filePath, $stage, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff) = @_;
+
+    $maxY = $maxY + ($maxY/10);
+    $minY = $minY - ($minY/10);
+    if ($maxY == 0) {$maxY = 1;}
+    if ($maxY == 0) {$maxY = 1;}
+    if ($minY == 0) {$minY = -1;}
+
+    my $divX = $timeDiff/2;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$self->{_outputPath}\";" .
+        "set title \"'$stage' stage for '$label' since $fromTime\";" .
+        "set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+        "set xrange [\"$minX\":\"$maxX\"];" .
+        "set yrange [$minY:$maxY];" .
+        "set format x \"%m/%d %H:%M\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+        "set grid;" .
+        "set xlabel \"Time\";" .
+        "set ylabel \"Exposures\";" .
+        "plot '$filePath' using 1:4 title \"processed\" with lines lt 2," .
+        "'$filePath' using 1:2 title \"pending\" with lines lt 3," .
+        "'$filePath' using 1:3 title \"faults\" with lines lt 1\n";
+    close GP;
+}
+
+###########################################################################
+#
+# Plots a histogram of processed stuff
+#
+###########################################################################
+sub plotHistogram {
+    my ($self, $filePath, $label, $fromTime, $toTime) = @_;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$self->{_outputPath}\";" .
+        "set title \"Processing status for $label since $fromTime\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set style histogram cluster gap 3;" .
+        "set style fill solid border -1;" .
+#        "set boxwidth 1.5;" .
+        "set bmargin 5;" .
+        "set xtic rotate by -90;" .
+#        "plot '$filePath' using 2:xtic(1) notitle\n";
+        "set ylabel \"Exposures\";" .
+        "plot '$filePath' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 3, '' using 4 title \"Faults\" lt 1\n";
+
+    close GP;
+}
+
+1;
Index: trunk/tools/czartool/Gpc1Db.pm
===================================================================
--- trunk/tools/czartool/Gpc1Db.pm	(revision 28704)
+++ trunk/tools/czartool/Gpc1Db.pm	(revision 28704)
@@ -0,0 +1,95 @@
+#!/usr/bin/perl i-w
+
+package czartool::Gpc1Db;
+
+use warnings;
+use strict;
+
+use czartool::MySQLDb;
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+###########################################################################
+#
+# Returns the right table for a given stage TODO make private
+#
+###########################################################################
+sub getTableForStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "chip" ) {return "chipRun";}
+    elsif ($stage eq "cam" ) {return "camRun";}
+    elsif ($stage eq "fake" ) {return "fakeRun";}
+    elsif ($stage eq "warp" ) {return "warpRun";}
+    elsif ($stage eq "stack" ) {return "stackRun";}
+    elsif ($stage eq "diff" ) {return "diffRun";}
+    elsif ($stage eq "magic" ) {return "magicRun";}
+    elsif ($stage eq "magicDS" ) {return "magicDSRun";}
+    elsif ($stage eq "dist" ) {return "distRun";}
+
+    return "ERROR";
+}
+
+
+###########################################################################
+#
+# Returns count of faults for this stage and label
+#
+###########################################################################
+sub countFaults {
+    my ($self, $label, $stage) = @_;
+
+
+    my $table = getTableForStage($self, $stage);
+    my $joinTable;
+    my $id = $stage."_id";
+
+    if ($stage eq "chip") {$joinTable="chipProcessedImfile";}
+    elsif ($stage eq "cam") {$joinTable="camProcessedExp";}
+    elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";}
+    elsif ($stage eq "warp") {$joinTable="warpSkyfile";}
+    elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";}
+    elsif ($stage eq "diff") {$joinTable="diffSkyfile";}
+    elsif ($stage eq "magic") {$joinTable="magicNodeResult";}
+    elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
+    elsif ($stage eq "dist") {$joinTable="distComponent";}
+    else {return -1;}
+
+    my $faultCol =  $joinTable.".fault";
+    my $stateCol =  $table.".state";
+
+    my $query =  $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(DISTINCT $id) 
+        FROM $table
+        JOIN $joinTable USING ($id)
+        WHERE label LIKE '$label'
+        AND $faultCol != 0
+        AND $stateCol = 'new'
+SQL
+
+        $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns count of exposures with this state for this label
+#
+###########################################################################
+sub countExposures {
+    my ($self, $label, $stage, $state) = @_;
+
+    my $table = getTableForStage($self, $stage);
+    if ($state eq "fault") {return countFaults($stage);}
+
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT count(state)  
+        FROM $table 
+        WHERE label LIKE '$label' 
+        AND state = '$state'
+SQL
+
+        $query->execute;
+    return scalar $query->fetchrow_array();
+}
+1;
Index: trunk/tools/czartool/MySQLDb.pm
===================================================================
--- trunk/tools/czartool/MySQLDb.pm	(revision 28704)
+++ trunk/tools/czartool/MySQLDb.pm	(revision 28704)
@@ -0,0 +1,134 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use DBI;
+
+package czartool::MySQLDb;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = { 
+        _dbName => shift,
+        _dbHost => shift,
+        _dbUser => shift,
+        _dbPass => shift,
+        _verbose => shift,
+    };                              
+
+    my $dbname = $self->{_dbName};
+    my $dbhost = $self->{_dbHost};
+    my $dbuser = $self->{_dbUser};
+    my $dbpass = $self->{_dbPass};
+
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+
+     $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
+            "mysql_socket=" . DB_SOCKET(),
+            ${dbuser},${dbpass},
+            { RaiseError => 1, AutoCommit => 1}
+            );
+
+    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
+
+    bless $self, $class;                                
+    return $self;                                           
+}                     
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}
+
+sub setDbHost {
+    my ( $self, $dbHost ) = @_;                              
+    $self->{_dbHost} = $dbHost if defined($dbHost);        
+    return $self->{_dbHost};                            
+}                                                                       
+
+sub getDbHost {
+    my( $self ) = @_;                                                       
+    return $self->{_dbHost};                                                 
+}                                                                               
+
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getIntervalInPast {
+    my ($self, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now() - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getNowTimestamp {
+    my ($self) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now(); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Checks whether a certain table exists 
+#
+#######################################################################################
+sub doesTableExist {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM information_schema.tables 
+        WHERE table_schema = '$self->{_dbName}' 
+        AND table_name = '$table';
+SQL
+
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Destructor 
+#
+#######################################################################################
+sub DESTROY {
+    my($self) = @_;                                                       
+
+    if ($self->{_verbose}) {
+        print "* " . ref($self) . " is disconnecting from " . $self->{_dbName} . "@" . $self->{_dbHost} . "\n";
+    }
+    #$self->{_db}->disconnect();
+}
+1;                                        
+
Index: trunk/tools/czartool/Pantasks.pm
===================================================================
--- trunk/tools/czartool/Pantasks.pm	(revision 28704)
+++ trunk/tools/czartool/Pantasks.pm	(revision 28704)
@@ -0,0 +1,134 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::Pantasks;
+
+
+my @servers = (
+        "addstar", 
+        "cleanup", 
+        "detrend", 
+        "distribution", 
+        "pstamp", 
+        "update", 
+        "publishing", 
+        "registration", 
+        "replication", 
+        "stdscience", 
+        "summitcopy"
+        );
+
+
+###########################################################################
+#
+# Returns the server list as an array 
+#
+###########################################################################
+sub getServerList {
+    my ($self) = @_;
+
+    return \@servers;
+}
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {};
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Returns the correct server for this processing stage 
+#
+###########################################################################
+sub getServerForThisStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "destreak" or $stage eq "dist") {return "distribution";}
+    return "stdscience";
+}
+
+
+###########################################################################
+#
+# Gets labels for provided server 
+#
+###########################################################################
+sub getLabels {
+    my ($self, $server) = @_;
+
+
+    my @labels;
+
+    my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/pantasks:\s+(.*)/) {
+
+            # HACK to get around 'dummy' label in stdscience
+            if ($1 !~ m/.*dummy.*/) {push(@labels, $1);};
+            $passedHeader=1; 
+            next;
+        }
+
+        if ($passedHeader) {push(@labels, $line);}
+    }
+
+    return \@labels;
+}
+
+###########################################################################
+#
+# Returns whether this server is alive and running 
+#
+###########################################################################
+sub getServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    ${$alive} = 0;
+    ${$running} = 0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/Scheduler is stopped/) {${$running} = 0; ${$alive}=1;last;}
+        if ($line =~ m/Scheduler is running/) {${$running} = 1; ${$alive}=1;last;}
+        if ($line =~ m/Task Status/) {last;}
+
+    }
+}
+
+###########################################################################
+#
+# Gets server status 
+#
+###########################################################################
+sub getServerCurrentStatus {
+    my ($self, $server) = @_;
+
+
+    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;}
+        if ($passedHeader){print "$line<br>";}
+    }
+}
+
+
Index: trunk/tools/roboczar.pl
===================================================================
--- trunk/tools/roboczar.pl	(revision 28704)
+++ trunk/tools/roboczar.pl	(revision 28704)
@@ -0,0 +1,198 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+
+
+my $period = 60;
+my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
+my $save_temps = 1;
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp");
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $pantasks = new czartool::Pantasks();
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+GetOptions (
+        "period|p=s" => \$period, # TODO more Db args
+        "dbname|d=s" => \$czarDbName,
+        );
+
+timePoll($period);
+
+###########################################################################
+#
+# Updates the labels from pantasks for all interested servers 
+#
+###########################################################################
+sub updateLabels {
+
+    print "* Updating labels\n";
+    my @servers = ("stdscience", "distribution", "publishing");
+
+    my $server = undef;
+    foreach $server (@servers) {
+
+        my @labels = @{$pantasks->getLabels($server)};
+        $czarDb->updateCurrentLabels($server, \@labels);
+
+    }
+}
+
+
+###########################################################################
+#
+# Updates pantasks server status TODO should really get info for all servers at once 
+#
+###########################################################################
+sub updateServerStatus {
+    print "* Checking Servers\n";
+
+    my $servers = $pantasks->getServerList();
+
+    my $server = undef;
+    my $alive = undef;
+    my $running = undef;
+    foreach $server (@{$servers}) {
+
+        $pantasks->getServerStatus($server, \$alive, \$running);
+        $czarDb->updateServerStatus($server, $alive, $running);
+    }
+}
+
+###########################################################################
+#
+# Polls with provided period (seconds) 
+#
+###########################################################################
+sub timePoll {
+    my ($period) = @_;
+
+    my $label;
+    my $new;
+    my $full;
+    my $faults;
+    my $currentTime;
+    my $stage;
+    my ($totalNew,$totalFull,$totalFaults);
+    my $query = undef;
+    my $str = undef;
+    my $reverting = 0;
+    my $server = undef;
+
+    my $rows = undef; # TODO just call label
+        my $row;
+
+    while (1) {
+
+        updateServerStatus();
+        updateLabels();
+        $rows = $czarDb->getCurrentLabels("stdscience");
+
+        foreach $stage (@stages) {
+            $totalNew=$totalFaults=$totalFull=0;
+
+            $server = $pantasks->getServerForThisStage($stage);
+            $str = `czartool_revert.pl -s $server -t $stage`;
+            if ($str =~ m/on/) {$reverting = 1;}
+            else {$reverting = 0;}
+            print "* Storing for stage $stage\n";
+
+            foreach $row ( @{$rows} ) {
+                my ($label) = @{$row};
+
+                chomp($label);
+
+                $new = $gpc1Db->countExposures($label, $stage, "new");
+                $full = $gpc1Db->countExposures($label, $stage, "full");
+                $faults = $gpc1Db->countFaults($label, $stage);
+                $currentTime = time();
+
+                #printf("%ld, %s, %s, %d, %d\n", $currentTime, $label, $stage, $new, $faults);
+                $totalNew += $new;
+                $totalFull += $full;
+                $totalFaults += $faults;
+
+                $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults, $reverting);
+            }
+
+            $czarDb->insertNewTimeData($stage, "all_labels", $totalNew, $totalFull, $totalFaults, $reverting);
+        }
+
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+            #routineChecks($label, "1 HOUR");
+        }
+        print "--------------------------------------------------------------------------\n";
+        print "* Going to sleep\n";
+        sleep($period);
+        print "* Waking up\n";
+
+        #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
+    };
+}
+
+###########################################################################
+#
+# Performs some routine checks on processing status and sends alerts if it needs to 
+#
+###########################################################################
+sub routineChecks {
+    my ($label, $interval) = @_;
+
+    my $faultsNow;
+    my $faultsInPast;
+    my $newFaults;
+    my $pendingNow;
+    my $processedRecently;
+    my $stage = undef;
+
+    print "* Checking all stages for label $label\n";
+
+    foreach $stage (@stages) {
+
+        # check for increasing faults
+        $faultsNow = $czarDb->countFaultsInPast($label, $stage, "0 MINUTE");
+        $faultsInPast = $czarDb->countFaultsInPast($label, $stage, $interval);
+        if ($faultsNow > $faultsInPast) {
+            $newFaults = $faultsNow - $faultsInPast;
+            print "There have been $newFaults new faults in the last $interval (label='$label', stage='$stage')\n";
+        }
+
+        # check for lack of processing
+        $pendingNow =  $czarDb->countPendingNow($label, $stage);
+        $processedRecently = $czarDb->countProcessed($label, $stage, $interval);
+        if ($pendingNow > 0 && $processedRecently < 1) {
+
+            print "Only $processedRecently exposures have processed out of $pendingNow($faultsNow) pending in the last $interval (label='$label', stage='$stage')\n";
+
+        }
+    }
+}
+
+
+###########################################################################
+#
+# Sends an email 
+#
+###########################################################################
+sub sendEmail {
+    my ($to, $from, $subject, $message) = @_;
+
+    my $sendmail = '/usr/lib/sendmail';
+    open(MAIL, "|$sendmail -oi -t");
+    print MAIL "From: $from\n";
+    print MAIL "To: $to\n";
+    print MAIL "Subject: $subject\n\n";
+    print MAIL "$message\n";
+    close(MAIL);
+}
+
+