Index: branches/eam_branches/ipp-20101103/tools/checkfits
===================================================================
--- branches/eam_branches/ipp-20101103/tools/checkfits	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/checkfits	(revision 29834)
@@ -0,0 +1,99 @@
+#!/bin/env perl
+
+# Check whether a fits file can be unpacked successfully
+
+use strict;
+use warnings;
+#use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Basename;
+use File::Temp qw(tempfile);
+use Carp;
+
+use IPC::Cmd qw( can_run run );
+
+my ($filename, $verbose, $output);
+
+GetOptions(
+    'verbose|v'      => \$verbose, 
+    'output|o=s'     => \$output,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "uri is required", -exitval => 2 )
+    if !@ARGV;
+
+my $ipprc =  PS::IPP::Config->new();
+
+my $file = shift;
+
+my $scheme = file_scheme($file);
+
+my $resolved;
+if ($scheme and $scheme = 'neb') {
+    $resolved = $ipprc->file_resolve($file);
+    die "failed to resolve $file\n" if !$resolved;
+} else {
+    $resolved = $file;
+}
+
+if (!-e $resolved) {
+    die "$file not found\n";
+}
+
+
+my $tempfile;
+if (!$output) {
+    ($tempfile, $output) = tempfile("/tmp/checkfits.XXXX", UNLINK => 1);
+}
+
+my $funpack = can_run('funpack') or die "can't find funpack\n";
+
+my $command = "$funpack -S $resolved > $output";
+
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+if (!$success) {
+    my $exit_status = $error_code >> 8;
+    warn ("funpack of $resolved failed with exit status $exit_status.\n") if $verbose;
+    warn ("$resolved is corrupt.\n");
+    exit $exit_status;
+}
+
+print "$file unpack successful.\n" if $verbose;
+
+
+exit 0;
+
+__END__
+
+=pod
+
+=head1 NAME
+
+checkfits - see whether a fits file can be successfully decompressesd
+
+=head1 SYNOPSIS
+    
+    checkfits <uri> [--output <file>] [--verbose]
+
+
+=over 4
+
+
+=item * <uri> 
+
+Uri of file to be checked.  (URI may be a nebulous path).
+
+=item * --output <file>
+
+Save the unpacked fits to <file>.
+
+=item * --verbose
+
+Causes checkfits to be chatty.
+
+
+=cut
Index: branches/eam_branches/ipp-20101103/tools/czarmetrics.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czarmetrics.pl	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/czarmetrics.pl	(revision 29834)
@@ -0,0 +1,70 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::DayMetrics;
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+
+my $czarDbName = "czardb";
+my $day = undef;
+my $begin = undef;
+my $end = undef;
+my $verbose = undef;
+my $save_temps = undef;
+
+GetOptions (
+        "dbname=s" => \$czarDbName,
+        "begin|b=s" => \$begin,
+        "end|e=s" => \$end,
+        "day|y=s" => \$day,
+        "verbose|v" => \$verbose,
+        );
+
+print "\n*******************************************************************************\n";
+print "* \n";
+my $quit = 0;
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!$day) {
+    print "* OPTIONAL: choose a single day             -y <date>               (default=today)\n";
+}
+if (!$begin) {
+    print "* OPTIONAL: choose a begin date             -b <date>               (default=today)\n";}
+if (!$end) 
+{
+    print "* OPTIONAL: choose an end date              -e <date>               (default=today)\n";
+}
+
+print "*\n*******************************************************************************\n";
+
+if ($quit) {exit;}
+
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+
+if (!$day && !$begin && !$end) {
+    
+    $day = strftime('%Y-%m-%d', localtime);
+}
+if ($day) {$begin = $end = $day;}
+
+
+my $thisDay = $begin;
+while (1) {
+
+    my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay);
+    $dayMetrics->writeHTML();
+
+    $thisDay = $czarDb->addInterval($thisDay, "1 DAY");
+    if ($czarDb->isBefore($end, $thisDay)) {last;}
+}
Index: branches/eam_branches/ipp-20101103/tools/czarplot.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czarplot.pl	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czarplot.pl	(revision 29834)
@@ -8,5 +8,7 @@
 
 use czartool::CzarDb;
+use czartool::Gpc1Db;
 use czartool::Plotter;
+use czartool::StageMetrics;
 
 my $czarDbName = "czardb";
@@ -14,5 +16,5 @@
 my $stage = undef;
 my $save_temps = undef;
-my $interval = undef;
+my $timeinpast = undef;
 my $begin = undef;
 my $end = undef;
@@ -23,4 +25,6 @@
 my $timeSeries = undef;
 my $rate = undef;
+my $magicMask = undef;
+my $rateInterval = undef;
 my $deriv = undef;
 my $showCleanup = undef;
@@ -28,4 +32,5 @@
 my $savingToFile = undef;
 my $analysis = undef;
+my $exposureId = undef;
 my $log = undef;
 
@@ -34,5 +39,8 @@
         "label|l=s" => \$label,
         "stage|s=s" => \$stage,
-        "interval|i=s" => \$interval,
+        "timeinpast|p=s" => \$timeinpast,
+        "rateinterval|i=s" => \$rateInterval,
+        "exposureid|x=s" => \$exposureId,
+        "mask|m" => \$magicMask,
         "begin|b=s" => \$begin,
         "end|e=s" => \$end,
@@ -57,10 +65,17 @@
     print "* UNKNKOWN: option                          @ARGV\n";
 }
+if ($analysis && !$stage) {
+    $quit = 1;
+    print "* REQUIRED: choose a stage for analsis      -s <chip|cam|warp|etc>      (default=none)\n";}
 if (!$histogram) {
     print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
 if (!$timeSeries) {
     print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";} 
+if (!$magicMask) {
+    print "* OPTIONAL: plot magic mask for these times -m                          (default=off)\n";} 
+if (!$exposureId) {
+    print "* OPTIONAL: set exposure ID for magic mask  -x                          (default=none)\n";} 
 if (!$rate) {
-    print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";} 
+    print "* OPTIONAL: plot histogram of rate          -r                          (default=off)\n";} 
 if (!$deriv) {
     $deriv = 0;
@@ -74,5 +89,5 @@
 if (!$log) {
     $log = 0;
-    print "* OPTIONAL: use log plots                   -g                          (default=$log\n";}
+    print "* OPTIONAL: use log plots                   -g                          (default=$log)\n";}
 if (!$nebulous) {
     print "* OPTIONAL: plot nebulous disk space        -n                          (default=off)\n";} 
@@ -82,6 +97,8 @@
 if (!$stage) {
     print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
-if (!$interval) {
-    print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=none\n";} 
+if (!$timeinpast) {
+    print "* OPTIONAL: choose time interval in past    -p <'1 hour'|'1 day'|etc>   (default=none\n";} 
+if (!$rateInterval) {
+    print "* OPTIONAL: time interval for rate plot     -i <'1 hour'|'1 day'|etc>   (default=depends on time frame)\n";} 
 if (!$begin) {
     print "* OPTIONAL: choose a begin time             -b <datetime>               (default=6:35am this morning)\n";} 
@@ -98,5 +115,5 @@
 
 # default values
-if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
+if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;}
 if (!$verbose) {$verbose = 0;}
 if (!$save_temps) {$save_temps = 0;}
@@ -104,4 +121,5 @@
 else {$savingToFile = 1;}
 
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
@@ -109,4 +127,5 @@
 
 my $plotter = new czartool::Plotter(
+        $gpc1Db, 
         $czarDb, 
         "%Y%m%d-%H%M%S", 
@@ -121,4 +140,6 @@
     $begin =  "$day 06:35";
     $end = "$day 23:59";
+        print "JKJKJK '$begin' '$end' \n";
+
 }
 else {
@@ -127,5 +148,5 @@
     if (!$begin) {
 
-        if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
+        if ($timeinpast) {$begin = $czarDb->subtractInterval($end, $timeinpast);}
         else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
     }
@@ -134,7 +155,5 @@
 
 if ($rate) {
-
-    if (!$interval) {$interval = "1 HOUR";}
-    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $interval, $log);
+    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval, $log);
     exit;
 }
@@ -143,17 +162,14 @@
 if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
 elsif ($nebulous) {$plotter->plotDiskUsageHistogram();}
+if ($magicMask) {
+    
+    if ($exposureId) {$plotter->plotMagicMaskFractionForThisExposure($exposureId);}
+    else {$plotter->plotMagicMaskFractionHistogram($begin, $end);}
 
+}
 
 if($analysis) {
 
-    my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
-    $czarDb->runAnalysis($label, $stage, $begin, $end, \$started, \$finished, \$stuck, \$processed, \$pending, \$faults, \$totalTime);
-
-    if (defined $started) {print "* Processing started at $started\n";}
-    if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
-    else {print "* Processing has not finished\n";}
-    if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
-    print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
-
-    print "*******************************************************************************\n";
+    my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end);
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
 }
Index: branches/eam_branches/ipp-20101103/tools/czarpoll.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czarpoll.pl	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czarpoll.pl	(revision 29834)
@@ -13,4 +13,6 @@
 use czartool::Plotter;
 use czartool::Burntool;
+use czartool::DayMetrics;
+
 
 my $period = 60;
@@ -27,6 +29,7 @@
 my $nebulous = new czartool::Nebulous($czarDb);
 my $pantasks = new czartool::Pantasks();
-my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
+my $plotter = new czartool::Plotter($gpc1Db, $czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
 my $burntool = new czartool::Burntool();
+
 
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
@@ -107,4 +110,5 @@
     my $lastDay = strftime('%Y-%m-%d', localtime);
     my $today = undef;
+    my $doneMetricsToday = 1;
 
     while (1) {
@@ -118,4 +122,5 @@
                 $czarDb->optimize();
                 $lastDay = $today;
+                $doneMetricsToday = 0;
         }
 
@@ -124,4 +129,16 @@
         $end = $czarDb->getNowTimestamp();
 
+        # if time now is after 06:35am, then create metrics for past 24 hours
+        if (!$doneMetricsToday && $czarDb->isBefore($begin, $end)) {
+        
+                print "* Creating metrics for last 24 hours\n";
+                # TODO hardcopded path needs to be in config
+                my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $today); 
+                $dayMetrics->writeHTML();
+
+                $doneMetricsToday = 1;
+        }
+
+        # if time now is before 06:35am, include data from previous day
         if ($czarDb->isBefore($end, $begin)) {
 
@@ -197,6 +214,9 @@
             chomp($label);
             $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
-        }
-    }
+            $plotter->createRateTimeSeries($label, $stage, $begin, $end, undef, 0);
+        }
+    }
+
+    my $allServerLabels = undef;
 
     # create plots for each label for all stages
@@ -205,13 +225,17 @@
 
         $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
+        $plotter->createRateTimeSeries($label, undef, $begin, $end, undef, 0);
         $plotter->createHistogram($label, $begin, $end);
-
-        #routineChecks($label, "1 HOUR");
-    }
-    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
-    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
+    }
+
+    $allServerLabels = "all_".$server."_labels";
+
+    $plotter->createLogAndLinearTimeSeries($allServerLabels, undef, $begin, $end);
+    $plotter->createRateTimeSeries($allServerLabels, undef, $begin, $end, undef, 0);
+    $plotter->createHistogram($allServerLabels, $begin, $end);
     foreach $stage (@stages) {
 
-        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
+        $plotter->createLogAndLinearTimeSeries($allServerLabels,  $stage, $begin, $end); # TODO must be a neater way...
+        $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0);
     }
 }
Index: branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm	(revision 29834)
@@ -6,8 +6,12 @@
 use strict;
 
+use File::Temp;
+
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
 
 use base 'czartool::MySQLDb';
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
 
 ###########################################################################
@@ -49,4 +53,33 @@
     return 1;
 }
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
 
 ###########################################################################
@@ -264,4 +297,28 @@
 }
 
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
 
 ###########################################################################
@@ -271,16 +328,14 @@
 ###########################################################################
 sub runAnalysis {
-    my ($self, 
-            $label, 
-            $stage, 
-            $fromTime, 
-            $toTime, 
-            $_started, 
-            $_finished, 
-            $_stuck,
-            $_processed,
-            $_pending,
-            $_faults,
-            $_totalTime) = @_;
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -293,6 +348,4 @@
 SQL
 
-    $query->execute;
-
     my $lastProcessed = -1;
     my $lastPending = -1;
@@ -300,20 +353,49 @@
     my $lastTimestamp = undef;
     my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $started = undef;
-    my $finished = undef;
-    
     my $notProcessing = undef;
     my $howLongNotProcessing = undef;
     my $notProcessingLongerThanInterval = 0;
-
     my $stuck = undef;
     my $howLongStuck = undef;
     my $stuckLongerThanInterval = 0;
-
     my $stuffToDo = undef;
-    my $interval = "00:20:00";
-
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
     $linearProcessed = 0;
+    $lastProcessed = -1;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -326,4 +408,10 @@
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
 
         # get first derivative values
@@ -363,19 +451,11 @@
                 if (defined $notProcessing) {
                     
-                    if ($stuckLongerThanInterval) {
-
-#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
-                    }
-                    if ($notProcessingLongerThanInterval) {
-
-            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
-                    }
                     $notProcessing = undef; 
                     $stuck = undef; 
                 }
                 if (!defined $started) {
+
                     $started = $lastTimestamp;
                     $finished = undef; 
-#                    print "* $stage processing started at $started\n";
                 }
             }
@@ -418,33 +498,44 @@
     }
 
-    ${$_processed} = $linearProcessed;
-    ${$_pending} = $stuffToDo;
-    ${$_faults} = $lastFaults;
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
 
     if (defined $finished) {
-        ${$_finished} = $finished;
-        ${$_totalTime} = $self->diffTimes($finished, $started);
-    }
-
-    elsif(defined $started) {
-    
-        ${$_finished} = undef;
-    }
-    
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
     if (!defined $started){
     
-        ${$_started} = undef;
+        $stageMetrics->setStarted(undef);
     }
     else {
     
-        ${$_started} = $started;
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
     }
     if ($stuck && $stuckLongerThanInterval && !defined $finished) {
 
-        ${$_stuck} = $stuck;
+        $stageMetrics->setStuck($stuck);
     }
     else {
     
-        ${$_stuck} = undef;
+        $stageMetrics->setStuck(undef);
     }
 
@@ -484,5 +575,7 @@
     $query->execute;
 
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
@@ -497,4 +590,5 @@
     
     $linearProcessed = 0;
+    my $someData = 0;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -517,5 +611,5 @@
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-            if ($thisProcessed >= $lastProcessed){
+            if ($thisProcessed >= $lastProcessed && $timeDiff){
                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
@@ -547,4 +641,6 @@
         else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
 
+        if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
+
         $lastProcessed = $thisProcessed;
         $lastPending = $thisPending;
@@ -555,5 +651,5 @@
     close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
 
-    return 1;
+    #return $someData;
 }
 
@@ -637,8 +733,7 @@
 ###########################################################################
 sub createHostsData {
-    my ($self, $limit) = @_;
-
-    my $dataFile = "/tmp/czarplot_hosts_space.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $limit, $tmpFile) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -686,7 +781,6 @@
         print GNUDAT "\n";
     }
+
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -714,8 +808,7 @@
 ###########################################################################
 sub createStorageTimeSeriesData {
-    my ($self, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
-
-    my $dataFile = "/tmp/czarplot_gnuplot_storage_timeseries.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $tmpFile, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -753,6 +846,4 @@
 
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -763,7 +854,7 @@
 ###########################################################################
 sub createProcessingRateData {
-    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
-
-    my $startTime = $startDay;
+    my ($self, $stage, $label, $begin, $end, $interval, $dataFile, $isLog) = @_;
+
+    my $startTime = $begin;
     my $endTime;
     my $quit = 0;
@@ -772,10 +863,17 @@
     my $faults;
     my $timestamp;
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
+
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     my $cleanupCarry = 0;
+    my $someData = 0;
     while(1) {
 
-        if (!$self->isBefore($startTime, $endDay)) {last;}
+
+        #print "NNN $startTime, $end\n";
+        if (!$self->isBefore($startTime, $end)) {last;}
         $endTime = $self->addInterval($startTime, $interval);
         $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
@@ -783,8 +881,12 @@
         print GNUDAT "$timestamp $processed 0 0\n";
 
+        if ($processed > 0 ) {$someData = 1;}
+
         $startTime = $endTime;
     }
 
-    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n";
+
+    #return $someData;
 }
 
@@ -1512,5 +1614,4 @@
     $self->setRevision(12);
 }
-
 1;
 
Index: branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm	(revision 29834)
@@ -0,0 +1,327 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+use czartool::Metrics;
+
+
+package czartool::DayMetrics;
+our @ISA = qw(czartool::Metrics);    # inherits from Metrics class
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new(
+            $_[1],  # gpc1Db 
+            $_[2],  # czarDb
+            $_[3],  # baseDir
+            $_[4],  # verbose
+            $_[5]   # save_temps
+            ); 
+
+    $self->{day} = $_[6];
+
+    # sort out times
+    $self->{begin} =  "$self->{day} 06:00";
+    $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY");
+    $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR");
+
+    # create path, dir and html file
+    $self->{dayDir} = "$self->{baseDir}/$self->{day}";
+    rmdir($self->{dayDir});
+    mkdir($self->{dayDir}, 0777);
+    $self->{plotter}->setOutputPath($self->{dayDir});
+
+    if ($self->{verbose}) {print "* Creating metrics for $self->{day}\n";}
+
+    bless $self, $class;
+
+    return $self;
+}
+
+###########################################################################
+#
+# Writes HTML document
+#
+###########################################################################
+sub writeHTML {
+    my ($self) = @_;
+
+    # create HTML file and write header stuff
+    open (HTMLDOC, ">$self->{dayDir}/index.html");
+    print HTMLDOC "<html>\n";
+    print HTMLDOC "<head>\n";
+    print HTMLDOC "<title>IPP Metrics for $self->{day}</title>\n";
+    print HTMLDOC "<a name=\"top\"></a>\n";
+    print HTMLDOC "<h1 align=\"middle\">IPP Metrics for $self->{day}</h1>\n";
+
+    # summit and burntool exposures
+    my $summitExposures = $self->{gpc1Db}->countScienceExposuresFromLastNight($self->{day});
+    my $burntoolMetrics = new czartool::StageMetrics("burntool", "all_stdscience_labels", $self->{begin}, $self->{burntoolEnd});
+    $self->{czarDb}->runAnalysis($burntoolMetrics);
+
+    my $previousDay = $self->{czarDb}->subtractInterval($self->{day}, "1 DAY");
+    my $nextDay = $self->{czarDb}->addInterval($self->{day}, "1 DAY");
+
+    print HTMLDOC "<h5  align=\"middle\">";
+    print HTMLDOC "<a href=\"../$previousDay/index.html\"> \< previous day</a> | <a href=\"../$nextDay/index.html\">next day \></a><br>\n";
+    print HTMLDOC "Measured from $self->{begin} to $self->{end} HST<br>\n";
+    printf ( HTMLDOC "%d exposures taken on summit last night, %d through burntool today</h5>\n", 
+            $summitExposures, $burntoolMetrics->getProcessed() ? $burntoolMetrics->getProcessed() : 0 );
+    print HTMLDOC "</head>\n";
+    print HTMLDOC "<body>\n";
+
+
+
+    my $labels = undef;
+    my $label;
+    if (!$self->{czarDb}->getStdscienceLabelsInTimePeriod($self->{begin}, $self->{end}, \$labels)) {next;}
+
+    my $size = @{$labels};
+    my $table;
+    my $haveLabelData;
+    my %initialPendingTotals;
+    my %processedTotals;
+    my %finalPendingTotals;
+    my %faultTotals;
+    my %labelTables;
+
+    my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+# loop through all labels and create tables for each that have content
+    foreach my $row ( @{$labels} ) {
+        my ($label) = @{$row};
+
+
+        $table = "<table border='1'>";
+        $table .= "  <tr>\n";
+        $table .= "    <th>Stage</th>\n";
+        $table .= "    <th>Initial Pending</th>\n";
+        $table .= "    <th>Started</th>\n";
+        $table .= "    <th>25% complete</th>\n";
+        $table .= "    <th>50% complete</th>\n";
+        $table .= "    <th>75% complete</th>\n";
+        $table .= "    <th>90% complete</th>\n";
+        $table .= "    <th>95% complete</th>\n";
+        $table .= "    <th>Finished</th>\n";
+        $table .= "    <th>Time taken</th>\n";
+        $table .= "    <th>Processed</th>\n";
+        $table .= "    <th>Exp per hour</th>\n";
+        $table .= "    <th>Final Pending</th>\n";
+        $table .= "  </tr>\n";
+
+        $haveLabelData = 0;
+        foreach my $stage ( @stages ){
+
+            my $stageMetrics = new czartool::StageMetrics($stage, $label, $self->{begin}, ($stage eq "burntool") ? $self->{burntoolEnd} : $self->{end});
+            if (!$self->{czarDb}->runAnalysis($stageMetrics)) {next;}
+
+            if (!$stageMetrics->anythingProcessed()) {next;}
+
+            if ($stageMetrics->getProcessed() > 0) {$haveLabelData = 1;};
+
+            $table .= "  <tr>\n";
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getStage() );
+            $table .= sprintf( "    <td>%d</td>\n", $stageMetrics->getInitialPending() ? $stageMetrics->getInitialPending() : "0" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getStarted() ? $stageMetrics->getStarted() : "");
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished25() ? $stageMetrics->getFinished25() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished50() ? $stageMetrics->getFinished50() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished75() ? $stageMetrics->getFinished75() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished90() ? $stageMetrics->getFinished90() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished95() ? $stageMetrics->getFinished95() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getFinished100() ? $stageMetrics->getFinished100() : "-" );
+            $table .= sprintf( "    <td>%s</td>\n", $stageMetrics->getTotalTime() ? $stageMetrics->getTotalTime() : "-" );
+            $table .= sprintf( "    <td>%d</td>\n", $stageMetrics->getProcessed() ? $stageMetrics->getProcessed() : "0" );
+            $table .= sprintf( "    <td>%.2f</td>\n", $stageMetrics->getRate() ? $stageMetrics->getRate() : "0" );
+            $table .= sprintf( "    <td>");
+            if ($stageMetrics->getFinalPending() > 0) {$table .= "<font color=\"red\">";}
+            $table .= sprintf( "%d\n", $stageMetrics->getFinalPending() );
+            if ($stageMetrics->getFinalPending() > 0) {$table .= "</font>\n";}
+            $table .= sprintf( "</td>");
+            $table .= "  </tr>\n";
+
+            if (!$initialPendingTotals{$stage}) {$initialPendingTotals{$stage} = 0;}
+            $initialPendingTotals{$stage} = $initialPendingTotals{$stage} + $stageMetrics->getInitialPending();
+            if (!$processedTotals{$stage}) {$processedTotals{$stage} = 0;}
+            $processedTotals{$stage} = $processedTotals{$stage} + $stageMetrics->getProcessed();
+            if (!$finalPendingTotals{$stage}) {$finalPendingTotals{$stage} = 0;}
+            $finalPendingTotals{$stage} = $finalPendingTotals{$stage} + $stageMetrics->getFinalPending();
+            if (!$faultTotals{$stage}) {$faultTotals{$stage} = 0;}
+            $faultTotals{$stage} = $faultTotals{$stage} + $stageMetrics->getFaults();
+        }
+
+        $table .= "</table>";
+        $table .= "<br>\n";
+
+        if ($haveLabelData) { 
+            $labelTables{$label} = $table; 
+            if ($self->{verbose}) {print "- $label\n";}
+        }
+    }
+
+    # create hyperlink list of list of active labels
+    print HTMLDOC "<h2>Active labels for this day</h1>\n";
+    foreach my $label (keys (%labelTables)) {
+
+        print HTMLDOC "<a href=\"#$label\">$label</a><br>\n";
+    }
+
+    # some plots
+    print HTMLDOC "<h1 align=\"middle\">Summary plots</h1>\n";
+    $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0);
+    print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_t.png\" alt=\"All labels and all stages for $self->{day}\" />\n";
+    $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0);
+    print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_h.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
+    $self->{plotter}->createRateTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, "1 HOUR", 0);
+    print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_r.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
+    $self->{plotter}->plotMagicMaskFractionHistogram($self->{day}, $self->{day});
+    print HTMLDOC "<img src=\"czarplot_magic_mask_fraction_h.png\" alt=\"\" />\n";
+    $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end});
+    print HTMLDOC "<img src=\"czarplot_cluster.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
+    print HTMLDOC "<img src=\"czarplot_magic_mask_fraction_d.png\" alt=\"\" />\n";
+
+
+    # survey table
+    $self->printSurveyTable();
+
+    # table for stage totals
+    $table =  "<h2  align=\"middle\">Totals for all labels</h1>\n";
+    $table .= "<table border='1'>";
+    $table .= "  <tr>\n";
+    $table .= "    <th>Stage</th>\n";
+    $table .= "    <th>Initial Pending</th>\n";
+    $table .= "    <th>Processed</th>\n";
+    $table .= "    <th>Final Pending</th>\n";
+    $table .= "    <th>Remaining Faults</th>\n";
+    $table .= "  </tr>\n";
+    foreach my $stage ( @stages ){
+
+        $table .= "  <tr>\n";
+        $table .= sprintf( "    <td>%s</td>\n", $stage );
+        $table .= sprintf( "    <td>%d</td>\n", $initialPendingTotals{$stage} ? $initialPendingTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $processedTotals{$stage} ? $processedTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $finalPendingTotals{$stage} ? $finalPendingTotals{$stage} : 0);
+        $table .= sprintf( "    <td>%d</td>\n", $faultTotals{$stage} ? $faultTotals{$stage} : 0);
+        $table .= "  </tr>\n";
+    }
+    $table .= "</table>";
+    $table .= "<br>\n";
+
+    print HTMLDOC $table;
+
+    # print all label tables to page
+    foreach my $label (keys (%labelTables)) {
+
+        print HTMLDOC "<a name=\"$label\"></a>\n";
+        print HTMLDOC "<h2  align=\"middle\">$label <a href=\"#top\">(top)</a></h1>\n";
+        $self->{plotter}->createTimeSeries($label, undef, $self->{begin}, $self->{end}, 0, 0, 0);
+        print HTMLDOC "<img src=\"czarplot_linear_".$label."_all_stages_t.png\" alt=\"All stages for $label for $self->{day}\" />\n";
+        $self->{plotter}->createHistogram($label, $self->{begin}, $self->{end}, 0, 0, 0);
+        print HTMLDOC "<img src=\"czarplot_linear_".$label."_all_stages_h.png\" alt=\"All stages for $label for $self->{day}\" />\n";
+
+        print HTMLDOC $labelTables{$label};
+    }
+
+
+    print HTMLDOC "<br>\n";
+    print HTMLDOC "</body>\n";
+    print HTMLDOC "</html>\n";
+    close(HTMLDOC);
+}
+
+###########################################################################
+#
+# Creates a survey table
+#
+###########################################################################
+sub printSurveyTable() {
+    my ($self) = @_;
+
+    print HTMLDOC "<h2  align=\"middle\">Survey statistics</h1>\n";
+    print HTMLDOC "<table border='1'>";
+    print HTMLDOC "  <tr>\n";
+    print HTMLDOC "    <th>Survey</th>\n";
+    print HTMLDOC "    <th>Started burntool</th>\n";
+    print HTMLDOC "    <th>Finished distribution</th>\n";
+    print HTMLDOC "    <th>Time taken</th>\n";
+    print HTMLDOC "  </tr>\n";
+
+    # OSS survey
+    my ($started, $finished, $timeTaken) = undef;
+    $self->{czarDb}->getDayTimings("OSS.nightlyscience", "burntool", "diff", $self->{begin}, \$started, \$finished, \$timeTaken);
+    $self->printSurveyDetails("OSS", $started, $finished, $timeTaken);
+
+    # MDF surveys
+    my @mdfs = ("M31.nightlyscience",
+            "MD01.nightlyscience", 
+            "MD02.nightlyscience", 
+            "MD03.nightlyscience", 
+            "MD04.nightlyscience", 
+            "MD05.nightlyscience", 
+            "MD06.nightlyscience", 
+            "MD07.nightlyscience", 
+            "MD08.nightlyscience", 
+            "MD09.nightlyscience", 
+            "MD10.nightlyscience",
+            "MD11.nightlyscience");
+
+    my $earliest = "2099-01-01 00:00:00";
+    my $latest = "2001-01-01 00:00:00";
+
+    my $mdf;
+    my $haveStart = 0;
+    my $haveEnd = 0;
+    foreach $mdf (@mdfs) {
+
+        if ($self->{czarDb}->getDayTimings($mdf, "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken)) {
+
+            if ($started && $self->{czarDb}->isBefore($started, $earliest)) {$earliest = $started; $haveStart = 1;}
+            if ($finished && $self->{czarDb}->isBefore($latest, $finished)) {$latest = $finished; $haveEnd = 1;}
+        }
+    }
+
+    if (!$haveStart) {$earliest = undef;}
+    if (!$haveEnd) {$latest = undef;}
+    if (!$haveStart || !$haveEnd) {$timeTaken = undef;}
+
+    else {$timeTaken = $self->{czarDb}->diffTimes($latest, $earliest);}
+    $self->printSurveyDetails("MDF", $earliest, $latest, $timeTaken);
+
+    $self->{czarDb}->getDayTimings("all_stdscience_labels", "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken);
+    $self->printSurveyDetails("All", $started, $finished, $timeTaken);
+
+    print HTMLDOC "</table>\n";
+}
+
+###########################################################################
+#
+# Creates a row in the survey table
+#
+###########################################################################
+sub printSurveyDetails() {
+    my ($self, $survey, $started, $finished, $timeTaken, $processed, $pending) = @_;
+
+    print HTMLDOC "  <tr>\n";
+    print HTMLDOC "    <td>$survey</td>\n";
+    printf (HTMLDOC "    <td>%s</td>\n", $started ? $started : "no");
+    printf (HTMLDOC "    <td>%s</td>\n", $finished ? $finished : "no");
+    printf (HTMLDOC "    <td>%s</td>\n", $timeTaken ? $timeTaken : "na");
+    print HTMLDOC "  </tr>\n";
+}
+1;
Index: branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm	(revision 29834)
@@ -60,4 +60,62 @@
 ###########################################################################
 #
+# Returns the number of science exposures taken 'last night' for a given day
+# 
+# Notes:
+#
+# - 03:00 HST = 17:00 last night UTC
+# - 17:00 HST = 07:00 this morning UTC
+# - 'OBJECT' = science exposures
+#
+###########################################################################
+sub countScienceExposuresFromLastNight {
+    my ($self, $day) = @_;
+
+    my $begin = "$day 03"; 
+    my $end = "$day 17"; 
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*)
+        FROM summitExp 
+        WHERE dateobs > '$begin' 
+        AND dateobs < '$end' AND
+        exp_type = 'OBJECT';
+SQL
+
+    $query->execute;
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures between two dates that have been registered with chip
+# 
+# Notes:
+#
+# necessary to convert times as rawExp.dateobs is in UTC
+#
+###########################################################################
+sub getProcessedExposures {
+    my ($self, $beginDate, $endDate, $expIds) = @_;
+
+    $beginDate = "$beginDate 03";
+    $endDate = "$endDate 17";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        SELECT rawExp.exp_id 
+        FROM chipRun, rawExp 
+        WHERE chipRun.exp_id = rawExp.exp_id 
+        AND rawExp.dateobs >= '$beginDate' 
+        AND rawExp.dateobs <= '$endDate';
+SQL
+
+    $query->execute;
+    ${$expIds} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
 # Returns count of faults for this stage and label
 #
@@ -158,7 +216,52 @@
     }
 
-
     $query->execute;
     return scalar $query->fetchrow_array();
 }
+
+###########################################################################
+#
+# Returns magic mask fraction across all chips for a particular exposure
+#
+###########################################################################
+sub getMagicMaskFraction {
+        my ($self, $exp_id, $fracs) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT component, streak_frac 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%";
+SQL
+    $query->execute;
+    ${$fracs} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns average magic mask fraction across all chips for a particular exposure
+#
+###########################################################################
+sub getAverageMagicMaskFraction {
+        my ($self, $exp_id) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT AVG(streak_frac) 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%";
+SQL
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
 1;
Index: branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm	(revision 29834)
@@ -0,0 +1,49 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+use CGI::Pretty qw[:standard];
+
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Plotter;
+use czartool::StageMetrics;
+
+package czartool::Metrics;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = {
+        gpc1Db => shift,
+        czarDb => shift,
+        baseDir => shift,
+        verbose => shift,
+        save_temps => shift,
+    };
+
+    # create path, dir
+    mkdir($self->{baseDir}, 0777);
+
+    # instantiate a plotter object
+    $self->{plotter} = new czartool::Plotter(
+            $self->{gpc1Db},
+            $self->{czarDb},
+            "%Y%m%d-%H%M%S",
+            "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8",
+            ".",
+            $self->{save_temps});
+
+    bless $self, $class;
+    return $self;
+}
+
+1;
Index: branches/eam_branches/ipp-20101103/tools/czartool/MySQLDb.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/MySQLDb.pm	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czartool/MySQLDb.pm	(revision 29834)
@@ -64,7 +64,8 @@
     return $self->{_dbHost};                                                 
 }                                                                               
-###########################################################################
-#
-# Adds the provided interval to the provided time
+
+###########################################################################
+#
+# Returns a date formatted wit hthe current formatting
 #
 ###########################################################################
@@ -82,4 +83,20 @@
 ###########################################################################
 #
+# Returns a full MySQL formatted date
+#
+###########################################################################
+sub getFullDate {
+    my ($self, $time) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DATE_FORMAT('$time','%Y-%m-%d %H:%i:%s'); 
+SQL
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
 # Adds the provided interval to the provided time
 #
@@ -98,5 +115,37 @@
 ###########################################################################
 #
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Formats a date/time string in a way suitable for TIMEDIFF calls
+#
+###########################################################################
+sub formatTimeForDiff {
+    my ($self, $time) = @_;
+
+    if (${$time} =~ m/^[0-9]{4}-[0-9]{2}-[0-9]{2}.*/) {${$time} = $self->getFullDate(${$time});}
+    elsif(${$time} !~ m/^[0-9]+:[0-9]{2}:[0-9]{2}$/) {return 0;}
+
+    return 1;
+}
+
+###########################################################################
+#
 # Finds the difference of two times
+#
+# Should be able to handle most date/time formats
 #
 ###########################################################################
@@ -104,6 +153,28 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIMEDIFF('$time1','$time2'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the provided time in seconds
+#
+###########################################################################
+sub getTimeInSecs {
+    my ($self, $time) = @_;
+
+    if (!$time) {return 0;}
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC('$time'); 
 SQL
 
@@ -120,5 +191,8 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2')); 
 SQL
@@ -146,20 +220,4 @@
 ###########################################################################
 #
-# Subtracts the provided interval from the provided time
-#
-###########################################################################
-sub subtractInterval {
-    my ($self, $time, $interval) = @_;
-
-      my $query = $self->{_db}->prepare(<<SQL);
-          SELECT '$time' - INTERVAL $interval; 
-SQL
-    $query->execute;
-
-return scalar $query->fetchrow_array();
-}
-
-###########################################################################
-#
 # Returns whether time1 is before time2
 #
@@ -168,10 +226,13 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
-          SELECT '$time1' < '$time2'; 
-SQL
-    $query->execute;
-
-return scalar $query->fetchrow_array();
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT TIME_TO_SEC(TIMEDIFF('$time1', '$time2'));
+SQL
+    $query->execute;
+
+    return (scalar $query->fetchrow_array() >= 0) ? 0 : 1;
 }
 
Index: branches/eam_branches/ipp-20101103/tools/czartool/OCMetrics.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/OCMetrics.pm	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/czartool/OCMetrics.pm	(revision 29834)
@@ -0,0 +1,93 @@
+
+
+use DateTime;
+use IPC::Cmd 0.36 qw( can_run run);
+my $end = '2010-11-01';
+my $start = $czarDb->subtractInterval($end, "28 DAY");
+
+
+my $days;
+getDaysFromFullMoon($start, \$days);
+print "$start is $days from full moon\n";
+$start = $czarDb->addInterval($start, "$days DAY");
+print "=$start\n";
+getDaysFromFullMoon($end, \$days);
+print "$end is $days from full moon\n";
+$end = $czarDb->addInterval($end, "$days DAY");
+print "=$end\n";
+
+
+
+my ($beginl,$endl) = get_lunation_extent("2010-11-09");
+
+
+print "begin=$beginl end=$endl\n";
+exit;
+
+
+
+
+###########################################################################
+#
+# Calls 'moondata' program to get number of days since/to full moon for provided date
+#
+###########################################################################
+sub getDaysFromFullMoon() {
+    my ($date, $days) = @_;
+
+    my $cmd = "moondata $date 0 0";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+        run ( command => $cmd, verbose => $verbose);
+    unless ($success) {print "* Can't run '$cmd'\n"; return 0;}
+
+    my @result = split /\s+/,(join "\n", @$stdout_buf);
+    ${$days} = $result[6];
+
+    return 1;
+}
+
+sub get_lunation_extent {
+    my $date = shift;
+    my ($year,$month,$day) = split /-/,$date;
+    my $dateobs_begin;
+    my $dateobs_end;
+
+    my $dt = DateTime->new(year => $year, month => $month, day => $day,
+            hour => 0, minute => 0, second => 0, nanosecond => 0,
+            time_zone => 'Pacific/Honolulu');
+    do {
+        $dt->subtract(days => 1);
+        my $ymd = $dt->ymd;
+
+
+        my $md_cmd = "moondata $ymd 0 0";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $md_cmd, verbose => $verbose);
+        unless ($success) {
+            print "can;t run '$md_cmd'\n"
+        }
+        my @result = split /\s+/,(join "\n", @$stdout_buf);
+        if (abs($result[6]) <= 0.5) {
+            $dateobs_end = $ymd;
+        }
+    } while (!defined($dateobs_end));
+
+    do {
+        $dt->subtract(days => 1);
+        my $ymd = $dt->ymd;
+        my $md_cmd = "moondata $ymd 0 0";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $md_cmd, verbose => $verbose);
+        unless ($success) {
+            print "can;t run '$md_cmd'\n"
+        }
+        my @result = split /\s+/,(join "", @$stdout_buf);
+        if (abs($result[6]) <= 0.5) {
+            $dateobs_begin = $ymd;
+        }
+    } while (!defined($dateobs_begin));
+
+    return($dateobs_begin,$dateobs_end);
+}
+
+
Index: branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm	(revision 29834)
@@ -4,4 +4,6 @@
 use warnings;
 use strict;
+
+use File::Temp;
 
 my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
@@ -16,4 +18,5 @@
     my $class = shift;
     my $self = {
+        _gpc1Db => shift,
         _czarDb => shift,
         _dateFormat => shift,
@@ -53,4 +56,34 @@
 }
 
+
+###########################################################################
+#
+# Figures out a suitable interval for rate plots
+# Could use a function here, but instead we want to choose meaningful intervals
+#
+###########################################################################
+sub getSuitableInterval {
+    my ($self, $begin, $end, $interval) = @_;
+
+    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);
+
+
+    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "24:00:00")) {${$interval} = "1 HOUR";} # under 1 day
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "72:00:00")) {${$interval} = "4 HOUR";} # under 3 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "96:00:00")) {${$interval} = "5 HOUR";} # under 4 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "144:00:00")) {${$interval} = "6 HOUR";} # under 6 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "168:00:00")) {${$interval} = "12 HOUR";} # under week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "672:00:00")) {${$interval} = "1 DAY";} # under 2 week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "2016:00:00")) {${$interval} = "1 WEEK";} # under 3 month
+    else {${$interval} = "1 MONTH";}
+
+    return 1;
+}
+
 ###########################################################################
 #
@@ -61,19 +94,16 @@
     my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
 
-    my $minX = 999999999;      
-    my $maxX = -9999999999;
-    my $timeDiff = -1;
-
+    # stages
     my $stages = undef;                 
-
-    if (!$selectedStage) {$stages = \@allStages;}
-    else {$stages = ["$selectedStage"]};        
-
-    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
-
-    my $divX;
-    my $timeFormat;
-    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
-
+    if (!$selectedStage) {
+        $stages = \@allStages;
+        $selectedStage = 'all_stages';
+    }
+    else {$stages = ["$selectedStage"]};
+
+    # interval
+    if (!$interval && !$self->getSuitableInterval($beginTime, $endTime, \$interval)) {return 0;}
+
+    # set up files
     my %gnuplotFiles;
     my $stage = undef;
@@ -89,10 +119,6 @@
         "set term $self->{_outputFormat};" .
         "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
-        #"set xdata time;" .
-        "set timefmt \"$self->{_dateFormat}\";" .
-       "set format x \"$timeFormat\";" .
-        "set xtics \"$minX\", $divX, \"$maxX\";" .
-#        "set xrange [\"$minX\":\"$maxX\"];" .
         "set boxwidth;" .
+        "set xtic rotate by -90 scale 0;" .
         "set style data histogram;" .
         "set style histogram rowstacked;" .
@@ -124,5 +150,11 @@
     print GP ";\n";
     close GP;
-}                                                    
+
+    # now delete temp dat files
+    foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}
+
+    return 1;
+}
+
 ###########################################################################
 #
@@ -168,5 +200,5 @@
     if ($numOfPlots == 0 ) {
 
-        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
+#        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
         return;
     } 
@@ -202,15 +234,13 @@
     my $outputFile = createImageFileName($self, $label, undef, "h", 0);
 
-    my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table
-        open (GNUDAT, ">$inputFile");
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_h.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my ($processed, $pending, $faults);
     my $stage = undef;
     my $pendingMinusFaults = undef;
-    my $maxY = 0;
-    my $sum;
     foreach $stage (@allStages) {
 
-        $self->{_czarDb}->countProcessedPendingAndFaults(
+        if (!$self->{_czarDb}->countProcessedPendingAndFaults(
                 $label, 
                 $stage, 
@@ -219,10 +249,8 @@
                 \$processed, 
                 \$pending, 
-                \$faults);
+                \$faults)) {next;}
 
         $pendingMinusFaults = $pending - $faults;
-        print GNUDAT "$stage $faults $processed, $pendingMinusFaults\n";
-        my $sum = $faults + $processed + $pendingMinusFaults;
-        if ($sum > $maxY) {$maxY = $sum;}
+        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
     }
 
@@ -232,8 +260,4 @@
     use FileHandle;
     GP->autoflush(1);
-
-    if ($maxY == 0) {$maxY = 1;}
-    else {$maxY = $maxY*1.1;}
-
 
     if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
@@ -243,5 +267,4 @@
         "set grid;" .
         "set boxwidth;" .
-        "set yrange [\"0\":\"$maxY\"];" .
         "set style data histogram;" .
         "set style histogram rowstacked;" .
@@ -249,9 +272,8 @@
         "set ylabel \"Exposures\";" .
         "set boxwidth 0.75;" .
-        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
+        "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
         "\n";
 
     close GP;
-    unlink($inputFile);
 }
 
@@ -295,6 +317,21 @@
     my ($self, $timeDiff, $divX, $timeFormat) = @_;
 
+    # if less than a few mins of data, show 1 minute tics
+    if ($timeDiff < 300) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 60;
+    }
+    # if less than a few mins of data, show 5 minute tics
+    elsif ($timeDiff < 1800) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 300;
+    }
+    # if less than an hour of data, show 15 minute tics
+    elsif ($timeDiff < 3600) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 900;
+    }
     # if less than a couple of hour's data plotted, show 30 mins tics
-    if ($timeDiff < 7200) {
+    elsif ($timeDiff < 7200) {
         ${$timeFormat} = "%H:%M";
         ${$divX} = 1800;
@@ -376,5 +413,8 @@
 
     my $firstIn = 1;
-    foreach my $stage (keys %$gnuplotFiles) {
+    # loop through stages array so that they are ordered properly (not maintained in hash)
+    foreach my $stage (@allStages) {
+
+        if(!$gnuplotFiles->{$stage}) {next;}
         if (!$firstIn) {print GP ",";}
 
@@ -398,8 +438,5 @@
 
     # now delete temp dat files
-    foreach my $stage (keys %$gnuplotFiles) {
-
-        unlink($gnuplotFiles->{$stage});
-    }
+    foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
 }
 
@@ -416,5 +453,6 @@
     my ($minX, $maxX, $minY, $maxY, $timeDiff);
 
-    my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
 
     my $timeFormat = undef;
@@ -440,9 +478,8 @@
         "set ylabel \"Available (TB)\";" .
         "plot " . 
-        "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
+        "'".$tmpFile->filename."' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
 
     print GP "\n";
     close GP;
-    unlink($gnuplotFile);
 }
 
@@ -458,5 +495,6 @@
         my $outputFile = "$prefix/czarplot_hosts_space.png";
     my $limit = 97.0;
-    my $inputFile = $self->{_czarDb}->createHostsData($limit);
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $self->{_czarDb}->createHostsData($limit, $tmpFile);
 
     my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();
@@ -477,5 +515,5 @@
         "set ylabel \"Space (TB)\";" .
         "set xtic rotate by -90 scale 0;" .
-        "plot '$inputFile' " .
+        "plot '".$tmpFile->filename."' " .
         "using 2:xtic(1) t \"Used\" lt 7," .
         "'' using 3 t \"Over $limit% used\" lt 1," . 
@@ -489,3 +527,181 @@
     close GP;
 }
+
+###########################################################################
+#
+# Generates 3D plot of magic mask fraction for provided exposure ID
+#
+###########################################################################
+sub plotMagicMaskFractionForThisExposure {
+    my ($self, $exp_id) = @_;
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $outputFile = "$prefix/czarplot_single_exp_magic_mask_fraction.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_single_exp_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => '.dat');
+
+    my $fracs;
+    if (!$self->{_gpc1Db}->getMagicMaskFraction($exp_id, \$fracs)) {return 0;}
+
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    print GNUDAT "0 0 0.0\n";
+    foreach my $row ( @{$fracs} ) {
+
+        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
+            print GNUDAT "$1 $2 @{$row}[1]\n";
+            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
+            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
+        }
+    }
+    print GNUDAT "7 7 0.0\n";
+
+
+    close(GNUDAT);
+
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    my $datFile = $tmpFile->filename;
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP <<PLOT;
+
+    set term $self->{_outputFormat}
+    set title "Magic mask fraction for exposure $exp_id"
+        set grid
+        set size square
+        set xrange [-0.5:7.5] reverse 
+        set yrange [-0.5:7.5]  
+        unset key
+        set palette rgb 22,13,10
+        set ylabel "Y"
+        set xlabel "X"
+        set cbrange [0:1]
+        set datafile missing "NaN"
+        plot "$datFile" using 1:2:3  with image
+PLOT
+
+    close GP;
+
+    return 1;
+}
+
+
+###########################################################################
+#
+# Plots mask fraction histogram for exposures taken between these dates
+#
+###########################################################################
+sub plotMagicMaskFractionHistogram {
+    my ($self, $begin, $end) = @_;
+
+    # get exposures between these dates
+    my $exp_ids = undef;
+    if (!$self->{_gpc1Db}->getProcessedExposures($begin, $end, \$exp_ids)) {return 0;}
+    my $totalExp = @{$exp_ids};
+    if ($totalExp < 1) {return 0;}
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $histoOutputFile = "$prefix/czarplot_magic_mask_fraction_h.png";
+    my $distOutputFile = "$prefix/czarplot_magic_mask_fraction_d.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+
+    # set up bins
+    my %histogram; 
+    my @bins;
+    my $interval = 0.05;
+    my $min = 1.0;
+    my $max = 1.0;
+    for (my $n=0.0; $n<$max; $n += $interval){
+
+        push(@bins, $n);
+        $histogram{$n} = 0;
+    }
+
+    # get mask for each exposure, and bin
+    my $mask = undef;
+    my $expCount = 0;
+    foreach my $row ( @{$exp_ids} ) {
+
+        $mask = $self->{_gpc1Db}->getAverageMagicMaskFraction(@{$row}[0]);
+
+        if (!$mask) {next;}
+        $expCount++;
+
+        foreach my $bin (@bins) {
+            if ($mask <= ($bin+$interval)) {
+
+                $histogram{$bin}++;
+                last;
+            }
+        }
+    }
+
+    # write data to GNUPLOT data file
+    print "* Found mask values for $expCount exposures out of $totalExp\n";
+    open (GNUDAT, ">".$tmpFile->filename);
+    my $accum = 0;
+    my $maxBin = 0;
+    foreach my $bin (@bins) {
+
+        $accum += $histogram{$bin};
+        if ($histogram{$bin} > $maxBin) {$maxBin = $histogram{$bin};}
+        print GNUDAT "$bin $histogram{$bin} $accum\n";
+    }
+    close(GNUDAT);
+
+    # now plot
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    $maxBin = $maxBin * 1.1;
+    $accum = $accum * 1.1;
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Magic mask fraction for $expCount exposures between '$begin' and '$end'\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set yrange [0:$maxBin];" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set boxwidth 0.75;" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot '".$tmpFile->filename."' using 2:xtic(1) notitle lt 1;" . 
+        "\n";
+
+    close GP;
+
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Cumulative distribution of magic streaks per image between '$begin' and '$end'\";" .
+        "set yrange [0:$accum];" .
+        "set key left top;" .
+        "set grid xtics;" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot " . 
+        "'".$tmpFile->filename."' using 1:3 notitle with lines lt 2 lw 2\n";
+
+    print GP "\n";
+    close GP;
+
+    return 1;
+}
 1;
Index: branches/eam_branches/ipp-20101103/tools/czartool/StageMetrics.pm
===================================================================
--- branches/eam_branches/ipp-20101103/tools/czartool/StageMetrics.pm	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/czartool/StageMetrics.pm	(revision 29834)
@@ -0,0 +1,129 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::StageMetrics;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {
+
+        stage => shift,
+        label => shift,
+        startTime => shift,
+        endTime => shift
+
+    };
+
+    $self->{started} = undef;
+    $self->{stuck} = undef;
+    $self->{processed} = undef;
+    $self->{initialPending} = undef;
+    $self->{finalPending} = undef;
+    $self->{faults} = undef;
+    $self->{totalTime} = undef;
+    $self->{finished25} = undef;
+    $self->{finished50} = undef;
+    $self->{finished75} = undef;
+    $self->{finished90} = undef;
+    $self->{finished95} = undef;
+    $self->{finished100} = undef;
+    $self->{rate} = undef;
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Returns whether anything has been processed or not
+#
+###########################################################################
+sub anythingProcessed() {
+    my $self = shift; 
+
+    if (!$self->{processed}) {return 0;}
+    if ($self->{processed} < 1) {return 0;}
+    return 1;
+}
+
+###########################################################################
+#
+# Prints all metrics
+#
+###########################################################################
+sub printMe() {
+    my $self = shift;
+
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+    print "|     stage     |       started       |       finished      | time taken | processed | pending |  rate | faults | stuck? |\n";
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+    printf( "| %10s    ", $self->{stage} ? $self->{stage} : "no");
+    printf( "| %20s", $self->{started} ? $self->{started} : "no");
+    #printf( "| %s", $self->{finished25} ? $self->{finished25} : "no");
+    #printf( "| %s", $self->{finished50} ? $self->{finished50} : "no");
+    #printf( "| %s", $self->{finished75} ? $self->{finished75} : "no");
+    #printf( "| %s", $self->{finished90} ? $self->{finished90} : "no");
+    #printf( "| %s", $self->{finished95} ? $self->{finished95} : "no");
+    printf( "| %20s", $self->{finished100} ? $self->{finished100} : "no");
+    printf( "| %10s ", $self->{totalTime} ? $self->{totalTime} : "na");
+    printf( "| %5d     ", $self->{processed} ? $self->{processed} : 0);
+    printf( "| %5d   ", $self->{pending} ? $self->{pending} : 0);
+    printf( "| %4.2f ", $self->{rate} ? $self->{rate} : 0);
+    printf( "| %5d  ", $self->{faults} ? $self->{faults} : 0);
+    printf( "| %4s   ", $self->{stuck} ? "YES" : "no");
+    print "|\n";
+    print "+---------------+---------------------+---------------------+------------+-----------+---------+-------+--------+--------+\n";
+}
+
+###########################################################################
+#
+# Setters for all class variables
+#
+###########################################################################
+sub setStarted() {my $self = shift; $self->{started} = shift;}
+sub setStuck() {my $self = shift; $self->{stuck} = shift;}
+sub setProcessed() {my $self = shift; $self->{processed} = shift;}
+sub setInitialPending() {my $self = shift; $self->{initialPending} = shift;}
+sub setFinalPending() {my $self = shift; $self->{finalPending} = shift;}
+sub setFaults() {my $self = shift; $self->{faults} = shift;}
+sub setTotalTime() {my $self = shift; $self->{totalTime} = shift;}
+sub setFinished25() {my $self = shift; $self->{finished25} = shift;}
+sub setFinished50() {my $self = shift; $self->{finished50} = shift;}
+sub setFinished75() {my $self = shift; $self->{finished75} = shift;}
+sub setFinished90() {my $self = shift; $self->{finished90} = shift;}
+sub setFinished95() {my $self = shift; $self->{finished95} = shift;}
+sub setFinished100() {my $self = shift; $self->{finished100} = shift;}
+sub setRate() {my $self = shift; $self->{rate} = shift;}
+
+###########################################################################
+#
+# Getters for all class variables
+#
+###########################################################################
+sub getStartTime() {my $self = shift; return $self->{startTime};}
+sub getEndTime() {my $self = shift; return $self->{endTime};}
+sub getLabel() {my $self = shift; return $self->{label};}
+sub getStage() {my $self = shift; return $self->{stage};}
+sub getStarted() {my $self = shift; return $self->{started};}
+sub getStuck() {my $self = shift; return $self->{stuck};}
+sub getProcessed() {my $self = shift; return $self->{processed};}
+sub getInitialPending() {my $self = shift; return $self->{initialPending};}
+sub getFinalPending() {my $self = shift; return $self->{finalPending};}
+sub getFaults() {my $self = shift; return $self->{faults};}
+sub getTotalTime() {my $self = shift; return $self->{totalTime};}
+sub getFinished25() {my $self = shift; return $self->{finished25};}
+sub getFinished50() {my $self = shift; return $self->{finished50};}
+sub getFinished75() {my $self = shift; return $self->{finished75};}
+sub getFinished90() {my $self = shift; return $self->{finished90};}
+sub getFinished95() {my $self = shift; return $self->{finished95};}
+sub getFinished100() {my $self = shift; return $self->{finished100};}
+sub getRate() {my $self = shift; return $self->{rate};}
+
+1;
Index: branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.133.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.133.pl	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.133.pl	(revision 29834)
@@ -0,0 +1,71 @@
+#!/usr/bin/perl -w
+use DBI;
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
+
+
+my $dbname = 'gpc1';
+my $dbserver = 'ippdb01';
+my $dbuser = 'ippuser';
+my $dbpass = 'ippuser';
+
+my %OSS_orig= ();
+my %OSS_reprocessed = ();
+
+
+my $db = DBI->connect(
+   "DBI:mysql:database=${dbname};host=${dbserver};mysql_socket=" .
+   DB_SOCKET(),${dbuser},${dbpass}, 
+   { RaiseError => 1, AutoCommit => 1}
+   ) or die "Unable to connect to database $DBI::errstr\n";
+
+my $qdiffs = "select distinct(warp1), warp2 from diffRun join diffInputSkyfile using (diff_id) where data_group like 'OSS.201010%' or data_group like 'OSS.201011%';";
+
+my $qosswarpsorig = "SELECT rawExp.exp_id,warpRun.warp_id FROM   warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id and (chipRun.data_group like 'OSS.201010%' or chipRun.data_group like 'OSS.201011%') ;";
+
+my $qosswarpsrep = "SELECT rawExp.exp_id,warpRun.warp_id FROM   warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id and (chipRun.data_group = 'OC133.OSS.20101117') ;";
+
+$diffs = $db->selectall_arrayref( $qdiffs ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+$osswarpsorig = $db->selectall_arrayref( $qosswarpsorig ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+$osswarpsrep = $db->selectall_arrayref( $qosswarpsrep ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+
+$db->disconnect();
+
+
+foreach $it (@{$osswarpsorig}) {
+   
+    $OSS_orig{${$it}[1] } = ${$it}[0];   #oss orig will sort by warps -> exp_id
+
+}
+
+foreach $it (@{$osswarpsrep}) {
+   
+    $OSS_reprocessed{${$it}[0] } = ${$it}[1];   #oss rep will sort by exp_id -> warps
+
+}
+
+$total =0;
+
+foreach $it (@{$diffs}) {
+    my ($oldexp1, $newwarp1, $oldexp2, $newwarp2);
+
+    $oldexp1= $OSS_orig{${$it}[0]};
+    $newwarp1= $OSS_reprocessed{$oldexp1};
+
+    $oldexp2= $OSS_orig{${$it}[1]};
+    $newwarp2= $OSS_reprocessed{$oldexp2};
+
+  #  print "${$it}[0] ${$it}[1] $oldexp1 $oldexp2 $newwarp1 $newwarp2 \n";
+
+    print "difftool -definewarpwarp -exp_id $oldexp2 -template_exp_id $oldexp1 -input_label OC133.OSS.20101117 -template_label OC133.OSS.20101117  -timediff 3600 -set_label OC133.OSS.20101117  -pretend -dbname gpc1 -set_workdir neb://\@HOST\@.0/gpc1/OC133.OSS.20101117  -simple";
+    print "\n";
+    $count = `difftool -definewarpwarp -exp_id $oldexp2 -template_exp_id $oldexp1 -input_label OC133.OSS.20101117 -template_label OC133.OSS.20101117 -timediff 3600 -set_label OC133.OSS.20101117  -dbname gpc1 -set_workdir neb://\@HOST\@.0/gpc1/OC133.OSS.20101117  -simple | wc -l`;
+    chomp $count;
+
+    $total = $count+$total;
+}
+
+
+print $total;
Index: branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.pl	(revision 29834)
+++ branches/eam_branches/ipp-20101103/tools/heather/reprocess.OSS.pl	(revision 29834)
@@ -0,0 +1,71 @@
+#!/usr/bin/perl -w
+use DBI;
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
+
+
+my $dbname = 'gpc1';
+my $dbserver = 'ippdb01';
+my $dbuser = 'ippuser';
+my $dbpass = 'ippuser';
+
+my %OSS_orig= ();
+my %OSS_reprocessed = ();
+
+
+my $db = DBI->connect(
+   "DBI:mysql:database=${dbname};host=${dbserver};mysql_socket=" .
+   DB_SOCKET(),${dbuser},${dbpass}, 
+   { RaiseError => 1, AutoCommit => 1}
+   ) or die "Unable to connect to database $DBI::errstr\n";
+
+my $qdiffs = "select distinct(warp1), warp2 from diffRun join diffInputSkyfile using (diff_id) where data_group = 'OSS.20101012' or data_group = 'OSS.20101011';";
+
+my $qosswarpsorig = "SELECT rawExp.exp_id,warpRun.warp_id FROM   warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id and (chipRun.data_group = 'OSS.20101012' or chipRun.data_group = 'OSS.20101011') ;";
+
+my $qosswarpsrep = "SELECT rawExp.exp_id,warpRun.warp_id FROM   warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id and (chipRun.data_group = 'MopsTest.20100708') ;";
+
+$diffs = $db->selectall_arrayref( $qdiffs ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+$osswarpsorig = $db->selectall_arrayref( $qosswarpsorig ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+$osswarpsrep = $db->selectall_arrayref( $qosswarpsrep ) ||
+        die "Unable to execute SQL: $DBI::errstr\n";
+
+$db->disconnect();
+
+
+foreach $it (@{$osswarpsorig}) {
+   
+    $OSS_orig{${$it}[1] } = ${$it}[0];   #oss orig will sort by warps -> exp_id
+
+}
+
+foreach $it (@{$osswarpsrep}) {
+   
+    $OSS_reprocessed{${$it}[0] } = ${$it}[1];   #oss rep will sort by exp_id -> warps
+
+}
+
+$total =0;
+
+foreach $it (@{$diffs}) {
+    my ($oldexp1, $newwarp1, $oldexp2, $newwarp2);
+
+    $oldexp1= $OSS_orig{${$it}[0]};
+    $newwarp1= $OSS_reprocessed{$oldexp1};
+
+    $oldexp2= $OSS_orig{${$it}[1]};
+    $newwarp2= $OSS_reprocessed{$oldexp2};
+
+  #  print "${$it}[0] ${$it}[1] $oldexp1 $oldexp2 $newwarp1 $newwarp2 \n";
+
+    print "difftool -definewarpwarp -exp_id $oldexp2 -template_exp_id $oldexp1 -input_label MopsTest.20100708 -template_label MopsTest.20100708 -timediff 3600 -set_label MopsTest.20100708 -pretend -dbname gpc1 -set_workdir neb://\@HOST\@.0/gpc1/MopsTest.20100708 -simple";
+    print "\n";
+    $count = `difftool -definewarpwarp -exp_id $oldexp2 -template_exp_id $oldexp1 -input_label MopsTest.20100708 -template_label MopsTest.20100708 -timediff 3600 -set_label MopsTest.20100708 -dbname gpc1 -set_workdir neb://\@HOST\@.0/gpc1/MopsTest.20100708 -simple | wc -l`;
+    chomp $count;
+
+    $total = $count+$total;
+}
+
+
+print $total;
Index: branches/eam_branches/ipp-20101103/tools/roboczar.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/roboczar.pl	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/roboczar.pl	(revision 29834)
@@ -13,4 +13,5 @@
 use czartool::Plotter;
 use czartool::Burntool;
+use czartool::StageMetrics;
 
 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
@@ -56,5 +57,5 @@
 
         sendEmail(
-                "roydhenderson\@gmail.com", 
+                "ps-ipp-ops\@ifa.hawaii.edu", 
                 "Roboczar warnings", 
                 "$message\n");
@@ -89,32 +90,14 @@
     my $stage;
     foreach $stage (@stages) {
-        my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
-        $czarDb->runAnalysis(
-                "all_stdscience_labels", 
-                $stage, 
-                $begin, 
-                $end, 
-                \$started, 
-                \$finished, 
-                \$stuck, 
-                \$processed, 
-                \$pending, 
-                \$faults, 
-                \$totalTime);
 
-        print "*     $stage $end $begin:\n";
-        if (defined $started) {print "* Processing started at $started\n";}
-        if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
-        else {print "* Processing has not finished\n";}
-        if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
-        print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
+    my $stageMetrics = new czartool::StageMetrics($stage, "all_stdscience_labels", $begin, $end);
 
-        print "*******************************************************************************\n";
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
 
-        if ($stuck && $stage eq "burntool" && !$worryAboutBurntool) {next;}
+        if ($stageMetrics->getStuck() && $stage eq "burntool" && !$worryAboutBurntool) {next;}
 
-        if ($stuck) {
+        if ($stageMetrics->getStuck()) {
         
-            ${$message} = ${$message} ."\n - '$stage' is stuck with $pending pending exposures (and $faults faults)";
+            ${$message} = ${$message} ."\n - '$stage' is stuck with ".$stageMetrics->getFinalPending()." pending exposures (and ".$stageMetrics->getFaults()." faults)";
             $anyStuckStages = 1;
         }
Index: branches/eam_branches/ipp-20101103/tools/runchipimfile.pl
===================================================================
--- branches/eam_branches/ipp-20101103/tools/runchipimfile.pl	(revision 29828)
+++ branches/eam_branches/ipp-20101103/tools/runchipimfile.pl	(revision 29834)
@@ -38,5 +38,5 @@
 
 my $query1 = "SELECT chipProcessedImfile.path_base, camera, exp_id, rawImfile.magicked AS raw_magicked, (rawImfile.user_1 is not NULL and rawImfile.user_1 > 0.5)  As deburned, chip_imfile_id, chipProcessedImfile.fault, rawImfile.uri, chipRun.reduction, chipRun.state";
-$query1 .= " FROM chipRun join chipProcessedImfile using(chip_id) JOIN chipImfile USING (chip_id, class_id) JOIN rawExp using(exp_id) JOIN rawImfile USING(exp_id) WHERE chip_id = $chip_id AND class_id = $class_id";
+$query1 .= " FROM chipRun join chipProcessedImfile using(chip_id,exp_id) JOIN chipImfile USING (chip_id, class_id) JOIN rawExp using(exp_id) JOIN rawImfile USING(exp_id, camera, class_id) WHERE chip_id = $chip_id AND class_id = '$class_id'";
 
 my $stmt1 = $dbh->prepare($query1);
@@ -55,6 +55,4 @@
 my $exp_id = $results->{exp_id};
 
-notdone
-
 die "cannot update database for a chip that is not faulted\n" if $update and !$fault;
 
@@ -69,10 +67,10 @@
     $run_state = 'new';
 } elsif ($state eq 'update') {
-    $state = 'update';
+    $run_state = 'update';
 } else {
-    die "unexpected warpRun.state found: $state\n";
+    die "unexpected chipRun.state found: $state\n";
 }
 
-my $command = "chip_imfile.pl --exp_id $exp_id --chip_id $chip_id --class_id $class_id --chip_imfile_id $chip_imfile_id --uri $uri --camera $camera --run-state $run_state --deburned $deburned --outroot $path_base" ;
+my $command = "chip_imfile.pl --exp_id $exp_id --chip_id $chip_id --chip_imfile_id $chip_imfile_id --class_id $class_id --uri $uri --camera $camera --run-state $run_state --deburned $deburned --outroot $path_base" ;
 
 $command .= " --magicked $magicked" if $magicked > 0;
@@ -83,5 +81,4 @@
 $command .= " --verbose" unless $no_verbose;
 $command .= " --dbname $dbname" if $dbname;
-
 
 if ($update) {