Index: /trunk/tools/czarplot.pl
===================================================================
--- /trunk/tools/czarplot.pl	(revision 29436)
+++ /trunk/tools/czarplot.pl	(revision 29437)
@@ -23,4 +23,5 @@
 my $timeSeries = undef;
 my $rate = undef;
+my $deriv = undef;
 my $showCleanup = undef;
 my $nebulous = undef;
@@ -29,5 +30,5 @@
 
 GetOptions (
-        "dbname|d=s" => \$czarDbName,
+        "dbname=s" => \$czarDbName,
         "label|l=s" => \$label,
         "stage|s=s" => \$stage,
@@ -41,4 +42,5 @@
         "cleanup|c" => \$showCleanup,
         "rate|r" => \$rate,
+        "deriv|d" => \$deriv,
         "timeseries|t" => \$timeSeries,
         "verbose|v" => \$verbose,
@@ -59,4 +61,7 @@
 if (!$rate) {
     print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";} 
+if (!$deriv) {
+    $deriv = 0;
+    print "* OPTIONAL: plot first derivative           -d                          (default=$deriv)\n";} 
 if (!$showCleanup) {
     $showCleanup = 0;
@@ -97,4 +102,5 @@
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
 
+
 my $plotter = new czartool::Plotter(
         $czarDb, 
@@ -128,5 +134,5 @@
     exit;
 }
-if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup);}
+if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);}
 if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
 if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
Index: /trunk/tools/czartool/CzarDb.pm
===================================================================
--- /trunk/tools/czartool/CzarDb.pm	(revision 29436)
+++ /trunk/tools/czartool/CzarDb.pm	(revision 29437)
@@ -271,5 +271,5 @@
 ###########################################################################
 sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup) = @_;
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -289,5 +289,5 @@
     $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed 
+        timestamp, pending, faults, processed 
         FROM $stage 
         WHERE label LIKE '$label' 
@@ -302,30 +302,70 @@
 
     my $lastProcessed = -1;
+    my $lastPending = -1;
+    my $lastFaults = -1;
+    my $lastTimestamp = undef;
     my $processed = 0;
+    my $runningProcessed = 0;
+    my $pending = 0;
+    my $faults = 0;
+    my $timestamp = undef;
     while (my @row = $query->fetchrow_array()) {
-        my ($timestamp, $pending, $faults, $thisProcessed) = @row;
-
-        if ($showCleanup) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        if ($showCleanup ) {
         
-        $processed = $thisProcessed - $minProcessed;
+            $runningProcessed = $thisProcessed - $minProcessed;
         }
         elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
             
-            $processed = $processed + ($thisProcessed - $lastProcessed);
-        }
-
-        if ($isLog) {
-
-            if ($processed < 0) {$processed = 0;}
-            if ($pending < 0) {$pending = 0;}
-            if ($faults < 0) {$faults = 0;}
-
-            $processed = log($processed + 1)/log(10);
-            $pending = log($pending + 1)/log(10);
+            $runningProcessed = $runningProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+
+        if ($firstDeriv) {
+
+            if (defined $lastTimestamp) {
+
+                my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+                
+                if ($thisProcessed > $lastProcessed){
+                $processed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+#print "$processed = ($thisProcessed - $lastProcessed)/$timeDiff\n";
+                }
+                else {$processed = 0;}
+                #$pending = abs($thisPending - $lastPending)/$timeDiff;    
+                #$faults = abs($thisFaults - $lastFaults)/$timeDiff;    
+            }
+            else {
+                $processed = 0;
+                $pending = 0;
+                $faults = 0; 
+            }
+        }
+        elsif ($isLog) {
+
+            $processed = ($runningProcessed < 0) ? 0 : $runningProcessed;
+            $pending =  ($thisPending < 0) ? 0 : $thisPending;
+            $faults =  ($thisFaults < 0) ? 0 : $thisFaults;
+
+            $processed = log($runningProcessed + 1)/log(10);
+            $pending = log($thisPending + 1)/log(10);
             $faults = log($faults + 1)/log(10);
         }
+        else {
+        
+            $processed = $runningProcessed;
+            $pending = $thisPending;
+            $faults = $thisFaults;
+        }
+
+        $timestamp = $self->getFormattedDate($thisTimestamp);
 
         print GNUDAT "$timestamp $pending $faults $processed\n";
+
         $lastProcessed = $thisProcessed;
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $lastTimestamp = $thisTimestamp;
     }
 
@@ -568,4 +608,167 @@
 ###########################################################################
 #
+# When did this stage finish? 
+#
+###########################################################################
+sub getFinishTime {
+    my ($self, $label, $stage, $begin, $end) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults
+        FROM $stage
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+SQL
+    $query->execute;
+
+    my $array = $query->fetchall_arrayref();
+
+    my $row;
+    my $thisPending = -1;
+    my $lastPending = -1;
+    my $thisFaults = -1;
+    my $lastFaults = -1;
+    my $timestamp;
+    my $started = undef;
+    my $finished = undef;
+    my $thisLeft = 0;
+    my $lastLeft = 0;
+    my $finishTimeout = "01:00:00";
+    my $numRows = @{$array};
+    print "$numRows rows\n";
+    my $n = 0;
+    foreach $row ( @{$array} ){
+
+        ($timestamp, $thisPending, $thisFaults) = @{$row};
+
+#        print "$timestamp, $thisPending, $thisFaults\n";
+
+        $thisLeft = $thisPending - $thisFaults;
+        $lastLeft = $lastPending - $lastFaults;
+
+
+        if ($n == 0 && $thisLeft > 0) {
+
+            print "Starting this time period with stuff pending ($thisLeft)\n";
+            $started = $timestamp;
+        }
+
+        if (!defined $started && $lastLeft == 0 && $thisLeft > 0) {
+
+            $started = $timestamp;
+            print "STARTED at $started\n";
+        }
+
+        if ($started && !defined $finished && $thisLeft == 0) {
+
+            $finished = $timestamp;
+            print "setting FINISHED to $finished\n";
+        }
+
+        if (defined $finished) {
+
+            if ($thisLeft != 0) {
+                
+                $finished = undef;
+            
+                print "NEW pending  at $timestamp\n";
+            
+            }
+            elsif ($thisLeft == 0) {
+                
+                my $howLongFinished = $self->diffTimes($timestamp, $finished); 
+
+print "$howLongFinished = $timestamp, $finished\n";
+
+                if ($self->isIntervalGreaterThan($howLongFinished, $finishTimeout)) {
+                    
+                    print "0 pending for $howLongFinished. We're done \n";
+                    last;
+                }
+
+            }
+
+        }
+
+
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $n++;
+        if ($n == $numRows) {
+
+            if ($thisLeft == 0) {
+
+                print "exceeded interval before reaching finished timeout\n";
+            }
+            else {
+                print "Reached end of interval and not finished yet. $thisLeft left\n";
+            }
+        }
+    }
+
+    if ($finished) {
+
+        my $timeTaken = $self->diffTimes($finished, $started);
+        print "FINISHED at $finished with $thisFaults remaining faults, time taken = $timeTaken\n";
+    }
+
+}
+
+###########################################################################
+#
+# Figures out if a stage has plateaued 
+#
+###########################################################################
+sub hasPlateaued {
+    my ($self, $label, $stage, $end, $interval) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$end - INTERVAL $interval'
+        AND timestamp <= '$end'
+        ORDER BY timestamp DESC;
+SQL
+    $query->execute;
+
+    my $array = $query->fetchall_arrayref();
+
+    my $thisPending = -1;
+    my $lastPending = -1;
+    my $thisFaults = -1;
+    my $lastFaults = -1;
+    my $timestamp;
+    my $thisLeft = 0;
+    my $lastLeft = 0;
+    my $row;
+    my $numRows = @{$array};
+    my $n = 0;
+    foreach $row ( @{$array} ){
+
+        ($timestamp, $thisPending, $thisFaults) = @{$row};
+
+        $thisLeft = $thisPending - $thisFaults;
+        $lastLeft = $lastPending - $lastFaults;
+
+        print "$timestamp $thisPending\n";
+
+        if ($n>0) {
+
+            #if () {}
+
+        }
+
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $n++;
+    }
+}
+
+###########################################################################
+#
 # Gets count of processed stuff in a given time period 
 #
@@ -582,5 +785,5 @@
         AND timestamp < '$end' 
 SQL
-    $query->execute;
+        $query->execute;
 
     my $processedArray = $query->fetchall_arrayref();
@@ -614,5 +817,5 @@
 
         if (!$self->isBefore($thisDay, $endDay)) {
-        
+
             $quit = 1;
         }
@@ -645,9 +848,9 @@
         $toTime = $self->addInterval($fromTime, $interval);
         if (!$self->isBefore($toTime, $endDay)) {
-        
+
             $toTime = $endDay;
             $quit = 1;
         }
-   
+
         my $stage = undef;
         $totalDeleted = 0;
@@ -703,5 +906,5 @@
                     AND server = '$server' 
 SQL
-                $query->execute;
+                    $query->execute;
 
                 my $toDelete = scalar $query->fetchrow_array() - 1;
@@ -714,5 +917,5 @@
                         AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
 SQL
-                    $query->execute;
+                        $query->execute;
 
                     $totalDeleted += $toDelete;
@@ -728,5 +931,5 @@
             AND timestamp <= '$toTime'
 SQL
-        $query->execute;
+            $query->execute;
 
         my $toDelete = scalar $query->fetchrow_array() - 1;
@@ -739,5 +942,5 @@
                 ORDER BY timestamp DESC LIMIT $toDelete
 SQL
-            $query->execute;
+                $query->execute;
 
             $totalDeleted += $toDelete;
@@ -755,5 +958,5 @@
 ###########################################################################
 sub optimize {
-        my ($self) = @_;
+    my ($self) = @_;
 
     my $stage = undef;
@@ -782,5 +985,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
     my $numOfReadings = scalar $query->fetchrow_array();
 
@@ -795,5 +998,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
     my $numNotRunning = scalar $query->fetchrow_array();
 
@@ -893,5 +1096,5 @@
     INSERT INTO revision (revision) VALUES ($revision);
 SQL
-    $query->execute;
+        $query->execute;
 }
 
@@ -912,5 +1115,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
     my @row = $query->fetchrow_array();
 
@@ -940,5 +1143,5 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         CREATE TABLE $stage (
                 timestamp TIMESTAMP DEFAULT NOW(),
@@ -969,5 +1172,5 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         ALTER TABLE $stage 
             ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
@@ -994,10 +1197,10 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         ALTER TABLE $stage 
             ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
 SQL
 
-      $query->execute;
+            $query->execute;
     }
 
@@ -1019,5 +1222,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
 
     $self->setRevision(4);
@@ -1041,5 +1244,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
 
     $self->setRevision(5);
@@ -1065,12 +1268,12 @@
 SQL
 
-      $query->execute;
-    }
-
-        my $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX serverIndex ON servers (timestamp, server);
-SQL
-
-      $query->execute;
+            $query->execute;
+    }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX serverIndex ON servers (timestamp, server);
+SQL
+
+        $query->execute;
 
 
@@ -1093,5 +1296,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     $self->setRevision(7);
@@ -1117,5 +1320,5 @@
 SQL
 
-    $query->execute;
+            $query->execute;
     }
 
@@ -1127,5 +1330,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     # insert stages into revert table
@@ -1137,5 +1340,5 @@
             ('$stage', 0);
 SQL
-       $query->execute;
+            $query->execute;
     }
 
@@ -1162,5 +1365,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
     $query = $self->{_db}->prepare(<<SQL);
     CREATE TABLE hosts (
@@ -1171,11 +1374,11 @@
 SQL
 
-      $query->execute;
-
-        $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
-SQL
-
-      $query->execute;
+        $query->execute;
+
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
+SQL
+
+        $query->execute;
 
     $self->setRevision(9);
@@ -1195,8 +1398,8 @@
     ALTER TABLE hosts 
         ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
-        ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
-SQL
-
-    $query->execute;
+            ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
+SQL
+
+        $query->execute;
 
     $self->setRevision(10);
@@ -1223,10 +1426,10 @@
 SQL
 
-      $query->execute;
-        $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX burntoolIndex ON burntool (timestamp, label);
-SQL
-
-      $query->execute;
+        $query->execute;
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX burntoolIndex ON burntool (timestamp, label);
+SQL
+
+        $query->execute;
 
     $self->setRevision(11);
@@ -1250,5 +1453,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     $self->setRevision(12);
Index: /trunk/tools/czartool/Plotter.pm
===================================================================
--- /trunk/tools/czartool/Plotter.pm	(revision 29436)
+++ /trunk/tools/czartool/Plotter.pm	(revision 29437)
@@ -130,5 +130,5 @@
 ###########################################################################
 sub createTimeSeries {
-    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup) = @_;
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_;
 
     my $minX = 999999999;      
@@ -156,5 +156,6 @@
                     \$gnuplotFile,
                     $isLog,
-                    $showCleanup)) {
+                    $showCleanup,
+                    $deriv)) {
 
             $gnuplotFiles{$stage} = $gnuplotFile;
@@ -186,5 +187,6 @@
             $minX, 
             $timeDiff, 
-            $isLog);
+            $isLog,
+            $deriv);
 }                                                    
 
@@ -325,20 +327,22 @@
 ###########################################################################
 sub plotTimeSeries {
-    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog) = @_;
+    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog, $isDeriv) = @_;
 
     my $timeFormat = undef;
     my $divX = undef;
     my $yTitle = undef;
-    if ($isLog) {$yTitle = "Log( numExposures )";}
-    else {$yTitle = "numExposures";}
+    if ($isLog) {$yTitle = "Log( Exposures )";}
+    elsif ($isDeriv) {$yTitle = "dExposures/dTime";}
+    else {$yTitle = "Exposures";}
 
     $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
 
     my $numOfPlots = keys %$gnuplotFiles;
-    my $title = undef;
 
     # sort out plot title
-    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}}
-    else {$title = "'All stages'"}
+    my $title = "";
+    if ($isDeriv) {$title .= "First derivatives of "}
+    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
+    else {$title .= "'all stages'"}
 
     $title .= " for '$label', '$fromTime' to '$toTime'";
@@ -369,6 +373,6 @@
         if ($numOfPlots == 1) {
 
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
-            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7 lw 2";
         }