Index: /trunk/tools/czartool/CzarDb.pm
===================================================================
--- /trunk/tools/czartool/CzarDb.pm	(revision 29581)
+++ /trunk/tools/czartool/CzarDb.pm	(revision 29582)
@@ -267,4 +267,190 @@
 ###########################################################################
 #
+# Analysis this stage between these times 
+#
+###########################################################################
+sub runAnalysis {
+    my ($self, 
+            $label, 
+            $stage, 
+            $fromTime, 
+            $toTime, 
+            $_started, 
+            $_finished, 
+            $_stuck,
+            $_processed,
+            $_pending,
+            $_faults,
+            $_totalTime) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults, processed 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    $query->execute;
+
+    my $lastProcessed = -1;
+    my $lastPending = -1;
+    my $lastFaults = -1;
+    my $lastTimestamp = undef;
+    my ($linearProcessed, $linearPending, $linearFaults);
+    my ($derivProcessed, $derivPending, $derivFaults);
+    my $started = undef;
+    my $finished = undef;
+    
+    my $notProcessing = undef;
+    my $howLongNotProcessing = undef;
+    my $notProcessingLongerThanInterval = 0;
+
+    my $stuck = undef;
+    my $howLongStuck = undef;
+    my $stuckLongerThanInterval = 0;
+
+    my $stuffToDo = undef;
+    my $interval = "00:20:00";
+
+    $linearProcessed = 0;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed);
+        }
+        $linearPending = $thisPending;
+        $linearFaults = $thisFaults;
+
+        # get first derivative values
+        if (defined $lastTimestamp) {
+
+            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+                
+            if ($thisProcessed >= $lastProcessed){
+                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+            }
+            else {$derivProcessed = 0;}
+            $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
+            $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
+        }
+        else {
+        
+            $derivProcessed = 0;
+            $derivPending = 0;
+            $derivFaults = 0;
+        }
+
+        # analysis
+        if (defined $lastTimestamp) {
+
+            # anything to do?
+            $stuffToDo = $linearPending - $linearFaults;
+
+            # we are not processing
+            if ($derivProcessed < 0.001 ) {
+
+                if (!defined $notProcessing) {$notProcessing = $lastTimestamp;}
+                if ($stuffToDo && !defined $stuck) {$stuck = $lastTimestamp;}
+            }
+
+            # we are processing
+            else {
+                if (defined $notProcessing) {
+                    
+                    if ($stuckLongerThanInterval) {
+
+#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
+                    }
+                    if ($notProcessingLongerThanInterval) {
+
+            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
+                    }
+                    $notProcessing = undef; 
+                    $stuck = undef; 
+                }
+                if (!defined $started) {
+                    $started = $lastTimestamp;
+                    $finished = undef; 
+#                    print "* $stage processing started at $started\n";
+                }
+            }
+
+            # how long stuck?
+            if (defined $stuck) {
+
+                $howLongStuck = $self->diffTimes($lastTimestamp, $stuck);
+                $stuckLongerThanInterval = $self->isBefore($interval, $howLongStuck)
+            }
+            else {
+
+                $howLongStuck = 0;
+                $stuckLongerThanInterval = 0;
+            }
+
+            # how long not processing?
+            if (defined $notProcessing) {
+                
+                $howLongNotProcessing = $self->diffTimes($lastTimestamp, $notProcessing);
+                $notProcessingLongerThanInterval = $self->isBefore($interval, $howLongNotProcessing);
+            }
+            else {
+            
+                $howLongNotProcessing = 0;
+                $notProcessingLongerThanInterval = 0;
+            }
+
+            if ($stuffToDo) {$finished = undef;}
+            elsif (!defined $finished && $started && $notProcessingLongerThanInterval) {
+
+                $finished = $notProcessing;
+            }
+        }
+
+        $lastProcessed = $thisProcessed;
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $lastTimestamp = $thisTimestamp;
+    }
+
+    ${$_processed} = $linearProcessed;
+    ${$_pending} = $stuffToDo;
+    ${$_faults} = $lastFaults;
+
+    if (defined $finished) {
+        ${$_finished} = $finished;
+        ${$_totalTime} = $self->diffTimes($finished, $started);
+    }
+
+    elsif(defined $started) {
+    
+        ${$_finished} = undef;
+    }
+    
+    if (!defined $started){
+    
+        ${$_started} = undef;
+    }
+    else {
+    
+        ${$_started} = $started;
+    }
+    if ($stuck && $stuckLongerThanInterval && !defined $finished) {
+
+        ${$_stuck} = $stuck;
+    }
+    else {
+    
+        ${$_stuck} = undef;
+    }
+
+    return 1;
+}
+###########################################################################
+#
 # Gets time series data and stores it to temp file
 #
@@ -282,5 +468,5 @@
 
     if (!$query->execute) {return 0;}
-    my $minProcessed = scalar  $query->fetchrow_array(); # TODO remove
+    my $minProcessed = scalar  $query->fetchrow_array();
     if (!defined $minProcessed) {return 0;}
 
@@ -305,62 +491,59 @@
     my $lastFaults = -1;
     my $lastTimestamp = undef;
-    my $processed = 0;
-    my $runningProcessed = 0;
-    my $pending = 0;
-    my $faults = 0;
+    my ($linearProcessed, $linearPending, $linearFaults);
+    my ($logProcessed, $logPending, $logFaults);
+    my ($derivProcessed, $derivPending, $derivFaults);
     my $timestamp = undef;
+    
+    $linearProcessed = 0;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
 
-        if ($showCleanup ) {
+        # get linear values
+        if ($showCleanup) {
         
-            $runningProcessed = $thisProcessed - $minProcessed;
+            $linearProcessed = $thisProcessed - $minProcessed;
         }
         elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
             
-            $runningProcessed = $runningProcessed + ($thisProcessed - $lastProcessed);
+            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed);
         }
-
-
-        if ($firstDeriv) {
-
-            if (defined $lastTimestamp) {
-
-                my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+        $linearPending = $thisPending;
+        $linearFaults = $thisFaults;
+
+        # get first derivative values
+        if (defined $lastTimestamp) {
+
+            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-                if ($thisProcessed > $lastProcessed){
-                $processed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
-#print "$processed = ($thisProcessed - $lastProcessed)/$timeDiff\n";
-                }
-                else {$processed = 0;}
-                #$pending = abs($thisPending - $lastPending)/$timeDiff;    
-                #$faults = abs($thisFaults - $lastFaults)/$timeDiff;    
+            if ($thisProcessed >= $lastProcessed){
+                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
-            else {
-                $processed = 0;
-                $pending = 0;
-                $faults = 0; 
-            }
-        }
-        elsif ($isLog) {
-
-            $processed = ($runningProcessed < 0) ? 0 : $runningProcessed;
-            $pending =  ($thisPending < 0) ? 0 : $thisPending;
-            $faults =  ($thisFaults < 0) ? 0 : $thisFaults;
-
-            $processed = log($runningProcessed + 1)/log(10);
-            $pending = log($thisPending + 1)/log(10);
-            $faults = log($faults + 1)/log(10);
+            else {$derivProcessed = 0;}
+            $derivPending = 0;
+            $derivFaults = 0;
+        #    $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
+        #    $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
         }
         else {
         
-            $processed = $runningProcessed;
-            $pending = $thisPending;
-            $faults = $thisFaults;
+            $derivProcessed = 0;
+            $derivPending = 0;
+            $derivFaults = 0;
         }
 
+        # get log values
+        $logProcessed = ($linearProcessed < 0) ? 0 : $linearProcessed;
+        $logPending =  ($linearPending < 0) ? 0 : $linearPending;
+        $logFaults =  ($linearFaults < 0) ? 0 : $linearFaults;
+        $logProcessed = log($logProcessed + 1)/log(10);
+        $logPending = log($logPending + 1)/log(10);
+        $logFaults = log($logFaults + 1)/log(10);
+
+        # print to data file
         $timestamp = $self->getFormattedDate($thisTimestamp);
-
-        print GNUDAT "$timestamp $pending $faults $processed\n";
+        if ($firstDeriv) {print GNUDAT "$timestamp $derivPending $derivFaults $derivProcessed\n";}
+        elsif($isLog) {print GNUDAT "$timestamp $logPending $logFaults $logProcessed\n";}
+        else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
 
         $lastProcessed = $thisProcessed;
@@ -608,167 +791,4 @@
 ###########################################################################
 #
-# When did this stage finish? 
-#
-###########################################################################
-sub getFinishTime {
-    my ($self, $label, $stage, $begin, $end) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        timestamp, pending, faults
-        FROM $stage
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$begin' 
-        AND timestamp <= '$end' 
-SQL
-    $query->execute;
-
-    my $array = $query->fetchall_arrayref();
-
-    my $row;
-    my $thisPending = -1;
-    my $lastPending = -1;
-    my $thisFaults = -1;
-    my $lastFaults = -1;
-    my $timestamp;
-    my $started = undef;
-    my $finished = undef;
-    my $thisLeft = 0;
-    my $lastLeft = 0;
-    my $finishTimeout = "01:00:00";
-    my $numRows = @{$array};
-    print "$numRows rows\n";
-    my $n = 0;
-    foreach $row ( @{$array} ){
-
-        ($timestamp, $thisPending, $thisFaults) = @{$row};
-
-#        print "$timestamp, $thisPending, $thisFaults\n";
-
-        $thisLeft = $thisPending - $thisFaults;
-        $lastLeft = $lastPending - $lastFaults;
-
-
-        if ($n == 0 && $thisLeft > 0) {
-
-            print "Starting this time period with stuff pending ($thisLeft)\n";
-            $started = $timestamp;
-        }
-
-        if (!defined $started && $lastLeft == 0 && $thisLeft > 0) {
-
-            $started = $timestamp;
-            print "STARTED at $started\n";
-        }
-
-        if ($started && !defined $finished && $thisLeft == 0) {
-
-            $finished = $timestamp;
-            print "setting FINISHED to $finished\n";
-        }
-
-        if (defined $finished) {
-
-            if ($thisLeft != 0) {
-                
-                $finished = undef;
-            
-                print "NEW pending  at $timestamp\n";
-            
-            }
-            elsif ($thisLeft == 0) {
-                
-                my $howLongFinished = $self->diffTimes($timestamp, $finished); 
-
-print "$howLongFinished = $timestamp, $finished\n";
-
-                if ($self->isIntervalGreaterThan($howLongFinished, $finishTimeout)) {
-                    
-                    print "0 pending for $howLongFinished. We're done \n";
-                    last;
-                }
-
-            }
-
-        }
-
-
-        $lastPending = $thisPending;
-        $lastFaults = $thisFaults;
-        $n++;
-        if ($n == $numRows) {
-
-            if ($thisLeft == 0) {
-
-                print "exceeded interval before reaching finished timeout\n";
-            }
-            else {
-                print "Reached end of interval and not finished yet. $thisLeft left\n";
-            }
-        }
-    }
-
-    if ($finished) {
-
-        my $timeTaken = $self->diffTimes($finished, $started);
-        print "FINISHED at $finished with $thisFaults remaining faults, time taken = $timeTaken\n";
-    }
-
-}
-
-###########################################################################
-#
-# Figures out if a stage has plateaued 
-#
-###########################################################################
-sub hasPlateaued {
-    my ($self, $label, $stage, $end, $interval) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        timestamp, pending, faults
-        FROM $stage
-        WHERE label LIKE '$label'
-        AND timestamp >= '$end - INTERVAL $interval'
-        AND timestamp <= '$end'
-        ORDER BY timestamp DESC;
-SQL
-    $query->execute;
-
-    my $array = $query->fetchall_arrayref();
-
-    my $thisPending = -1;
-    my $lastPending = -1;
-    my $thisFaults = -1;
-    my $lastFaults = -1;
-    my $timestamp;
-    my $thisLeft = 0;
-    my $lastLeft = 0;
-    my $row;
-    my $numRows = @{$array};
-    my $n = 0;
-    foreach $row ( @{$array} ){
-
-        ($timestamp, $thisPending, $thisFaults) = @{$row};
-
-        $thisLeft = $thisPending - $thisFaults;
-        $lastLeft = $lastPending - $lastFaults;
-
-        print "$timestamp $thisPending\n";
-
-        if ($n>0) {
-
-            #if () {}
-
-        }
-
-        $lastPending = $thisPending;
-        $lastFaults = $thisFaults;
-        $n++;
-    }
-}
-
-###########################################################################
-#
 # Gets count of processed stuff in a given time period 
 #
@@ -972,14 +992,36 @@
 ###########################################################################
 #
+# Returns if a particular server has been stopped for a certain interval 
+#
+###########################################################################
+sub isServerStopped {
+    my ($self, $server, $interval, $since) = @_;
+
+    return $self->getServerStatus($server, $interval, $since, "running");
+}
+
+###########################################################################
+#
 # Returns if a particular server has been down for a certain interval 
 #
 ###########################################################################
 sub isServerDown {
-    my ($self, $server, $interval) = @_;
+    my ($self, $server, $interval, $since) = @_;
+
+    return $self->getServerStatus($server, $interval, $since, "alive");
+}
+
+###########################################################################
+#
+# Returns if a particular server has been down for a certain interval 
+#
+###########################################################################
+sub getServerStatus {
+    my ($self, $server, $interval, $since, $mode) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT COUNT(*) 
         FROM servers
-        WHERE server = "$server" 
+        WHERE server = '$server' 
         AND timestamp > now() - INTERVAL $interval
 SQL
@@ -993,7 +1035,7 @@
     SELECT COUNT(*) 
         FROM servers
-        WHERE server = "$server" 
+        WHERE server = '$server' 
         AND timestamp > now() - INTERVAL $interval
-        AND !running
+        AND !$mode
 SQL
 
@@ -1001,7 +1043,20 @@
     my $numNotRunning = scalar $query->fetchrow_array();
 
-    if ($numOfReadings == $numNotRunning) {return 1;}
-
-    return 0;
+    if ($numOfReadings != $numNotRunning) {return 0;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp
+        FROM servers
+        WHERE server = '$server'
+        AND $mode 
+        ORDER BY timestamp DESC 
+        LIMIT 1
+
+SQL
+    $query->execute;
+    ${$since} = scalar $query->fetchrow_array();
+
+    return 1;
 }