Index: /trunk/tools/roboczar.pl
===================================================================
--- /trunk/tools/roboczar.pl	(revision 29580)
+++ /trunk/tools/roboczar.pl	(revision 29581)
@@ -25,38 +25,134 @@
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
 
-my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
-my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration");
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks
+my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration", "pstamp");
 
 
+my $stuckMessage;
+my $serversMessage;
+my $message;
+my $lastMessage = "";
+my $anythingToReport;
 while(1) {
 
-    checkServers("20 MINUTE");
+    $anythingToReport = 0;
+
+    $message = "";
+
+    if (anyStoppedServers("20 MINUTE", \$serversMessage)) {
+
+        $message .= "\n\n" . $serversMessage;
+        $anythingToReport = 1;
+    }
+    if (anyStuckStages("2 HOUR", \$stuckMessage)) {
+
+        $message .= "\n\n" . $stuckMessage;
+        $anythingToReport = 1;
+    }
+
+    if ($anythingToReport && $message ne $lastMessage) {
+    
+print "\n\n$message\n\n";
+
+        sendEmail(
+                "roydhenderson\@gmail.com", 
+                "Roboczar warnings", 
+                "$message\n");
+    }
+
+    $lastMessage = $message;
+
+    print "* Going to sleep\n";
     sleep(1200);
+}
+
+
+###########################################################################
+#
+# Checks if anything is stuck
+#
+###########################################################################
+sub anyStuckStages {
+    my ($interval, $message) = @_;
+
+    my $end = $czarDb->getNowTimestamp();
+    #$end = '2010-10-24 15';
+    my $begin = $czarDb->subtractInterval($end, $interval);
+    my $anyStuckStages = 0;
+
+    # exception - we don't care if burntool is stalled before 6:30am
+    my $burntime = strftime('%Y-%m-%d 06:35', localtime);
+    my $worryAboutBurntool = $czarDb->isBefore($burntime, $end);
+
+    ${$message} = "Processing stages:\n";
+
+    my $stage;
+    foreach $stage (@stages) {
+        my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
+        $czarDb->runAnalysis(
+                "all_stdscience_labels", 
+                $stage, 
+                $begin, 
+                $end, 
+                \$started, 
+                \$finished, 
+                \$stuck, 
+                \$processed, 
+                \$pending, 
+                \$faults, 
+                \$totalTime);
+
+        print "*     $stage $end $begin:\n";
+        if (defined $started) {print "* Processing started at $started\n";}
+        if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
+        else {print "* Processing has not finished\n";}
+        if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
+        print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
+
+        print "*******************************************************************************\n";
+
+        if ($stuck && $stage eq "burntool" && !$worryAboutBurntool) {next;}
+
+        if ($stuck) {
+        
+            ${$message} = ${$message} ."\n - '$stage' is stuck with $pending pending exposures (and $faults faults)";
+            $anyStuckStages = 1;
+        }
+    }
+
+    return $anyStuckStages;
 }
 
 ###########################################################################
 #
-# Checks tha the important servers are running 
+# Checks that the important servers are running 
 #
 ###########################################################################
-sub checkServers {
-    my ($interval) = @_;
+sub anyStoppedServers {
+    my ($interval, $message) = @_;
 
+    my $anythingToReport = 0;
     my $server;
+    my $since;
+
+    ${$message} = "Pantasks servers:\n";
+
     foreach $server (@serversWeCareAbout) {
 
-        if ($czarDb->isServerDown($server, $interval)) {
+        # is server alice?
+        if ($czarDb->isServerDown($server, $interval, \$since)) {
 
-            print "$server has been down for the last $interval\n";
-            sendEmail(
-                    "roydhenderson\@gmail.com", 
-                    "roboczar\@ipp.com", 
-                    "Roboczar update", 
-                    "\n\n* '$server' server has been down for the last $interval\n\n");
+            ${$message} = ${$message} . "\n - '$server' has been DOWN since '$since'";
+            $anythingToReport = 1;
         }
-        else {
-            #print "$server has been running for some of the last $interval\n";
+        # is it running?
+        elsif ($czarDb->isServerStopped($server, $interval, \$since)) {
+
+            ${$message} = ${$message} . "\n - '$server' has been stopped since '$since'";
+            $anythingToReport = 1;
         }
     }
+
+    return $anythingToReport;
 } 
 
@@ -67,9 +163,9 @@
 ###########################################################################
 sub sendEmail {
-    my ($to, $from, $subject, $message) = @_;
+    my ($to, $subject, $message) = @_;
 
     my $sendmail = '/usr/lib/sendmail';
     open(MAIL, "|$sendmail -oi -t");
-    print MAIL "From: $from\n";
+    print MAIL "From: roboczar\@ipp.org\n";
     print MAIL "To: $to\n";
     print MAIL "Subject: $subject\n\n";