Index: /trunk/ippMonitor/czartool/roboczar.pl
===================================================================
--- /trunk/ippMonitor/czartool/roboczar.pl	(revision 32098)
+++ /trunk/ippMonitor/czartool/roboczar.pl	(revision 32098)
@@ -0,0 +1,151 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+# local classes
+use czartool::Config;
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+use czartool::Nebulous;
+use czartool::Plotter;
+use czartool::Burntool;
+use czartool::StageMetrics;
+
+my $save_temps = 0;
+
+my $config = new czartool::Config();
+my $czarDb = $config->getCzarDbInstance();
+
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks
+
+my @interestedServers = $config->getRoboczarInterestedServers();
+
+my $stuckMessage;
+my $serversMessage;
+my $message;
+my $lastMessage = "";
+my $anythingToReport;
+while(1) {
+
+    $anythingToReport = 0;
+
+    $message = "";
+
+    if (anyStoppedServers($config->getRoboczarServerInterval(), \$serversMessage)) {
+
+        $message .= "\n\n" . $serversMessage;
+        $anythingToReport = 1;
+    }
+#    if (anyStuckStages("2 HOUR", \$stuckMessage)) {
+
+ #       $message .= "\n\n" . $stuckMessage;
+  #      $anythingToReport = 1;
+   # }
+
+    if ($anythingToReport && $message ne $lastMessage) {
+    
+print "\n\n$message\n\n";
+
+        sendEmail($config->getRoboczarEmail(),
+                "Roboczar warnings", 
+                "$message\n");
+    }
+
+    $lastMessage = $message;
+
+    print "* Going to sleep\n";
+    sleep(1200);
+}
+
+
+###########################################################################
+#
+# Checks if anything is stuck
+#
+###########################################################################
+sub anyStuckStages {
+    my ($interval, $message) = @_;
+
+    my $end = $czarDb->getNowTimestamp();
+    #$end = '2010-10-24 15';
+    my $begin = $czarDb->subtractInterval($end, $interval);
+    my $anyStuckStages = 0;
+
+    # exception - we don't care if burntool is stalled before 6:30am TODO we do now
+    my $burntime = strftime('%Y-%m-%d 06:35', localtime);
+    my $worryAboutBurntool = $czarDb->isBefore($burntime, $end);
+
+    ${$message} = "Processing stages:\n";
+
+    my $stage;
+    foreach $stage (@stages) {
+
+    my $stageMetrics = new czartool::StageMetrics($stage, "all_stdscience_labels", $begin, $end);
+
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
+
+        if ($stageMetrics->getStuck() && $stage eq "burntool" && !$worryAboutBurntool) {next;}
+
+        if ($stageMetrics->getStuck()) {
+        
+            ${$message} = ${$message} ."\n - '$stage' is stuck with ".$stageMetrics->getFinalPending()." pending exposures (and ".$stageMetrics->getFaults()." faults)";
+            $anyStuckStages = 1;
+        }
+    }
+
+    return $anyStuckStages;
+}
+
+###########################################################################
+#
+# Checks that the important servers are running 
+#
+###########################################################################
+sub anyStoppedServers {
+    my ($interval, $message) = @_;
+
+    my $anythingToReport = 0;
+    my $server;
+    my $since;
+
+    ${$message} = "Pantasks servers:\n";
+
+    foreach $server (@interestedServers) {
+
+        # is server alice?
+        if ($czarDb->isServerDown($server, $interval, \$since)) {
+
+            ${$message} = ${$message} . "\n - '$server' has been DOWN since '$since'";
+            $anythingToReport = 1;
+        }
+        # is it running?
+        elsif ($czarDb->isServerStopped($server, $interval, \$since)) {
+
+            ${$message} = ${$message} . "\n - '$server' has been stopped since '$since'";
+            $anythingToReport = 1;
+        }
+    }
+
+    return $anythingToReport;
+} 
+
+###########################################################################
+#
+# Sends an email 
+#
+###########################################################################
+sub sendEmail {
+    my ($to, $subject, $message) = @_;
+
+    my $sendmail = '/usr/lib/sendmail';
+    open(MAIL, "|$sendmail -oi -t");
+    print MAIL "From: roboczar\@ipp.org\n";
+    print MAIL "To: $to\n";
+    print MAIL "Subject: $subject\n\n";
+    print MAIL "$message\n";
+    close(MAIL);
+}