Index: /trunk/tools/czarpoll.pl
===================================================================
--- /trunk/tools/czarpoll.pl	(revision 29252)
+++ /trunk/tools/czarpoll.pl	(revision 29252)
@@ -0,0 +1,268 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+use czartool::Nebulous;
+use czartool::Plotter;
+use czartool::Burntool;
+
+my $period = 60;
+my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
+my $save_temps = 0;
+
+GetOptions (
+        "period|p=s" => \$period, # TODO more Db args
+        "dbname|d=s" => \$czarDbName,
+        );
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $nebulous = new czartool::Nebulous($czarDb);
+my $pantasks = new czartool::Pantasks();
+my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
+my $burntool = new czartool::Burntool();
+
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+timePoll($period);
+
+###########################################################################
+#
+# Updates the labels from pantasks for all interested servers 
+#
+###########################################################################
+sub updateLabels {
+
+    print "* Updating labels\n";
+    my @servers = ("stdscience", "distribution", "publishing", "update");
+
+    my $server = undef;
+    foreach $server (@servers) {
+
+        my @labels = @{$pantasks->getLabels($server)};
+        if (@labels) { 
+        
+            $czarDb->updateCurrentLabels($server, \@labels);
+        }
+        else {
+        
+             print "WARNING: No labels to update for '$server'\n";
+        }
+    }
+}
+
+###########################################################################
+#
+# Updates pantasks server status TODO should really get info for all servers at once 
+#
+###########################################################################
+sub updateServerStatus {
+    print "* Checking all pantasks servers\n";
+
+    my $servers = $pantasks->getServerList();
+
+    my $server = undef;
+    my $alive = undef;
+    my $running = undef;
+    foreach $server (@{$servers}) {
+
+        $pantasks->getServerStatus($server, \$alive, \$running);
+        $czarDb->updateServerStatus($server, $alive, $running);
+    }
+}
+
+###########################################################################
+#
+# Polls with provided period (seconds) 
+#
+###########################################################################
+sub timePoll {
+    my ($period) = @_;
+
+    my $label;
+    my $new;
+    my $full;
+    my $faults;
+    my $stage;
+    my $query = undef;
+    my $str = undef;
+    my $labels = undef;
+    my $updateLabels = undef;
+    my $row = undef;
+    my $begin = undef;
+    my $end = undef;
+    my $priority = undef;
+    my $newState = undef;
+    my $nsStatus = undef;
+
+    while (1) {
+
+        # sort out times
+        $begin =  strftime('%Y-%m-%d 06:35',localtime);
+        $end = $czarDb->getNowTimestamp();
+
+        if ($czarDb->isBefore($end, $begin)) {
+
+            $begin = $czarDb->subtractInterval($begin, "1 DAY");
+        }
+
+        # check nightly science status
+        print "* Checking nightly science status\n";
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus);
+
+        # check nebulous
+        print "* Checking Nebulous\n";
+        $nebulous->updateClusterSpaceInfo();
+        $plotter->plotDiskUsageHistogram();
+        updateServerStatus();
+
+        # check labels
+        updateLabels();
+
+        # servers to check
+        my @serversToCheck = ("stdscience", "update");
+
+        my $thisServer = undef;
+        foreach $thisServer (@serversToCheck) {
+
+            if ($thisServer eq "update") {$newState = "update";}
+            else {$newState = "new";}
+
+            # deal with stdscience labels
+            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
+            my $size = @{$labels};
+            if($size > 0) {
+
+                # get priority
+                foreach $row ( @{$labels} ) {
+                    my ($label) = @{$row};
+                    $priority = $gpc1Db->getPriority($label);
+                    $czarDb->setLabelPriority($label, $priority);
+                }
+
+                updateAllStages($thisServer, $newState, $labels, $begin, $end);
+                createPlots($thisServer, $labels, $begin, $end);
+            }
+            else { print "* WARNING: no $thisServer labels found in Db\n";}
+        }
+
+        print "--------------------------------------------------------------------------\n";
+        print "* Going to sleep\n";
+        sleep($period);
+        print "* Waking up\n";
+    };
+}
+
+###########################################################################
+#
+# Loops through labels and creates time series and histogram plots
+#
+###########################################################################
+sub createPlots {
+    my ($server, $rows, $begin, $end) = @_;
+
+    my $stage = undef;
+    my $row = undef;
+
+    print "* Generating plots\n";
+
+    # create plots for each label for each stage
+    foreach $stage (@stages) {
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
+        }
+    }
+
+    # create plots for each label for all stages
+    foreach $row ( @{$rows} ) {
+        my ($label) = @{$row};
+
+        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
+        $plotter->createHistogram($label, $begin, $end);
+
+        #routineChecks($label, "1 HOUR");
+    }
+    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
+    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
+    foreach $stage (@stages) {
+
+        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
+    }
+}
+
+###########################################################################
+#
+# Loops through some labels and updates processed/pending/faults in the Db
+#
+###########################################################################
+sub updateAllStages {
+    my ($labelServer, $newState, $rows, $begin, $end) = @_;
+
+    print "* Updating stage data\n";
+    my $totalNew = undef;
+    my $totalFaults = undef;
+    my $totalFull = undef;
+    my $stage = undef;
+    my $reverting = 0;
+    my $row = undef;
+    my $new = undef;
+    my $full = undef;
+    my $faults = undef;
+    my $server = undef;
+    my $state = undef;
+    
+    foreach $stage (@stages) {
+
+        $server = $pantasks->getServerForThisStage($stage);
+        $pantasks->getRevertStatus($stage, \$reverting);
+        $czarDb->updateRevertStatus($stage, $reverting);
+
+        print "* Checking labels for $stage stage\n";
+
+        $totalNew=$totalFaults=$totalFull=0;
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+
+            if ($stage eq "burntool") {
+
+                if ($labelServer eq "stdscience") {
+
+                    $burntool->getPendingAndProcessed($label, \$new, \$full);
+                    $faults = 0;
+                }
+                else { $new = $full = $faults = 0;}
+            }
+            else {
+
+                $new = $gpc1Db->countExposures($label, $stage, $newState);
+                $full = $gpc1Db->countExposures($label, $stage, "full");
+                $faults = $gpc1Db->countFaults($label, $stage, $newState);
+            }
+            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
+            $totalNew += $new;
+            $totalFull += $full;
+            $totalFaults += $faults;
+
+            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
+        }
+
+        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
+    }
+}
+