Changeset 40840

Timestamp:

Jul 23, 2019, 2:24:01 PM (7 years ago)

Author:

tdeboer

Message:

updates to ippMonitor and czartool to handle gpc2 labels and improve czartool

Location:

trunk/ippMonitor

Files:

• Makefile.in (modified) (2 diffs)
• config.dat.in (modified) (2 diffs)
• czartool/czarconfig.xml.in (modified) (1 diff)
• czartool/czarpoll.pl (modified) (17 diffs)
• czartool/czartool.pl (modified) (11 diffs)
• czartool/czartool/CzarDb.pm (modified) (9 diffs)
• czartool/czartool/Pantasks.pm (modified) (8 diffs)
• raw/czartool_getplot.php (modified) (1 diff)
• raw/czartool_labels.php (modified) (17 diffs)
• raw/ipp.czar.dat (modified) (2 diffs)
• raw/ipp.menu.dat (modified) (1 diff)
• raw/ipp.sky.dat (modified) (1 diff)

trunk/ippMonitor/Makefile.in

@@ -165 +165 @@
 $(DESTWWW)/warpFailedSkyfiles.php \
 $(DESTWWW)/failedWarpSkyfiles.php \
+$(DESTWWW)/fullforceFailedSkyfile.php \
+$(DESTWWW)/fullforceProcessedSkyfile.php \
+$(DESTWWW)/fullforceRun.php \
+$(DESTWWW)/fullforceSummary.php \
+$(DESTWWW)/fullforce_failure.php \
 $(DESTWWW)/failedFullForceSkyfiles.php \
 $(DESTWWW)/diffSummary.php \
@@ -172 +177 @@
 $(DESTWWW)/diffProcessedSkyfile.php \
 $(DESTWWW)/diffProcessedSkyfile_Images.php \
+$(DESTWWW)/diffFailedSkyfile.php \
 $(DESTWWW)/failedDiffSkyfile.php \
 $(DESTWWW)/stackRun.php \

trunk/ippMonitor/config.dat.in

@@ -1 +1 @@
 
+// host and user information for the main processing database (contains, e.g., gpc1, gpc2, etc)
 $DBHOST    = "";
 $DBUSER    = "";
 $DBPASS    = "";
+
+
+// location of nebulous apache interface machine
+$NEB_SERVER = "";
+
+// Czartool configuration variables
+// NOTE: These should match the configuration file used by the czartool perl modules: czartool/czarconfig.xml
+$CZARDBNAME = "";
+$CZARDBHOST = "";
+
+// location of plots produced by czarpoll.pl
+$CZARPLOTDIR = "";
 
 // Temporary files belonging to DELETION_USER and older than
@@ -9 +22 @@
 
 $DELETION_USER   = "";
-$DELETION_DELAY  = 15;
+$DELETION_DELAY  =5;
 
-$REPL_HOST_IPPADMIN      = "";
-$REPL_USER_IPPADMIN      = "";
-$REPL_PASSWORD_IPPADMIN  = "";
-$REPL_DBNAME_IPPADMIN    = "" ;
+$HOST_NEBULOUS  = "";
+$USER_NEBULOUS  = "";
+$PASSWORD_NEBULOUS  = "";
+$DBNAME_NEBULOUS  = "";
 
-$REPL_HOST_NEBULOUS      = "";
-$REPL_USER_NEBULOUS      = "";
+$REPL_HOST_NEBULOUS  = "";
+$REPL_USER_NEBULOUS  = "";
 $REPL_PASSWORD_NEBULOUS  = "";
-$REPL_DBNAME_NEBULOUS    = "";
+$REPL_DBNAME_NEBULOUS  = "";
 
-$REPL_HOST_PSTAMP      = "";
-$REPL_USER_PSTAMP      = "";
+$REPL_HOST_NEBULOUS_SECONDARY  = "";
+$REPL_USER_NEBULOUS_SECONDARY  = "";
+$REPL_PASSWORD_NEBULOUS_SECONDARY  = "";
+$REPL_DBNAME_NEBULOUS_SECONDARY  = "";
+
+$REPL_HOST_PSTAMP  = "";
+$REPL_USER_PSTAMP  = "";
 $REPL_PASSWORD_PSTAMP  = "";
-$REPL_DBNAME_PSTAMP    = "";
+$REPL_DBNAME_PSTAMP  = "";
 
-$REPL_HOST_GPC1      = "";
-$REPL_USER_GPC1      = "";
+$REPL_HOST_ISP  = "";
+$REPL_USER_ISP  = "";
+$REPL_PASSWORD_ISP  = "";
+$REPL_DBNAME_ISP  = "";
+
+$HOST_GPC1  = "";
+$USER_GPC1  = "";
+$PASSWORD_GPC1  = "";
+$DBNAME_GPC1  = "";
+
+$REPL_HOST_GPC1  = "";
+$REPL_USER_GPC1  = "";
 $REPL_PASSWORD_GPC1  = "";
-$REPL_DBNAME_GPC1    = "";
+$REPL_DBNAME_GPC1  = "";
 
-$REPL_HOST_ISP      = "";
-$REPL_USER_ISP      = "";
-$REPL_PASSWORD_ISP  = "";
-$REPL_DBNAME_ISP    = "";
+$REPL_HOST_GPC1_SECONDARY  = "";
+$REPL_USER_GPC1_SECONDARY  = "";
+$REPL_PASSWORD_GPC1_SECONDARY  = "";
+$REPL_DBNAME_GPC1_SECONDARY  = "";
 
-$REPL_HOST_NEBULOUS_SECONDARY      = "";
-$REPL_USER_NEBULOUS_SECONDARY      = "";
-$REPL_PASSWORD_NEBULOUS_SECONDARY  = "";
-$REPL_DBNAME_NEBULOUS_SECONDARY    = "";
+$REPL_HOST_DATASTORE_SECONDARY  = "";
+$REPL_USER_DATASTORE_SECONDARY  = "";
+$REPL_PASSWORD_DATASTORE_SECONDARY  = "";
+$REPL_DBNAME_DATASTORE_SECONDARY  = "";
+
+//Important machines that should be checked for accessibiltiy
+$HOST_GATEWAY  = "";
+$DBNAME_GATEWAY  = "";
+
+$HOST_GATEWAY_SECONDARY  = "";
+$DBNAME_GATEWAY_SECONDARY  = "";
+
+$HOST_APACHE1  = "";
+$DBNAME_APACHE1  = "";
+
+$HOST_APACHE2  = "";
+$DBNAME_APACHE2  = "";
+
+$HOST_APACHE3  = "";
+$DBNAME_APACHE3  = "";
+
+$HOST_APACHE4  = "";
+$DBNAME_APACHE4  = "";
+
+$HOST_APACHE5  = "";
+$DBNAME_APACHE5  = "";
+
+$HOST_APACHE6  = "";
+$DBNAME_APACHE6  = "";
+
+$HOST_PROXY  = "";
+$DBNAME_PROXY  = "";
+
+$HOST_SVN  = "";
+$DBNAME_SVN  = "";
+
+$HOST_CONFLUENCE  = "";
+$DBNAME_CONFLUENCE  = "";
+
+

trunk/ippMonitor/czartool/czarconfig.xml.in

@@ -10 +10 @@
     <size>8</size>
     <path>PATH/TO/czartool_plots</path>
+    <exec>PATH/TO/gnuplot</exec>
   </gnuplot>
 

trunk/ippMonitor/czartool/czarpoll.pl

@@ -102 +102 @@
     foreach $server (@servers) {
 
-        my @dates = @{$pantasks->getDates($server)};
+        my @dates = @{$pantasks->getDates($server,'gpc1')};
         if (@dates) {
-
-            $czarDb->updateServerDates($server, \@dates);
+            $czarDb->updateServerDates($server, \@dates,'gpc1');
         }
         else {
-
-            print "WARNING: No dates to update for '$server'\n";
+            print "WARNING: No dates to update for '$server' in gpc1\n";
+        }
+
+        my @dates2 = @{$pantasks->getDates($server,'gpc2')};
+        if (@dates2) {
+            $czarDb->updateServerDates($server, \@dates2,'gpc2');
+        }
+        else {
+            print "WARNING: No dates to update for '$server' in gpc2\n";
         }
     }
@@ -126 +132 @@
 
     print "* Updating labels\n";
-    # my @servers = ("stdscience", "distribution", "publishing", "update");
     my @servers = ("stdscience", "distribution");
 
     my @allLabels = ();
-
+    my @allLabels2 = ();
+    my @fields = ();
+
+    #first grab the labels fom the relevant pantasks
     my $server = undef;
     foreach $server (@servers) {
-        my @labels = @{$pantasks->getLabels($server)};
+        my @labels = @{$pantasks->getLabels($server,'gpc1')};
         push (@allLabels, @labels);
-    }
+
+        my @labels2 = @{$pantasks->getLabels($server,'gpc2')};
+        push (@allLabels2, @labels2);
+    }
+
+    ## adding extra labels from external file
+    my $filename = 'extra_labels.dat';
+    if (open(my $fh, '<:encoding(UTF-8)', $filename)) {
+      my $header = <$fh>;     
+      $header = <$fh>;     
+      while (my $row = <$fh>) {
+          chomp $row;
+          @fields = split(' ', $row);
+          if ($fields[1] eq 'gpc1') {
+              push (@allLabels, $fields[0]);
+          }
+          if ($fields[1] eq 'gpc2') {
+              push (@allLabels2, $fields[0]);
+          }
+      }
+    } else {
+      warn "No extra labels found in '$filename' $!";
+    }
+
+    # Now, grab the priorities for each label
+    my @priorities = ();
+    my $row = undef;
+    my $label = undef;
+    my $priority = undef;
+    foreach $label ( @allLabels ) {
+        $priority = $gpc1Db->getPriority($label);
+        push(@priorities, $priority);
+    }
+
+    #also grab gpc2 priorities from the gpc2 Label table
+    my @priorities2 = ();
+    my $priority2 = undef;
+    foreach $label ( @allLabels2 ) {
+        $priority2 = $gpc2Db->getPriority($label);
+        push(@priorities2, $priority2);
+    }
+
+    #sort labels by priority
+    my @idx = sort {$priorities[$a] <=> $priorities[$b]} 0 ..$#priorities;
+    @allLabels = @allLabels[@idx];
+    @priorities = @priorities[@idx]; 
+
+    my @idx2 = sort {$priorities2[$a] <=> $priorities2[$b]} 0 ..$#priorities2;
+    @allLabels2 = @allLabels2[@idx2];
+    @priorities2 = @priorities2[@idx2]; 
 
     # store them all in stdscience:
     $server = "stdscience";
     if (@allLabels) {
-        $czarDb->updateCurrentLabels($server, \@allLabels);
+        $czarDb->updateCurrentLabels($server, \@allLabels,'gpc1', \@priorities);
     } else {
-        print "WARNING: No labels to update for '$server'\n";
-    }
+        print "WARNING: No labels to update for gpc1 for '$server'\n";
+    }
+
+    if (@allLabels2) {
+        $czarDb->updateCurrentLabels($server, \@allLabels2,'gpc2', \@priorities2);
+    } else {
+        print "WARNING: No labels to update for gpc2 for '$server'\n";
+    }
+    return(\@allLabels,\@allLabels2);
 }
 
@@ -160 +224 @@
     my $running = undef;
     foreach $server (@{$servers}) {
-
-        $pantasks->getServerStatus($server, \$alive, \$running);
-        $czarDb->updateServerStatus($server, $alive, $running);
-    }
+        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
+        $czarDb->updateServerStatus($server,'gpc1', $alive, $running);
+        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
+        $czarDb->updateServerStatus($server,'gpc2', $alive, $running);
+   }
 }
 
@@ -182 +247 @@
     my $str = undef;
     my $labels = undef;
+    my $labels2 = undef;
     my $updateLabels = undef;
     my $row = undef;
@@ -187 +253 @@
     my $end = undef;
     my $priority = undef;
+    my $priority2 = undef;
     my $newState = undef;
     my $nsStatus = undef;
@@ -227 +294 @@
 
                 # now cleanup tables from yesterday and optimize
-                print "* Performing database cleanup\n";
-                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
-                print "* Optimizing the database\n";
-                $czarDb->optimize();
+                #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB. 
+                #Therefore, it makes no sense to run the cleanup+optimising twice
+                #print "* Performing database cleanup\n";
+                #$czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
+                #print "* Optimizing the database\n";
+                #$czarDb->optimize();
                 $lastDayDailyTasks = $yesterday;
             }
@@ -237 +306 @@
         # check nightly science status
         print "* Checking nightly science status\n";
-        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
-        $czarDb->updateNightlyScience($nsStatus);
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc1')) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus,'gpc1');
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc2')) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus,'gpc2');
 
         # check nebulous
@@ -252 +323 @@
 
         # check labels
-        updateLabels();
+        my ($labels, $labels2) = updateLabels();
+        my @labels = @$labels;
+        my @labels2 = @$labels2;
 
         # servers to check
@@ -265 +338 @@
 
             # deal with stdscience labels
-            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
             my $size = @{$labels};
             if($size > 0) {
-
-                # get priority
-                foreach $row ( @{$labels} ) {
-                    my ($label) = @{$row};
-                    # for now use priorities from gpc1 database
-                    $priority = $gpc1Db->getPriority($label);
-                    $czarDb->setLabelPriority($label, $priority);
-                }
-
-                updateAllStages($thisServer, $newState, $labels, $begin, $end);
+                updateAllStages($thisServer, $newState, $labels,$labels2, $begin, $end);
                 if ($thisServer eq "stdscience") {
                     createPlots($thisServer, $plotter1, $labels, $begin, $end);
-                    createPlots($thisServer, $plotter2, $labels, $begin, $end);
+                    createPlots($thisServer, $plotter2, $labels2, $begin, $end);
                 }
             }
@@ -376 +439 @@
     #  #  $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
 
-    #    #TdB 20190411: Make the all stages plots for the nightlyscience labels only
     #    if ($label =~ m/nightlyscience/) {
     #      $plotter->createTimeSeries($label, undef, $begin, $end, 1, 0, 1);
@@ -382 +444 @@
     #}
 
-    $allServerLabels = "all_".$server."_labels";
-
-    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 0, 1);
-
-    #TdB 20190411: We are turning off these plots for the time being. They take up a lot of time and are rarely looked at. If need be, can be generated on the fly using czartool.
+    $allServerLabels = $server."_labels";
+
+    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1);
+
     foreach $stage (@stages) {
-        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 1); # TODO must be a neater way...
+        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 0); # TODO must be a neater way...
     }
 }
@@ -398 +459 @@
 ###########################################################################
 sub updateAllStages {
-    my ($labelServer, $newState, $rows, $begin, $end) = @_;
+    my ($labelServer, $newState, $rows, $rows2, $begin, $end) = @_;
 
     print "* Updating stage data\n";
@@ -419 +480 @@
     foreach $stage (@stages) {
         $server = $pantasks->getServerForThisStage($stage);
-        $pantasks->getRevertStatus($stage, \$reverting);
-        $czarDb->updateRevertStatus($stage, $reverting);
+        $pantasks->getRevertStatus($stage,'gpc1', \$reverting);
+        $czarDb->updateRevertStatus($stage,'gpc1', $reverting);
+
+        $pantasks->getRevertStatus($stage,'gpc2', \$reverting);
+        $czarDb->updateRevertStatus($stage,'gpc2', $reverting);
 
         print "* Checking labels for $stage stage\n";
@@ -427 +491 @@
         if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) { 
             ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage);
-            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $new, $full, $faults);
+            $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $new, $full, $faults);
 
             ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage);
-            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $new, $full, $faults);
+            $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $new, $full, $faults);
 
             next;
         }
 
-
+        #count the number of exposures under each label, and their faults
         $totalNew1=$totalFaults1=$totalFull1=0;
-        $totalNew2=$totalFaults2=$totalFull2=0;
         foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
+            my ($label) = $row;
 
             chomp($label);
@@ -447 +510 @@
             $full = $gpc1Db->countExposures($label, $stage, "full");
             $faults = $gpc1Db->countFaults($label, $stage, $newState);
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
             $totalNew1 += $new;
             $totalFull1 += $full;
@@ -453 +515 @@
 
             $czarDb->insertNewTimeData($stage, $label, 'gpc1', $new, $full, $faults);
-
+        }
+
+        $totalNew2=$totalFaults2=$totalFull2=0;
+        foreach $row ( @{$rows2} ) {
+            my ($label) = $row;
+
+            chomp($label);
+
+            # Move this to a function and pass in the database
             $new = $gpc2Db->countExposures($label, $stage, $newState);
             $full = $gpc2Db->countExposures($label, $stage, "full");
             $faults = $gpc2Db->countFaults($label, $stage, $newState);
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
             $totalNew2 += $new;
             $totalFull2 += $full;
@@ -465 +534 @@
         }
 
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1);
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2);
-    }
-}
-
+        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1);
+        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2);
+    }
+}
+

trunk/ippMonitor/czartool/czartool.pl

@@ -15 +15 @@
 my $save_temps = 1;
 my $interval = undef;
+my $telescope = "gpc1";
 
 GetOptions (
         "dbname|d=s" => \$czarDbName,
         "interval|i=s" => \$interval,
+        "telescope|t=s" => \$telescope,
         );
-
 
 my @states = ("full", "new", "drop", "wait");
@@ -26 +27 @@
 my $czarDb = $config->getCzarDbInstance();
 my $gpc1Db = $config->getGpc1Instance();
+my $gpc2Db = $config->getGpc2Instance();
 my $pantasks = new czartool::Pantasks();
 
@@ -55 +57 @@
 sub promptPoll {
 
-    @stdscienceLabels = @{$pantasks->getLabels("stdscience")};
-    @distributionLabels = @{$pantasks->getLabels("distribution")};
-    # @publishingLabels = @{$pantasks->getLabels("publishing")};
+    @stdscienceLabels = @{$pantasks->getLabels("stdscience",$telescope)};
+    @distributionLabels = @{$pantasks->getLabels("distribution",$telescope)};
+    # @publishingLabels = @{$pantasks->getLabels("publishing", $telescope)};
     @publishingLabels = @stdscienceLabels;
-    checkAllLabels("new");
+    checkAllLabels("new", $telescope);
     printInstructions();
 
@@ -68 +70 @@
 
         if ($key eq "s") {checkServers();}
-        elsif ($key eq "l") {checkAllLabels("new");}
-        elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience");}
-        elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1]);}
+        elsif ($key eq "l") {checkAllLabels("new", $telescope);}
+        elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience",$telescope);}
+        elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1], $telescope);}
         printInstructions();
 
@@ -139 +141 @@
 ###########################################################################
 sub labelDetails {
+    my ($telescope) = @_;
 
     print "\nPlease enter the label name, or number from table above: ";
@@ -144 +147 @@
     chomp($input);
     if ($input =~ m/^([0-9]|[1-9][0-9]|[1-9][0-9][0-9])$/) {
-        return checkOneLabel($stdscienceLabels[$input]);
-    }
-
-    checkOneLabel($input);
+        return checkOneLabel($stdscienceLabels[$input], $telescope);
+    }
+
+    checkOneLabel($input, $telescope);
 }
 
@@ -156 +159 @@
 ###########################################################################
 sub checkOneLabel {
-    my ($label) = @_;
+    my ($label, $telescope) = @_;
 
     chomp($label);
@@ -188 +191 @@
         foreach $stage (@stages) {
 
-            printf("|%14s", getStateAndFaultsString($label, $state, $stage));
+            printf("|%14s", getStateAndFaultsString($label, $state, $stage, $telescope));
         }
 
@@ -209 +212 @@
 ###########################################################################
 sub checkAllLabels {
-    my ($state) = @_;
+    my ($state, $telescope) = @_;
 #print time, $/;
     my $stage;
@@ -259 +262 @@
         foreach $stage (@stages) {
 
-            printf("|%14s", getStateAndFaultsString($stdsLabel, $state, $stage));
+            printf("|%14s", getStateAndFaultsString($stdsLabel, $state, $stage, $telescope));
         }
 
@@ -280 +283 @@
 ###########################################################################
 sub getStateAndFaultsString {
-    my ($label, $state, $stage) = @_;
-
-    my $new = $gpc1Db->countExposures($label, $stage, $state);
-
-    #if ($state ne "new") {return $new;}
-
-    my $faults = $gpc1Db->countFaults($label, $stage, $state);
+    my ($label, $state, $stage, $telescope) = @_;
+
+    my $new = undef;
+    my $faults = undef;
+
+    if ($telescope eq 'gpc1') {
+      $new = $gpc1Db->countExposures($label, $stage, $state);
+      #if ($state ne "new") {return $new;}
+
+      $faults = $gpc1Db->countFaults($label, $stage, $state);
+    }
+    elsif ($telescope eq 'gpc2') {
+      $new = $gpc2Db->countExposures($label, $stage, $state);
+      #if ($state ne "new") {return $new;}
+
+      $faults = $gpc2Db->countFaults($label, $stage, $state);
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     if ($faults < 1) {return $new;}

trunk/ippMonitor/czartool/czartool/CzarDb.pm

@@ -137 +137 @@
 ###########################################################################
 sub updateServerStatus {
-    my ($self, $server, $alive, $running) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    INSERT INTO servers
-        (server, alive, running)
+    my ($self, $server,$telescope, $alive, $running) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO science_servers
+        (server,telescope, alive, running)
         VALUES
-        ('$server', $alive, $running);
+        ('$server','$telescope', $alive, $running);
 SQL
 
@@ -173 +173 @@
 ###########################################################################
 sub updateRevertStatus {
-    my ($self, $stage, $reverting) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-     UPDATE reverts
+    my ($self, $stage,$telescope, $reverting) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+     UPDATE science_reverts
          SET reverting = $reverting
-         WHERE stage LIKE '$stage';
-SQL
-
-    $query->execute;
-}
-
-###########################################################################
-#
-# Sets priority for this label
-#
-###########################################################################
-sub setLabelPriority {
-    my ($self, $label, $priority) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-     UPDATE current_labels
-         SET priority = $priority
-         WHERE label LIKE '$label'
-         AND server LIKE 'stdscience';
+         WHERE stage LIKE '$stage' AND telescope LIKE '$telescope';
 SQL
 
@@ -211 +193 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    DELETE FROM server_dates;
+    DELETE FROM science_server_dates;
+
 SQL
 
@@ -223 +206 @@
 ###########################################################################
 sub updateServerDates {
-    my ($self, $server, $dates) = @_;
+    my ($self, $server, $dates,$telescope) = @_;
 
     my $size = scalar @{$dates};
@@ -233 +216 @@
 
         my $query = $self->{_db}->prepare(<<SQL);
-        INSERT INTO server_dates
-            (server, date)
+        INSERT INTO science_server_dates
+            (server, telescope, date)
             VALUES
-            ('$server', '$date');
+            ('$server','$telescope', '$date');
 SQL
 
@@ -249 +232 @@
 ###########################################################################
 sub updateNightlyScience {
-    my ($self, $status) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    DELETE FROM nightlyscience;
+    my ($self, $status,$telescope) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM science_nightlyscience WHERE telescope LIKE '$telescope';
 SQL
 
@@ -258 +241 @@
 
     $query = $self->{_db}->prepare(<<SQL);
-    INSERT INTO nightlyscience
-        (status)
+    INSERT INTO science_nightlyscience
+        (status, telescope)
         VALUES
-        ('$status');
+        ('$status', '$telescope');
 SQL
 
@@ -269 +252 @@
 ###########################################################################
 #
-# Updates current_labels table
+# Updates science_labels table
 #
 ###########################################################################
 sub updateCurrentLabels {
-    my ($self, $server, $labels) = @_;
+    my ($self, $server, $labels, $telescope,$priorities) = @_;
 
     my $size = scalar @{$labels};
     if ($size < 1) { return; }
-    if ($server eq 'stdscience') {
+#    if (($server eq 'stdscience') && ($telescope eq 'gpc1')) {
         ## EAM 20180510 : manually adding label used for Pole reprocessing
         ## I would like to add an external file to manage these extra 
         ## labels, but this suffices for now
-        push @{$labels}, 'PV3.Pole.Reprocess.20180510';
-        push @{$labels}, 'ps_ud_QUB';
-    }
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    DELETE FROM current_labels WHERE server LIKE '$server';
+#        push @{$labels}, 'PV3.Pole.Reprocess.20180510';
+#       push @{$labels}, 'ps_ud_QUB';
+#        push @{priorities}, 50000;
+#       push @{priorities}, 415;
+#    }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM science_labels WHERE server LIKE '$server' AND telescope LIKE '$telescope';
 SQL
 
@@ -292 +277 @@
 
     my $label = undef;
-
-    foreach $label (@{$labels}) {
+    my $priority = undef;
+
+    foreach my $i (0 .. $#$labels) {
+        $label = @{$labels}[$i];
+        $priority = @{$priorities}[$i];
 
         my $query = $self->{_db}->prepare(<<SQL);
-        INSERT INTO current_labels
-            (server, label)
+        INSERT INTO science_labels
+            (server, label,telescope,priority)
             VALUES
-            ('$server', '$label');
+            ('$server', '$label','$telescope',$priority);
 SQL
 

trunk/ippMonitor/czartool/czartool/Pantasks.pm

@@ -24 +24 @@
         );
 
-my $ipphome = "/data/ippc64.1/ippitc";
-
 ###########################################################################
 #
@@ -86 +84 @@
 ###########################################################################
 sub getNightlyScienceStatus {
-    my ($self, $status) = @_;
+    my ($self, $status,$telescope) = @_;
 
     my @labels;
+    my $ipphome;
+
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     my ($day, $month, $year) = (localtime)[3,4,5];
@@ -102 +111 @@
         if ($line =~ m/$today\s+(.+)/) {
             ${$status} = $1;
+            if (${$status} =~ m/ $telescope/) {
+            ${$status} = $`;
+            }
+
             return 1;
         }
@@ -111 +124 @@
 ###########################################################################
 #
-# Gets labels for provided server 
+# Gets labels for provided server for gpc1
 #
 ###########################################################################
 sub getLabels {
-    my ($self, $server) = @_;
+    my ($self, $server,$telescope) = @_;
 
     my @labels;
-
-    # print "stdscience : $ipphome/$server/ptolemy.rc\n";
+    my $ipphome;
+
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
+
     my @cmdOut = `echo "show.labels;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
     my $line;
@@ -149 +172 @@
 ###########################################################################
 sub getServerStatus {
-    my ($self, $server, $alive, $running) = @_;
+    my ($self, $server,$telescope, $alive, $running) = @_;
+
+    my $ipphome;
+
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     my @cmdOut = `echo "status ; quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
@@ -170 +205 @@
 ###########################################################################
 sub getServerCurrentStatus {
-    my ($self, $server) = @_;
+    my ($self, $server,$telescope) = @_;
+
+    my $ipphome;
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
@@ -189 +235 @@
 ###########################################################################
 sub getRevertStatus {
-    my ($self, $stage, $reverting) = @_;
+    my ($self, $stage,$telescope, $reverting) = @_;
+
+    my $ipphome;
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     my $server = $self->getServerForThisStage($stage);
@@ -214 +271 @@
 ###########################################################################
 sub getDates {
-    my ($self, $server) = @_;
+    my ($self, $server,$telescope) = @_;
 
     my @dates;
 
     my $prefix;
+    my $ipphome;
+
+    if ($telescope eq 'gpc1') {
+       $ipphome = "/data/ippc64.1/ippitc";
+    }
+    elsif ($telescope eq 'gpc2') {
+       $ipphome = "/data/ippc18.0/home/ippps2";
+    }
+    else {
+        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
+    }
 
     if ($server eq "stdscience") {$prefix = "ns";}

trunk/ippMonitor/raw/czartool_getplot.php

@@ -22 +22 @@
 $path = $CZARPLOTDIR;
 
-if ($label == "all") $label = "all_".$mode."_labels";
+if ($label == "all") $label = $mode."_labels";
 
 if ($type=="t" || $type=="r" || $type=="rt" || $type=="rh")

trunk/ippMonitor/raw/czartool_labels.php

@@ -90 +90 @@
         # "publishing",
         "registration",
-        "replication",
         "stack",
         # "deepstack",
@@ -139 +138 @@
 
 // $labels is generated with sequential integer key values
-$labels = getLabels($czardb, $selectedMode);
-$distLabelsIndex = getLabels($czardb, "distribution");
+$labels = getLabels($czardb, $proj, $selectedMode);
+$distLabelsIndex = getLabels($czardb, $proj, "distribution");
 for ($i = 0; $i < count($distLabelsIndex); $i++) {
   $distLabelsExist[$distLabelsIndex[$i]] = true;
@@ -195 +194 @@
 echo "<td style=width:100px;text-align:top;\">";
  # time series plot
-echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=$plotType\"><br>";
+echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
+# log time series plot
+echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=log\"><br>";
  # rate time series plot
 echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=rt&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
- # rate stacked histo plot
-# echo "<img src=\"czartool_getplot.php?mode=$selectedMode&type=rh&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
  # histogram plot
 # echo "<img src=\"czartool_getplot.php?mode=$selectedMode&type=h&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
@@ -218 +217 @@
 
 // status table at top
-$nsStatus = getNightlyScienceStatus($czardb);
+$nsStatus = getNightlyScienceStatus($czardb, $proj);
 $plotTypeLink = ($plotType == "linear") ? "log" : "linear";
 $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
@@ -232 +231 @@
 echo "<tr><td>Documentation</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\"><font color=\"blue\">here</font></a></td></tr>";
 echo "<tr><td>Cluster load</td><td><a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&hc=6&c=IPP%2520Production\"><font color=\"blue\">here</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=cpu_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">cpu_report</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=mem_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">mem_report</font></a></td></tr>";
-echo "<tr><td>Who's using the cluster?</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/ippMonitor/clusterMonitor/index.php\"><font color=\"blue\">here</font></a></td></tr>";
-echo "<tr><td>Files monitor</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/diskMonitor\"><font color=\"blue\">here</font></a></td></tr>";
-echo "<tr><td>IPP metrics</td><td><a href=\"http://ipp0022.ifa.hawaii.edu/ps1sc/ippMetrics\"><font color=\"blue\">here</font></a></td></tr>";
-echo "<tr><td>Czar log pages</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_CzarLogs\"><font color=\"blue\">All</font></a>&nbsp;&nbsp;&nbsp;";
-echo "<a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_Czarlog_".(date("Ymd")-date("w")+"1")."\">This week</a>";
-echo "</td></tr>";
+echo "<tr><td>IPP metrics</td><td><a href=\"https://ps1wiki.ifa.hawaii.edu/data/metrics/index.html\"><font color=\"blue\">here</font></a></td></tr>";
+echo "<tr><td>Czar log pages</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/pages/viewrecentblogposts.action?key=IPPCZAR\"><font color=\"blue\">here</font></a></td></tr>";
 echo "<tr><td>Exposure summary</td><td><a href=\"czartool_exposures.php?pass=$pass&proj=$proj\"><font color=\"blue\">here</font></a></td></tr>";
+echo "<tr><td>Confluence</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/display/IPPCZAR/IPP-Czaring+Home\"><font color=\"blue\">here</font></a></td></tr>";
+echo "<tr><td>JIRA</td><td><a href=\"https://jira.ipp.ifa.hawaii.edu/secure/Dashboard.jspa\"><font color=\"blue\">here</font></a></td></tr>";
 
 $plotTypeLink = ($plotType == "linear") ? "log" : "linear";
@@ -266 +263 @@
       createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "<br>";
       createSummitDataTable($projectdb); echo "<br>"; // this is slow because of the join between summitExp and rawExp
-      createDatesTable($czardb); echo "<br>";
-
-      # This is awful
+      createChunkDataTable($projectdb); echo "<br>"; 
+      createDatesTable($czardb, $proj); echo "<br>";
+
       echo $table;
       echo "<tr>";
-      createTableTitle("Database replication status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
+      createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
 
       echo "<tr>";
       createTableColumnHeader("Database");
       createTableColumnHeader("Status (sec behind master)");
+      showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1);
       showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
       showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY);
 
+      echo "<tr>";
+
+      showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS);
       showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
       showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY);
+      showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY);
 
 #     some mysql replication slaves to add when they are returned to service
 #     showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP);
-#     showReplicationsStatus($REPL_HOST_IPPADMIN, $REPL_USER_IPPADMIN, $REPL_PASSWORD_IPPADMIN, $REPL_DBNAME_IPPADMIN);  --- this one is in GPC1
 #     showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP);
       echo "</table>";
+
+      echo $table;
+      echo "<tr>";
+      createTableTitle("Critical machine status", 2);
+
+      echo "<tr>";
+      createTableColumnHeader("Hostname");
+      createTableColumnHeader("Status");
+      showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY);
+      showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY);
+
+#      showPingStatus($HOST_APACHE1, $DBNAME_APACHE1);
+      showPingStatus($HOST_APACHE2, $DBNAME_APACHE2);
+      showPingStatus($HOST_APACHE3, $DBNAME_APACHE3);
+      showPingStatus($HOST_APACHE4, $DBNAME_APACHE4);
+      showPingStatus($HOST_APACHE5, $DBNAME_APACHE5);
+      showPingStatus($HOST_APACHE6, $DBNAME_APACHE6);
+      showPingStatus($HOST_PROXY, $DBNAME_PROXY);
+      showPingStatus($HOST_SVN, $DBNAME_SVN);
+      showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE);
+
+      echo "</table>";
+
 echo "</table>";
 menu_end();
@@ -401 +425 @@
 ###########################################################################
 #
+# Creates a table of chunk progress
+#
+###########################################################################
+function createChunkDataTable($projectdb) {
+
+    #First find the reference exposure ID to speed up the subsequent queries
+    $date = gmdate("Y-m-d");
+    $sql = "SELECT MAX(exp_id) FROM rawExp WHERE dateobs >= date_sub('$date', interval 20 day) AND dateobs <= date_sub('$date', interval 10 day)";
+    $qry = $projectdb->query($sql);
+    if (dberror($qry)) {
+        print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n";
+        $refExpID = 0; 
+    }
+    $qry->fetchInto($refExpID);
+
+    #Find the number of chunks
+    $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM";
+    $query .= "   (";
+    $query .= "     SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit";
+    $query .= "     FROM rawExp";
+    $query .= "     WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0]";
+    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'";
+    $query .= "     AND comment LIKE '%visit 1%'";
+    $query .= "     GROUP BY chunk";
+    $query .= "   ) AS groupchunk";
+    $query .= "   LEFT JOIN";
+    $query .= "   (";
+    $query .= "     SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM ";
+    $query .= "       (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits";
+    $query .= "         FROM rawExp";
+    $query .= "         WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
+    $query .= "         AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'";
+    $query .= "         GROUP BY object, filter, chunk";
+    $query .= "       ) AS groupobj";
+    $query .= "     WHERE groupobj.Nvisits = 4 ";
+    $query .= "     GROUP BY groupobj.chunk";
+    $query .= "   ) AS group4";
+    $query .= "   on groupchunk.chunk=group4.chunk ORDER BY dateobs";
+
+    #$sql = "SELECT DISTINCT substr(comment, 1, position(' ' in comment)) FROM rawExp WHERE dateobs LIKE '$date%' AND exp_id > $refExpID AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit 1%'";
+
+    $qry = $projectdb->query($query);
+    if (dberror($qry)) {
+        echo "<b>error reading rawExp table</b><br>\n";
+        echo "<br><small><b> table query : $query </b></small><br>\n";
+        menu_end();
+    }
+
+    // set up the table
+    global $table;
+    echo $table;
+    echo "<tr>";
+    createTableTitle("Status of last night's chunks", 5);
+    echo "<tr>";
+    createTableColumnHeader("Chunk");
+    createTableColumnHeader("Nobs_1stvisit");
+    createTableColumnHeader("Nobs_4visits");
+    createTableColumnHeader("Ndiffs_published");
+    createTableColumnHeader("Completion");
+
+    $msg = "No science images taken since $date";
+
+    // list the results
+    while ($qry->fetchInto($row)) {
+        $Nfirstvisits = 0;
+        $Nfourvisits = 0;
+
+        $chunkname = $row[0];
+        if($row[1] > 0) $Nfirstvisits = $row[1];
+        if($row[2] > 0) $Nfourvisits = $row[2];
+
+        $query2 = "     SELECT warp_id FROM ";
+        $query2 .= "           warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
+        $query2 .= "           WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
+        $query2 .= "           AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'";
+        $qry2 = $projectdb->query($query2);
+
+        $Ndiffs = 0;
+        while ($qry2->fetchInto($row2)) {
+            $query3 = "     SELECT diff_id FROM ";
+            $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)";
+            $query3 .= "           WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id";
+            $qry3 = $projectdb->query($query3);
+            while ($qry3->fetchInto($row3)) {
+                if ($row3[0] !=0) $Ndiffs ++;
+            }
+
+            $query3 = "     SELECT diff_id FROM ";
+            $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)";
+            $query3 .= "           WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id";
+            $qry3 = $projectdb->query($query3);
+            while ($qry3->fetchInto($row3)) {
+                if ($row3[0] !=0) $Ndiffs ++;
+            }
+        }
+
+        if ($Ndiffs != ($Nfirstvisits*4)) $complete = "chunk not done";  
+        if ($Ndiffs == ($Nfirstvisits*4)) $complete = "chunk done";        
+
+        if ($Nfirstvisits != $Nfourvisits) {
+            $Ndiffs .= "/?  ";
+        }
+        if ($Nfirstvisits == $Nfourvisits) {
+            $Ndiffs .= "/";
+            $Ndiffs .= $Nfirstvisits*4;
+        }
+
+        echo "<tr>";
+
+        echo "<td>$chunkname</td>";
+        echo "<td>$Nfirstvisits</td>";
+        echo "<td>$Nfourvisits</td>";
+        echo "<td>$Ndiffs</td>";
+        echo "<td>$complete</td>";
+
+        echo "</tr>";
+    }
+    echo "</table>\n";
+
+}
+
+###########################################################################
+#
 # Gets labels for this server
 #
 ###########################################################################
-function getLabels($db, $server) {
+function getLabels($db, $proj, $server) {
 
     # order by descending priority as set in project database
-    $sql = "SELECT label FROM current_labels WHERE server LIKE '$server' ORDER BY priority DESC, label";
+    $sql = "SELECT label FROM science_labels WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY priority DESC, label";
     if ($debug) {echo "$sql<br>";}
 
@@ -547 +694 @@
     foreach ($stages as &$stage) {
 
-        $reverting = getRevertStatus($db, $stage);
+        $reverting = getRevertStatus($db, $proj, $stage);
         if(!$reverting) {$label =  "";}
         if($reverting) {$label = "reverting";}
@@ -608 +755 @@
 #
 ###########################################################################
-function setRevertStatus($db, $stage, $reverting) {
-
-    $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'";
+function setRevertStatus($db, $proj, $stage, $reverting) {
+
+    $sql = "UPDATE science_reverts SET reverting = $reverting WHERE stage LIKE '$stage' AND telescope LIKE '$proj'";
     if ($debug) {echo "$sql<br>";}
 
@@ -622 +769 @@
 #
 ###########################################################################
-function getNightlyScienceStatus($db) {
-
-    $sql = "SELECT status FROM nightlyscience";
+function getNightlyScienceStatus($db, $proj) {
+
+    $sql = "SELECT status FROM science_nightlyscience WHERE telescope LIKE '$proj'";
     if ($debug) {echo "$sql<br>";}
 
@@ -639 +786 @@
 #
 ###########################################################################
-function turnRevertsOnOff($db, $stage, $mode) {
+function turnRevertsOnOff($db, $proj, $stage, $mode) {
 
     exec("czartool_revert.pl -t $stage -o $mode", $response, $status);
@@ -646 +793 @@
     else if ($response[0] == "on") $currentRevertMode = 1;
 
-    setRevertStatus($db, $stage, $currentRevertMode);
+    setRevertStatus($db, $proj, $stage, $currentRevertMode);
 }
 
@@ -654 +801 @@
 #
 ###########################################################################
-function getRevertStatus($db, $stage) {
+function getRevertStatus($db, $proj, $stage) {
 
     $anyFaults = false;
 
-    $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'";
+    $sql = "SELECT reverting FROM science_reverts WHERE stage LIKE '$stage' AND telescope LIKE '$proj'";
     if ($debug) {echo "$sql<br>";}
 
@@ -706 +853 @@
 #
 ###########################################################################
-function createDatesTable($db) {
+function createDatesTable($db, $proj) {
 
     global $table;
@@ -717 +864 @@
     echo "</tr>\n";
 
-    createServerDateRow($db, "stdscience");
-    createServerDateRow($db, "registration");
+    createServerDateRow($db, $proj, "stdscience");
+    createServerDateRow($db, $proj, "registration");
 
     echo "</table>\n";
@@ -729 +876 @@
 #
 ###########################################################################
-function createServerDateRow($db, $server) {
-
-    $sql = "SELECT date FROM server_dates WHERE server LIKE '$server' ORDER BY date";
+function createServerDateRow($db, $proj, $server) {
+
+    $sql = "SELECT date FROM science_server_dates WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY date";
     if ($debug) {echo "$sql<br>";}
 
@@ -982 +1129 @@
 }
 
+###########################################################################
+#
+# Shows the status of the various replication mysql servers
+#
+###########################################################################
+function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
+    #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
+    $db = DB::connect("mysql://$User:$Password@$Host");
+    if (PEAR::isError($db)) {
+        echo "<tr><td>$DatabaseName</td><td bgcolor=\"red\">MySQL DB connection error</td></tr>";
+        //die("MySQL DB connection error: Check the configuration for $DatabaseName");
+        //die("");
+        return -1;
+
+    }
+    $Status = ($errorStatusInMySql==0?"OK":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
+    echo "<tr><td>$DatabaseName</td><td>$Status</td></tr>";
+
+    $db->disconnect();
+}
+
+###########################################################################
+#
+# Shows the status of a machine using ping
+#
+###########################################################################
+function showPingStatus($Host, $DatabaseName) {
+    exec("ping -c 1 $Host", $res, $rval);
+    if ($rval == 0){
+      $PingStatus= "UP";
+    }else{
+      $PingStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">DOWN</font>";
+    }
+    echo "<tr><td>$DatabaseName</td><td>$PingStatus</td></tr>";
+}
+
 ?>
 

trunk/ippMonitor/raw/ipp.czar.dat

@@ -3 +3 @@
 menutop   | menutop      | plain   |     |   
 
-menulink   | menuselect      | link    | czartool - old  | czartool_labels_old.php
 menulink   | menuselect      | link    | czartool        | czartool_labels.php
 menulink   | menuselect      | link    | mask stats                   | maskStats.php
@@ -10 +9 @@
 menulink  | menuselect   | link    | Exposures Status             | exposureStatus.php
 menulink  | menuselect   | link    | MOPS Exposures Status        | mopsStatus.php
-menulink  | menuselect   | link    | GPC1 MySql ProcessList            | gpc1MysqlProcessList.php
+menulink  | menuselect   | link    | MySql ProcessList            | gpc1MysqlProcessList.php
 

trunk/ippMonitor/raw/ipp.menu.dat

@@ -23 +23 @@
 menutop   | menutop      | link    | Stack Steps                  | ipp.stack.php
 menutop   | menutop      | link    | Static Sky Steps             | ipp.sky.php
+menutop   | menutop      | link    | Full Force Steps             | ipp.ffs.php
 menutop   | menutop      | link    | Distribution                 | ipp.dist.php
 menutop   | menutop      | link    | Large Area Processing        | ipp.lap.php

trunk/ippMonitor/raw/ipp.sky.dat

@@ -16 +16 @@
 menulink  | menuselect   | link    | Skycal Failed                | skycal_failure.php
 
-menutop   | menutop      | plain   |                         | 
-menulink  | menuselect   | link    | Static Sky Summary           | staticskySummary.php
-menulink  | menuselect   | link    | Static Sky Run               | staticskyRun.php
-menulink  | menuselect   | link    | Static Sky Input Skyfile     | staticskyInputSkyfile.php
-menulink  | menuselect   | link    | Static Sky Processed Skyfile | staticskyProcessedSkyfile.php
-menulink  | menuselect   | link    | Static Sky Failed Skyfile    | staticskyFailedSkyfile.php
-menulink  | menuselect   | link    | Static Sky Failed            | failedStaticsky.php