Changeset 40840 for trunk/ippMonitor/czartool/czarpoll.pl

Timestamp:

Jul 23, 2019, 2:24:01 PM (7 years ago)

Author:

tdeboer

Message:

updates to ippMonitor and czartool to handle gpc2 labels and improve czartool

File:

• trunk/ippMonitor/czartool/czarpoll.pl (modified) (17 diffs)

trunk/ippMonitor/czartool/czarpoll.pl

@@ -102 +102 @@
     foreach $server (@servers) {
 
-        my @dates = @{$pantasks->getDates($server)};
+        my @dates = @{$pantasks->getDates($server,'gpc1')};
         if (@dates) {
-
-            $czarDb->updateServerDates($server, \@dates);
+            $czarDb->updateServerDates($server, \@dates,'gpc1');
         }
         else {
-
-            print "WARNING: No dates to update for '$server'\n";
+            print "WARNING: No dates to update for '$server' in gpc1\n";
+        }
+
+        my @dates2 = @{$pantasks->getDates($server,'gpc2')};
+        if (@dates2) {
+            $czarDb->updateServerDates($server, \@dates2,'gpc2');
+        }
+        else {
+            print "WARNING: No dates to update for '$server' in gpc2\n";
         }
     }
@@ -126 +132 @@
 
     print "* Updating labels\n";
-    # my @servers = ("stdscience", "distribution", "publishing", "update");
     my @servers = ("stdscience", "distribution");
 
     my @allLabels = ();
-
+    my @allLabels2 = ();
+    my @fields = ();
+
+    #first grab the labels fom the relevant pantasks
     my $server = undef;
     foreach $server (@servers) {
-        my @labels = @{$pantasks->getLabels($server)};
+        my @labels = @{$pantasks->getLabels($server,'gpc1')};
         push (@allLabels, @labels);
-    }
+
+        my @labels2 = @{$pantasks->getLabels($server,'gpc2')};
+        push (@allLabels2, @labels2);
+    }
+
+    ## adding extra labels from external file
+    my $filename = 'extra_labels.dat';
+    if (open(my $fh, '<:encoding(UTF-8)', $filename)) {
+      my $header = <$fh>;     
+      $header = <$fh>;     
+      while (my $row = <$fh>) {
+          chomp $row;
+          @fields = split(' ', $row);
+          if ($fields[1] eq 'gpc1') {
+              push (@allLabels, $fields[0]);
+          }
+          if ($fields[1] eq 'gpc2') {
+              push (@allLabels2, $fields[0]);
+          }
+      }
+    } else {
+      warn "No extra labels found in '$filename' $!";
+    }
+
+    # Now, grab the priorities for each label
+    my @priorities = ();
+    my $row = undef;
+    my $label = undef;
+    my $priority = undef;
+    foreach $label ( @allLabels ) {
+        $priority = $gpc1Db->getPriority($label);
+        push(@priorities, $priority);
+    }
+
+    #also grab gpc2 priorities from the gpc2 Label table
+    my @priorities2 = ();
+    my $priority2 = undef;
+    foreach $label ( @allLabels2 ) {
+        $priority2 = $gpc2Db->getPriority($label);
+        push(@priorities2, $priority2);
+    }
+
+    #sort labels by priority
+    my @idx = sort {$priorities[$a] <=> $priorities[$b]} 0 ..$#priorities;
+    @allLabels = @allLabels[@idx];
+    @priorities = @priorities[@idx]; 
+
+    my @idx2 = sort {$priorities2[$a] <=> $priorities2[$b]} 0 ..$#priorities2;
+    @allLabels2 = @allLabels2[@idx2];
+    @priorities2 = @priorities2[@idx2]; 
 
     # store them all in stdscience:
     $server = "stdscience";
     if (@allLabels) {
-        $czarDb->updateCurrentLabels($server, \@allLabels);
+        $czarDb->updateCurrentLabels($server, \@allLabels,'gpc1', \@priorities);
     } else {
-        print "WARNING: No labels to update for '$server'\n";
-    }
+        print "WARNING: No labels to update for gpc1 for '$server'\n";
+    }
+
+    if (@allLabels2) {
+        $czarDb->updateCurrentLabels($server, \@allLabels2,'gpc2', \@priorities2);
+    } else {
+        print "WARNING: No labels to update for gpc2 for '$server'\n";
+    }
+    return(\@allLabels,\@allLabels2);
 }
 
@@ -160 +224 @@
     my $running = undef;
     foreach $server (@{$servers}) {
-
-        $pantasks->getServerStatus($server, \$alive, \$running);
-        $czarDb->updateServerStatus($server, $alive, $running);
-    }
+        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
+        $czarDb->updateServerStatus($server,'gpc1', $alive, $running);
+        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
+        $czarDb->updateServerStatus($server,'gpc2', $alive, $running);
+   }
 }
 
@@ -182 +247 @@
     my $str = undef;
     my $labels = undef;
+    my $labels2 = undef;
     my $updateLabels = undef;
     my $row = undef;
@@ -187 +253 @@
     my $end = undef;
     my $priority = undef;
+    my $priority2 = undef;
     my $newState = undef;
     my $nsStatus = undef;
@@ -227 +294 @@
 
                 # now cleanup tables from yesterday and optimize
-                print "* Performing database cleanup\n";
-                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
-                print "* Optimizing the database\n";
-                $czarDb->optimize();
+                #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB. 
+                #Therefore, it makes no sense to run the cleanup+optimising twice
+                #print "* Performing database cleanup\n";
+                #$czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
+                #print "* Optimizing the database\n";
+                #$czarDb->optimize();
                 $lastDayDailyTasks = $yesterday;
             }
@@ -237 +306 @@
         # check nightly science status
         print "* Checking nightly science status\n";
-        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
-        $czarDb->updateNightlyScience($nsStatus);
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc1')) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus,'gpc1');
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc2')) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus,'gpc2');
 
         # check nebulous
@@ -252 +323 @@
 
         # check labels
-        updateLabels();
+        my ($labels, $labels2) = updateLabels();
+        my @labels = @$labels;
+        my @labels2 = @$labels2;
 
         # servers to check
@@ -265 +338 @@
 
             # deal with stdscience labels
-            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
             my $size = @{$labels};
             if($size > 0) {
-
-                # get priority
-                foreach $row ( @{$labels} ) {
-                    my ($label) = @{$row};
-                    # for now use priorities from gpc1 database
-                    $priority = $gpc1Db->getPriority($label);
-                    $czarDb->setLabelPriority($label, $priority);
-                }
-
-                updateAllStages($thisServer, $newState, $labels, $begin, $end);
+                updateAllStages($thisServer, $newState, $labels,$labels2, $begin, $end);
                 if ($thisServer eq "stdscience") {
                     createPlots($thisServer, $plotter1, $labels, $begin, $end);
-                    createPlots($thisServer, $plotter2, $labels, $begin, $end);
+                    createPlots($thisServer, $plotter2, $labels2, $begin, $end);
                 }
             }
@@ -376 +439 @@
     #  #  $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
 
-    #    #TdB 20190411: Make the all stages plots for the nightlyscience labels only
     #    if ($label =~ m/nightlyscience/) {
     #      $plotter->createTimeSeries($label, undef, $begin, $end, 1, 0, 1);
@@ -382 +444 @@
     #}
 
-    $allServerLabels = "all_".$server."_labels";
-
-    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 0, 1);
-
-    #TdB 20190411: We are turning off these plots for the time being. They take up a lot of time and are rarely looked at. If need be, can be generated on the fly using czartool.
+    $allServerLabels = $server."_labels";
+
+    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1);
+
     foreach $stage (@stages) {
-        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 1); # TODO must be a neater way...
+        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 0); # TODO must be a neater way...
     }
 }
@@ -398 +459 @@
 ###########################################################################
 sub updateAllStages {
-    my ($labelServer, $newState, $rows, $begin, $end) = @_;
+    my ($labelServer, $newState, $rows, $rows2, $begin, $end) = @_;
 
     print "* Updating stage data\n";
@@ -419 +480 @@
     foreach $stage (@stages) {
         $server = $pantasks->getServerForThisStage($stage);
-        $pantasks->getRevertStatus($stage, \$reverting);
-        $czarDb->updateRevertStatus($stage, $reverting);
+        $pantasks->getRevertStatus($stage,'gpc1', \$reverting);
+        $czarDb->updateRevertStatus($stage,'gpc1', $reverting);
+
+        $pantasks->getRevertStatus($stage,'gpc2', \$reverting);
+        $czarDb->updateRevertStatus($stage,'gpc2', $reverting);
 
         print "* Checking labels for $stage stage\n";
@@ -427 +491 @@
         if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) { 
             ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage);
-            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $new, $full, $faults);
+            $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $new, $full, $faults);
 
             ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage);
-            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $new, $full, $faults);
+            $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $new, $full, $faults);
 
             next;
         }
 
-
+        #count the number of exposures under each label, and their faults
         $totalNew1=$totalFaults1=$totalFull1=0;
-        $totalNew2=$totalFaults2=$totalFull2=0;
         foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
+            my ($label) = $row;
 
             chomp($label);
@@ -447 +510 @@
             $full = $gpc1Db->countExposures($label, $stage, "full");
             $faults = $gpc1Db->countFaults($label, $stage, $newState);
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
             $totalNew1 += $new;
             $totalFull1 += $full;
@@ -453 +515 @@
 
             $czarDb->insertNewTimeData($stage, $label, 'gpc1', $new, $full, $faults);
-
+        }
+
+        $totalNew2=$totalFaults2=$totalFull2=0;
+        foreach $row ( @{$rows2} ) {
+            my ($label) = $row;
+
+            chomp($label);
+
+            # Move this to a function and pass in the database
             $new = $gpc2Db->countExposures($label, $stage, $newState);
             $full = $gpc2Db->countExposures($label, $stage, "full");
             $faults = $gpc2Db->countFaults($label, $stage, $newState);
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
             $totalNew2 += $new;
             $totalFull2 += $full;
@@ -465 +534 @@
         }
 
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1);
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2);
-    }
-}
-
+        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1);
+        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2);
+    }
+}
+