Changeset 34930 for trunk/ippMonitor

Timestamp:

Jan 15, 2013, 1:46:48 PM (14 years ago)

Author:

Serge CHASTEL

Message:

Theoretically usable space added to czardb

Location:

trunk/ippMonitor/czartool

Files:

• czarDbSchema.sql (modified) (1 diff)
• czartool/CzarDb.pm (modified) (52 diffs)
• czartool/Nebulous.pm (modified) (11 diffs)

trunk/ippMonitor/czartool/czarDbSchema.sql

@@ -78 +78 @@
   `total` float default NULL,
   `available` float default NULL,
+  `usable` float default NULL,
   `used` float default NULL,
   `hostsOver98` smallint(6) default NULL,

trunk/ippMonitor/czartool/czartool/CzarDb.pm

@@ -9 +9 @@
 
 my @stages = (
-        "chip", 
-        "cam", 
-        "fake", 
-        "warp", 
-        "stack", 
-        "diff", 
-        "dist", 
+        "chip",
+        "cam",
+        "fake",
+        "warp",
+        "stack",
+        "diff",
+        "dist",
         "pub",
         "chipbackground"); # TODO put elsewhere
@@ -64 +64 @@
     SELECT DISTINCT label
         FROM processed
-        WHERE timestamp >= '$begin' 
-        AND timestamp <= '$end' 
+        WHERE timestamp >= '$begin'
+        AND timestamp <= '$end'
 SQL
 
@@ -109 +109 @@
 
     SELECT DISTINCT label
-        FROM chip 
-        WHERE timestamp >= '$begin' 
-        AND timestamp <= '$end' 
+        FROM chip
+        WHERE timestamp >= '$begin'
+        AND timestamp <= '$end'
         AND label NOT LIKE '%_ud%'
         AND label NOT LIKE 'update%'
@@ -153 +153 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT reverting 
+    SELECT reverting
         FROM reverts
-         WHERE stage LIKE '$stage'; 
+         WHERE stage LIKE '$stage';
 SQL
 
@@ -171 +171 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-     UPDATE reverts 
-         SET reverting = $reverting 
+     UPDATE reverts
+         SET reverting = $reverting
          WHERE stage LIKE '$stage';
 SQL
@@ -188 +188 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-     UPDATE current_labels 
-         SET priority = $priority 
-         WHERE label LIKE '$label' 
+     UPDATE current_labels
+         SET priority = $priority
+         WHERE label LIKE '$label'
          AND server LIKE 'stdscience';
 SQL
@@ -323 +323 @@
 ###########################################################################
 sub insertNewClusterSpace {
-    my ($self, $total, $available, $used, $hostsOver98) = @_;
+    my ($self, $total, $available, $usable, $used, $hostsOver98) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     INSERT INTO cluster_space
-        (total, available, used, hostsOver98)
+        (total, available, usable, used, hostsOver98)
         VALUES
-        ( $total, $available, $used, $hostsOver98);
+        ( $total, $available, $usable, $used, $hostsOver98);
 SQL
 
@@ -362 +362 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+    SELECT
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
         DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
         TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
-            FROM $stage 
+            FROM $stage
             WHERE label LIKE '$label'
-            AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+            AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
 SQL
 
@@ -413 +413 @@
 ###########################################################################
 #
-# Analysis this stage between these times 
+# Analysis this stage between these times
 #
 ###########################################################################
@@ -429 +429 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        timestamp, pending, faults, processed 
-        FROM $stage 
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+    SELECT
+        timestamp, pending, faults, processed
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
         ORDER BY timestamp;
 SQL
@@ -452 +452 @@
         # get linear values
         if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
-            
+
             $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
         }
@@ -480 +480 @@
     my $started = undef;
     my $finished = undef;
-   
+
     my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
 
@@ -492 +492 @@
         # get linear values
         if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
-            
+
             $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed);
         }
@@ -508 +508 @@
 
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp) + 0.001;
-                
+
             if ($thisProcessed >= $lastProcessed){
-                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff;
             }
             else {$derivProcessed = 0;}
-            $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
-            $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
+            $derivPending = abs($thisPending - $lastPending)/$timeDiff;
+            $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;
         }
         else {
-        
+
             $derivProcessed = 0;
             $derivPending = 0;
@@ -539 +539 @@
             else {
                 if (defined $notProcessing) {
-                    
-                    $notProcessing = undef; 
-                    $stuck = undef; 
+
+                    $notProcessing = undef;
+                    $stuck = undef;
                 }
                 if (!defined $started) {
 
                     $started = $lastTimestamp;
-                    $finished = undef; 
+                    $finished = undef;
                 }
             }
@@ -564 +564 @@
             # how long not processing?
             if (defined $notProcessing) {
-                
+
                 $howLongNotProcessing = $self->diffTimes($lastTimestamp, $notProcessing);
                 $notProcessingLongerThanInterval = $self->isBefore($interval, $howLongNotProcessing);
             }
             else {
-            
+
                 $howLongNotProcessing = 0;
                 $notProcessingLongerThanInterval = 0;
@@ -611 +611 @@
 
     if (!defined $started){
-    
+
         $stageMetrics->setStarted(undef);
     }
     else {
-    
+
         my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
         $stageMetrics->setRate($rate);
@@ -625 +625 @@
     }
     else {
-    
+
         $stageMetrics->setStuck(undef);
     }
@@ -642 +642 @@
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT  COUNT(*)
-        FROM $stage 
+        FROM $stage
         WHERE label LIKE '$label'
-        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
 SQL
 
@@ -657 +657 @@
     # grab all the data for the provided time range
     $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        timestamp, pending, faults, processed 
-        FROM $stage 
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+    SELECT
+        timestamp, pending, faults, processed
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
         ORDER BY timestamp;
 SQL
@@ -724 +724 @@
             $diffFaults = $thisFaults - $lastFaults;
 
-            # First, look for large positive jumps in processed, most likely a label 
+            # First, look for large positive jumps in processed, most likely a label
             # added, and store value to subtract from linear value below. If > 1000
             # processed per hour, then it must be wrong
@@ -741 +741 @@
             }
             else {
-            
+
                 $diffProcessed = 0;
                 $linearProcessed = $linearProcessed;
@@ -747 +747 @@
 
             # calculate first derivative in units of images per hour
-            $derivPending = $diffPending/($timeSep/3600); 
-            $derivFaults = $diffFaults/($timeSep/3600); 
-            $derivProcessed = $diffProcessed/($timeSep/3600); 
+            $derivPending = $diffPending/($timeSep/3600);
+            $derivFaults = $diffFaults/($timeSep/3600);
+            $derivProcessed = $diffProcessed/($timeSep/3600);
 
         }
         # first time in
         else {
-        
+
             $derivPending = 0;
             $derivFaults = 0;
@@ -777 +777 @@
             my $totalPending = 0;
             my $totalFaults = 0;
-            for (my $i=$size-1; $i>=($size - $range); $i--) { 
-               
+            for (my $i=$size-1; $i>=($size - $range); $i--) {
+
                 $totalPending = $totalPending + $ratePending[$i];
                 $totalFaults = $totalFaults + $rateFaults[$i];
@@ -833 +833 @@
     # get earliest time for this label
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval) 
+    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval)
         FROM $stage
-        WHERE label LIKE '$label'; 
-SQL
-    $query->execute;
-
-my $timestamp = $query->fetchrow_array(); 
+        WHERE label LIKE '$label';
+SQL
+    $query->execute;
+
+my $timestamp = $query->fetchrow_array();
 
     $query = $self->{_db}->prepare(<<SQL);
-    SELECT faults 
+    SELECT faults
         FROM $stage
-        WHERE label LIKE '$label' 
+        WHERE label LIKE '$label'
         AND timestamp <= '$timestamp' LIMIT 1;
 SQL
@@ -861 +861 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT pending - faults 
-        FROM $stage 
-        WHERE label LIKE '$label' 
+    SELECT pending - faults
+        FROM $stage
+        WHERE label LIKE '$label'
         ORDER BY timestamp DESC LIMIT 1;
 SQL
@@ -880 +880 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        pending, faults 
+    SELECT
+        pending, faults
         FROM $stage
         WHERE label LIKE '$label'
@@ -890 +890 @@
     if (!$query->execute) {return 0;}
 
-    (${$pending}, ${$faults}) = $query->fetchrow_array();    
+    (${$pending}, ${$faults}) = $query->fetchrow_array();
 
     ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
@@ -912 +912 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        host, used, available, writable, readable 
+    SELECT
+        host, used, available, writable, readable
         FROM hosts
         ORDER BY host;
@@ -926 +926 @@
 
         my ($host, $used, $available, $writable, $readable) = @row;
-   
-        if (($used+$available) == 0) {
-          $underLimit = 1;
-        } elsif (($used/($used+$available))*100 > $limit) {
-          $underLimit = 0;
-        } else {
-          $underLimit = 1;
-        }
+
+        if (($used+$available) == 0) {
+          $underLimit = 1;
+        } elsif (($used/($used+$available))*100 > $limit) {
+          $underLimit = 0;
+        } else {
+          $underLimit = 1;
+        }
 
         # Col 1: host
         print GNUDAT "$host";
-        
+
         # Col 2: available, readable used space under limit
         if ($readable && $underLimit) {print GNUDAT " $used";}
@@ -973 +973 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT used/total*100 
-        FROM cluster_space 
+    SELECT used/total*100
+        FROM cluster_space
         ORDER BY timestamp DESC LIMIT 1;
 SQL
@@ -983 +983 @@
 ###########################################################################
 #
-# Gets time series data for cluster storage and saves it to file 
+# Gets time series data for cluster storage and saves it to file
 #
 ###########################################################################
@@ -992 +992 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        MAX(available), MIN(available),  
-        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+    SELECT
+        MAX(available), MIN(available),
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
         DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
         TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
-            FROM cluster_space 
-            WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+            FROM cluster_space
+            WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime';
 SQL
 
@@ -1009 +1009 @@
 
     $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available 
-        FROM cluster_space 
-        WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+    SELECT
+        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available
+        FROM cluster_space
+        WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'
         ORDER BY timestamp;
 SQL
@@ -1074 +1074 @@
 ###########################################################################
 #
-# Gets count of processed stuff in a given time period 
+# Gets count of processed stuff in a given time period
 #
 ###########################################################################
@@ -1081 +1081 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        processed 
+    SELECT
+        processed
         FROM $stage
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$begin' 
-        AND timestamp < '$end' 
+        WHERE label LIKE '$label'
+        AND timestamp >= '$begin'
+        AND timestamp < '$end'
 SQL
         $query->execute;
@@ -1168 +1168 @@
 
                 $query = $self->{_db}->prepare(<<SQL);
-                SELECT COUNT(*) 
-                    FROM $stage 
+                SELECT COUNT(*)
+                    FROM $stage
                     WHERE timestamp > '$fromTime'
                     AND timestamp <= '$toTime'
-                    AND label = '$label' 
+                    AND label = '$label'
 SQL
 
@@ -1181 +1181 @@
 
                 $query = $self->{_db}->prepare(<<SQL);
-                DELETE FROM $stage 
-                    WHERE timestamp > '$fromTime' 
-                    AND timestamp <= '$toTime' 
+                DELETE FROM $stage
+                    WHERE timestamp > '$fromTime'
+                    AND timestamp <= '$toTime'
                     AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete
 SQL
@@ -1203 +1203 @@
 
                 $query = $self->{_db}->prepare(<<SQL);
-                SELECT COUNT(*) 
-                    FROM servers 
+                SELECT COUNT(*)
+                    FROM servers
                     WHERE timestamp > '$fromTime'
                     AND timestamp <= '$toTime'
-                    AND server = '$server' 
+                    AND server = '$server'
 SQL
                     $query->execute;
@@ -1215 +1215 @@
 
                     $query = $self->{_db}->prepare(<<SQL);
-                    DELETE FROM servers 
-                        WHERE timestamp > '$fromTime' 
-                        AND timestamp <= '$toTime' 
+                    DELETE FROM servers
+                        WHERE timestamp > '$fromTime'
+                        AND timestamp <= '$toTime'
                         AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
 SQL
@@ -1229 +1229 @@
         # now deal with cluster_space table
         $query = $self->{_db}->prepare(<<SQL);
-        SELECT COUNT(*) 
-            FROM cluster_space 
+        SELECT COUNT(*)
+            FROM cluster_space
             WHERE timestamp > '$fromTime'
             AND timestamp <= '$toTime'
@@ -1240 +1240 @@
 
             $query = $self->{_db}->prepare(<<SQL);
-            DELETE FROM cluster_space 
-                WHERE timestamp > '$fromTime' 
-                AND timestamp <= '$toTime' 
+            DELETE FROM cluster_space
+                WHERE timestamp > '$fromTime'
+                AND timestamp <= '$toTime'
                 ORDER BY timestamp DESC LIMIT $toDelete
 SQL
@@ -1257 +1257 @@
 ###########################################################################
 #
-# Optimizes all tables that need to be optimized 
+# Optimizes all tables that need to be optimized
 #
 ###########################################################################
@@ -1275 +1275 @@
 ###########################################################################
 #
-# Returns if a particular server has been stopped for a certain interval 
+# Returns if a particular server has been stopped for a certain interval
 #
 ###########################################################################
@@ -1286 +1286 @@
 ###########################################################################
 #
-# Returns if a particular server has been down for a certain interval 
+# Returns if a particular server has been down for a certain interval
 #
 ###########################################################################
@@ -1297 +1297 @@
 ###########################################################################
 #
-# Returns if a particular server has been down for a certain interval 
+# Returns if a particular server has been down for a certain interval
 #
 ###########################################################################
@@ -1304 +1304 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT COUNT(*) 
+    SELECT COUNT(*)
         FROM servers
-        WHERE server = '$server' 
+        WHERE server = '$server'
         AND timestamp > now() - INTERVAL $interval
 SQL
@@ -1316 +1316 @@
 
     $query = $self->{_db}->prepare(<<SQL);
-    SELECT COUNT(*) 
+    SELECT COUNT(*)
         FROM servers
-        WHERE server = '$server' 
+        WHERE server = '$server'
         AND timestamp > now() - INTERVAL $interval
         AND !$mode
@@ -1329 +1329 @@
 
     $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
+    SELECT
         timestamp
         FROM servers
         WHERE server = '$server'
-        AND $mode 
-        ORDER BY timestamp DESC 
+        AND $mode
+        ORDER BY timestamp DESC
         LIMIT 1
 
@@ -1346 +1346 @@
 ###########################################################################
 #
-# Returns an array of servers 
+# Returns an array of servers
 #
 ###########################################################################
@@ -1369 +1369 @@
 ###########################################################################
 #
-# Returns an array of labels present during the provided time frame 
+# Returns an array of labels present during the provided time frame
 #
 ###########################################################################
@@ -1376 +1376 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT DISTINCT label 
-        FROM $stage 
-        WHERE timestamp > '$fromTime' 
+    SELECT DISTINCT label
+        FROM $stage
+        WHERE timestamp > '$fromTime'
         AND timestamp <= '$toTime';
 SQL

trunk/ippMonitor/czartool/czartool/Nebulous.pm

@@ -28 +28 @@
 ###########################################################################
 #
-# Converts Kb to Tb 
-# 
+# Converts Kb to Tb
+#
 ###########################################################################
 sub convertKbToTb {
@@ -50 +50 @@
 ###########################################################################
 #
-# Returns total cluster usage in T 
+# Returns total cluster usage in T
 #
 ###########################################################################
@@ -61 +61 @@
 ###########################################################################
 #
-# Returns total cluster space in T 
+# Returns total cluster space in T
 #
 ###########################################################################
@@ -72 +72 @@
 ###########################################################################
 #
-# Returns total cluster available in T 
+# Returns total cluster available in T
 #
 ###########################################################################
@@ -83 +83 @@
 ###########################################################################
 #
-# Runs neb-df and puts output in gnuplot ready format 
+# Runs neb-df and puts output in gnuplot ready format
 #
 ###########################################################################
@@ -93 +93 @@
     $self->{_totalHosts} = 0;
     $self->{_hostsOverNinetyEightPC} = 0;
+    $self->{_totalUsable} = 0;
     my $readable;
     my $writable;
@@ -100 +101 @@
         if ($line =~ m/.*Filesystem.*/i) {next;}
         elsif ($line =~ m/.*summmary.*/i) {next;}
-        elsif ($line =~ m/.*total.*/i) {next;}
-        elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) {
+        elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*total.*/i) {
+            $self->{_totalUsable} += $self->convertKbToTb($2);
+        } elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) {
 
             $self->{_totalUsed} = $self->convertKbToTb($1);
@@ -109 +111 @@
 
             next;
-        }
-        elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
-               ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
-               ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
-        
+        } elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
+               ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
+               ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
+
             $self->{_totalHosts}++;
             if ($4 >= 98) {$self->{_hostsOverNinetyEightPC}++;}
@@ -119 +120 @@
             $self->getReadableWritableStatus($1, \$readable, \$writable);
 
-            $self->{_czarDb}->updateHost($1, 
-                    $self->convertKbToTb($2+$3), 
-                    $self->convertKbToTb($3), 
-                    $self->convertKbToTb($2), 
+            $self->{_czarDb}->updateHost($1,
+                    $self->convertKbToTb($2+$3),
+                    $self->convertKbToTb($3),
+                    $self->convertKbToTb($2),
                     $readable, $writable);
         }
@@ -128 +129 @@
 
     $self->{_czarDb}->insertNewClusterSpace(
-            $self->{_totalCluster}, 
-            $self->{_totalAvail},  
-            $self->{_totalUsed}, 
+            $self->{_totalCluster},
+            $self->{_totalAvail},
+            $self->{_totalUsable},
+            $self->{_totalUsed},
             $self->{_hostsOverNinetyEightPC});
 }
@@ -136 +138 @@
 ###########################################################################
 #
-# Runs neb-host and readable/writable status for this host 
+# Runs neb-host and readable/writable status for this host
 #
 ###########################################################################