Changeset 29686 for trunk/tools/czartool/CzarDb.pm

Timestamp:

Nov 5, 2010, 3:54:50 PM (16 years ago)

Author:

rhenders

Message:

major changes to support generaionof IPP metrics

File:

• trunk/tools/czartool/CzarDb.pm (modified) (11 diffs)

trunk/tools/czartool/CzarDb.pm

@@ -10 +10 @@
 use base 'czartool::MySQLDb';
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
 
 ###########################################################################
@@ -49 +51 @@
     return 1;
 }
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
 
 ###########################################################################
@@ -264 +295 @@
 }
 
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
 
 ###########################################################################
@@ -271 +326 @@
 ###########################################################################
 sub runAnalysis {
-    my ($self, 
-            $label, 
-            $stage, 
-            $fromTime, 
-            $toTime, 
-            $_started, 
-            $_finished, 
-            $_stuck,
-            $_processed,
-            $_pending,
-            $_faults,
-            $_totalTime) = @_;
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -293 +346 @@
 SQL
 
-    $query->execute;
-
     my $lastProcessed = -1;
     my $lastPending = -1;
@@ -300 +351 @@
     my $lastTimestamp = undef;
     my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $started = undef;
-    my $finished = undef;
-    
     my $notProcessing = undef;
     my $howLongNotProcessing = undef;
     my $notProcessingLongerThanInterval = 0;
-
     my $stuck = undef;
     my $howLongStuck = undef;
     my $stuckLongerThanInterval = 0;
-
     my $stuffToDo = undef;
-    my $interval = "00:20:00";
-
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
     $linearProcessed = 0;
+    $lastProcessed = -1;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -326 +406 @@
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
 
         # get first derivative values
@@ -363 +449 @@
                 if (defined $notProcessing) {
                     
-                    if ($stuckLongerThanInterval) {
-
-#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
-                    }
-                    if ($notProcessingLongerThanInterval) {
-
-            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
-                    }
                     $notProcessing = undef; 
                     $stuck = undef; 
                 }
                 if (!defined $started) {
+
                     $started = $lastTimestamp;
                     $finished = undef; 
-#                    print "* $stage processing started at $started\n";
                 }
             }
@@ -418 +496 @@
     }
 
-    ${$_processed} = $linearProcessed;
-    ${$_pending} = $stuffToDo;
-    ${$_faults} = $lastFaults;
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
 
     if (defined $finished) {
-        ${$_finished} = $finished;
-        ${$_totalTime} = $self->diffTimes($finished, $started);
-    }
-
-    elsif(defined $started) {
-    
-        ${$_finished} = undef;
-    }
-    
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
     if (!defined $started){
     
-        ${$_started} = undef;
+        $stageMetrics->setStarted(undef);
     }
     else {
     
-        ${$_started} = $started;
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
     }
     if ($stuck && $stuckLongerThanInterval && !defined $finished) {
 
-        ${$_stuck} = $stuck;
+        $stageMetrics->setStuck($stuck);
     }
     else {
     
-        ${$_stuck} = undef;
+        $stageMetrics->setStuck(undef);
     }
 
@@ -517 +606 @@
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-            if ($thisProcessed >= $lastProcessed){
+            if ($thisProcessed >= $lastProcessed && $timeDiff){
                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
@@ -1512 +1601 @@
     $self->setRevision(12);
 }
-
 1;