Changeset 29834 for branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm

Timestamp:

Nov 26, 2010, 10:33:29 AM (16 years ago)

Author:

eugene

Message:

merge updates from trunk

File:

• branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm (modified) (22 diffs)

branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm

@@ -6 +6 @@
 use strict;
 
+use File::Temp;
+
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
 
 use base 'czartool::MySQLDb';
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
 
 ###########################################################################
@@ -49 +53 @@
     return 1;
 }
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
 
 ###########################################################################
@@ -264 +297 @@
 }
 
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
 
 ###########################################################################
@@ -271 +328 @@
 ###########################################################################
 sub runAnalysis {
-    my ($self, 
-            $label, 
-            $stage, 
-            $fromTime, 
-            $toTime, 
-            $_started, 
-            $_finished, 
-            $_stuck,
-            $_processed,
-            $_pending,
-            $_faults,
-            $_totalTime) = @_;
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -293 +348 @@
 SQL
 
-    $query->execute;
-
     my $lastProcessed = -1;
     my $lastPending = -1;
@@ -300 +353 @@
     my $lastTimestamp = undef;
     my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $started = undef;
-    my $finished = undef;
-    
     my $notProcessing = undef;
     my $howLongNotProcessing = undef;
     my $notProcessingLongerThanInterval = 0;
-
     my $stuck = undef;
     my $howLongStuck = undef;
     my $stuckLongerThanInterval = 0;
-
     my $stuffToDo = undef;
-    my $interval = "00:20:00";
-
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
     $linearProcessed = 0;
+    $lastProcessed = -1;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -326 +408 @@
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
 
         # get first derivative values
@@ -363 +451 @@
                 if (defined $notProcessing) {
                     
-                    if ($stuckLongerThanInterval) {
-
-#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
-                    }
-                    if ($notProcessingLongerThanInterval) {
-
-            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
-                    }
                     $notProcessing = undef; 
                     $stuck = undef; 
                 }
                 if (!defined $started) {
+
                     $started = $lastTimestamp;
                     $finished = undef; 
-#                    print "* $stage processing started at $started\n";
                 }
             }
@@ -418 +498 @@
     }
 
-    ${$_processed} = $linearProcessed;
-    ${$_pending} = $stuffToDo;
-    ${$_faults} = $lastFaults;
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
 
     if (defined $finished) {
-        ${$_finished} = $finished;
-        ${$_totalTime} = $self->diffTimes($finished, $started);
-    }
-
-    elsif(defined $started) {
-    
-        ${$_finished} = undef;
-    }
-    
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
     if (!defined $started){
     
-        ${$_started} = undef;
+        $stageMetrics->setStarted(undef);
     }
     else {
     
-        ${$_started} = $started;
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
     }
     if ($stuck && $stuckLongerThanInterval && !defined $finished) {
 
-        ${$_stuck} = $stuck;
+        $stageMetrics->setStuck($stuck);
     }
     else {
     
-        ${$_stuck} = undef;
+        $stageMetrics->setStuck(undef);
     }
 
@@ -484 +575 @@
     $query->execute;
 
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
@@ -497 +590 @@
     
     $linearProcessed = 0;
+    my $someData = 0;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -517 +611 @@
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-            if ($thisProcessed >= $lastProcessed){
+            if ($thisProcessed >= $lastProcessed && $timeDiff){
                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
@@ -547 +641 @@
         else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
 
+        if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
+
         $lastProcessed = $thisProcessed;
         $lastPending = $thisPending;
@@ -555 +651 @@
     close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
 
-    return 1;
+    #return $someData;
 }
 
@@ -637 +733 @@
 ###########################################################################
 sub createHostsData {
-    my ($self, $limit) = @_;
-
-    my $dataFile = "/tmp/czarplot_hosts_space.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $limit, $tmpFile) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -686 +781 @@
         print GNUDAT "\n";
     }
+
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -714 +808 @@
 ###########################################################################
 sub createStorageTimeSeriesData {
-    my ($self, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
-
-    my $dataFile = "/tmp/czarplot_gnuplot_storage_timeseries.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $tmpFile, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -753 +846 @@
 
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -763 +854 @@
 ###########################################################################
 sub createProcessingRateData {
-    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
-
-    my $startTime = $startDay;
+    my ($self, $stage, $label, $begin, $end, $interval, $dataFile, $isLog) = @_;
+
+    my $startTime = $begin;
     my $endTime;
     my $quit = 0;
@@ -772 +863 @@
     my $faults;
     my $timestamp;
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
+
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     my $cleanupCarry = 0;
+    my $someData = 0;
     while(1) {
 
-        if (!$self->isBefore($startTime, $endDay)) {last;}
+
+        #print "NNN $startTime, $end\n";
+        if (!$self->isBefore($startTime, $end)) {last;}
         $endTime = $self->addInterval($startTime, $interval);
         $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
@@ -783 +881 @@
         print GNUDAT "$timestamp $processed 0 0\n";
 
+        if ($processed > 0 ) {$someData = 1;}
+
         $startTime = $endTime;
     }
 
-    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n";
+
+    #return $someData;
 }
 
@@ -1512 +1614 @@
     $self->setRevision(12);
 }
-
 1;