Changeset 31886 for trunk/tools/czartool/CzarDb.pm

Timestamp:

Jul 15, 2011, 4:16:20 PM (15 years ago)

Author:

rhenders

Message:

all time-series plots now created at same time for speed

File:

• trunk/tools/czartool/CzarDb.pm (modified) (7 diffs)

trunk/tools/czartool/CzarDb.pm

@@ -590 +590 @@
 ###########################################################################
 #
-# Gets time series data and stores it to temp file
+# Gets time series data and stores it to temp files
 #
 ###########################################################################
 sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        MIN(processed)
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $linDataFile, $logDataFile, $ratDataFile) = @_;
+
+    # get total number of data points
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT  COUNT(*)
         FROM $stage 
         WHERE label LIKE '$label'
@@ -605 +605 @@
 
     if (!$query->execute) {return 0;}
-    my $minProcessed = scalar  $query->fetchrow_array();
-    if (!defined $minProcessed) {return 0;}
-
+    my ($count) = $query->fetchrow_array();
+
+    # decide on a number for the running mean for rate plots - use 10% of total data points
+    my $range = int($count/10);
     $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
-
+    # grab all the data for the provided time range
     $query = $self->{_db}->prepare(<<SQL);
     SELECT 
@@ -621 +622 @@
     $query->execute;
 
+    # set up Gnuplot datafiles
     my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     $tmpFile->unlink_on_destroy( 0 );
-    ${$dataFile} = $tmpFile->filename;
-    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+    ${$linDataFile} = $tmpFile->filename;
+    open (LINDAT, ">${$linDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$logDataFile} = $tmpFile->filename;
+    open (LOGDAT, ">${$logDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$ratDataFile} = $tmpFile->filename;
+    open (RATDAT, ">${$ratDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
     my $lastProcessed = -1;
@@ -633 +645 @@
     my ($logProcessed, $logPending, $logFaults);
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $timestamp = undef;
+    my (@rateProcessed, @ratePending, @rateFaults);
+    my $formattedTimestamp = undef;
+    my $middleTime = undef;
+    my $formattedMiddleTimestamp = undef;
     my $positiveJump = 0;
+    my @rateTimes;
+    my $diffPending = 0;
+    my $diffProcessed = 0;
+    my $diffFaults = 0;
+    my $lastDiffProcessed = 0;
+    $derivProcessed = 0;
 
     $linearProcessed = 0;
     my $someData = 0;
+    my $timeSep;
     while (my @row = $query->fetchrow_array()) {
+
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
 
-
-        # get first derivative values
+        $formattedTimestamp = $self->getFormattedDate($thisTimestamp);
+
+        # if not first time in....
         if (defined $lastTimestamp) {
 
-            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
-                
-            if ($thisProcessed >= $lastProcessed && $timeDiff){
-                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+            # times
+            $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+            $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp);
+            $formattedMiddleTimestamp = $self->getFormattedDate($middleTime);
+
+            $diffPending = $thisPending - $lastPending;
+            $diffFaults = $thisFaults - $lastFaults;
+
+            # we only count increases in processing - drops are due to cleanup etc
+            if ($thisProcessed > $lastProcessed) {
+
+                $diffProcessed = ($thisProcessed - $lastProcessed) - $positiveJump;
+                $linearProcessed = $linearProcessed + $diffProcessed;
             }
-            else {$derivProcessed = 0;}
+            else {
+            
+                $diffProcessed = 0;
+                $linearProcessed = $linearProcessed;
+            }
+
+            # calculate first derivative in units of images per hour
+            $derivPending = $diffPending/($timeSep/3600); 
+            $derivFaults = $diffFaults/($timeSep/3600); 
+            $derivProcessed = $diffProcessed/($timeSep/3600); 
+
+            # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below
+            if ($derivProcessed > 1000) {
+                $positiveJump = $thisProcessed - $lastProcessed;
+            }
+            else {
+
+                $positiveJump = 0;
+            }
+        }
+        # first time in
+        else {
+        
             $derivPending = 0;
             $derivFaults = 0;
-        #    $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
-        #    $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
-
-            # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below
-            if ($derivProcessed > 0.3) {
-
-                $positiveJump = $thisProcessed - $lastProcessed;
-                #print "$derivProcessed at $thisTimestamp   $positiveJump\n";
+            $derivProcessed = 0;
+            $formattedMiddleTimestamp = $formattedTimestamp;
         }
-            else {
-            $positiveJump = 0;
+
+        # deal with rate plot
+        push(@ratePending, $derivPending);
+        push(@rateFaults, $derivFaults);
+        push(@rateProcessed, $derivProcessed);
+        push(@rateTimes, $middleTime);
+
+        # get last few values
+        my $size = scalar (@rateProcessed);
+        if ($size > $range) {
+
+            my $counter = 0;
+            my $oldTime;
+            my $newTime = $rateTimes[$size-1];
+            my $totalProcessed = 0;
+            my $totalPending = 0;
+            my $totalFaults = 0;
+            for (my $i=$size-1; $i>=($size - $range); $i--) { 
+               
+                $totalPending = $totalPending + $ratePending[$i];
+                $totalFaults = $totalFaults + $rateFaults[$i];
+                $totalProcessed = $totalProcessed + $rateProcessed[$i];
+                $oldTime = $rateTimes[$i];
+                $counter++;
             }
+
+            my $meanPending = $totalPending / $counter;
+            my $meanFaults = $totalFaults / $counter;
+            my $meanProcessed = $totalProcessed / $counter;
+
+            my $formattedTimestamp = $self->getFormattedDate($self->getMiddleTime($oldTime, $newTime));
+            print RATDAT "$formattedTimestamp $meanPending $meanFaults $meanProcessed\n";
         }
-        else {
-        
-            $derivProcessed = 0;
-            $derivPending = 0;
-            $derivFaults = 0;
-        }
-
-        # get linear values
-        if ($showCleanup) {
-        
-            $linearProcessed = $thisProcessed - $minProcessed;
-        }
-        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
-            
-            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed) - $positiveJump;
-        }
+
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
@@ -694 +757 @@
 
         # print to data file
-        $timestamp = $self->getFormattedDate($thisTimestamp);
-        if ($firstDeriv) {print GNUDAT "$timestamp $derivPending $derivFaults $derivProcessed\n";}
-        elsif($isLog) {print GNUDAT "$timestamp $logPending $logFaults $logProcessed\n";}
-        else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
+        print LOGDAT "$formattedTimestamp $logPending $logFaults $logProcessed\n";
+        print LINDAT "$formattedTimestamp $linearPending $linearFaults $linearProcessed\n";
 
         if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
@@ -705 +766 @@
         $lastFaults = $thisFaults;
         $lastTimestamp = $thisTimestamp;
-    }
-
-    close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
-
-    #return $someData;
+        $lastDiffProcessed = $diffProcessed;
+    }
+
+    close(LINDAT) or print "* Problem closing linear gnuplot data file for plot for '$label' '$stage'\n";
+    close(LOGDAT) or print "* Problem closing log gnuplot data file for plot for '$label' '$stage'\n";
+    close(RATDAT) or print "* Problem closing rate gnuplot data file for plot for '$label' '$stage'\n";
 }
 
@@ -1017 +1079 @@
 #
 # Deletes all but one row per interval from all tables for the provided date range
-# TODO this is very clumsy, I just have time to thinmk of something more elegant
+# TODO this is very clumsy, I just don't have time to think of something more elegant
 #
 ###########################################################################