Index: trunk/tools/czartool/CzarDb.pm =================================================================== --- trunk/tools/czartool/CzarDb.pm (revision 31875) +++ trunk/tools/czartool/CzarDb.pm (revision 31886) @@ -590,13 +590,13 @@ ########################################################################### # -# Gets time series data and stores it to temp file +# Gets time series data and stores it to temp files # ########################################################################### sub createTimeSeriesData { - my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<execute) {return 0;} - my $minProcessed = scalar $query->fetchrow_array(); - if (!defined $minProcessed) {return 0;} - + my ($count) = $query->fetchrow_array(); + + # decide on a number for the running mean for rate plots - use 10% of total data points + my $range = int($count/10); $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff); - + # grab all the data for the provided time range $query = $self->{_db}->prepare(<execute; + # set up Gnuplot datafiles my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); $tmpFile->unlink_on_destroy( 0 ); - ${$dataFile} = $tmpFile->filename; - open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + ${$linDataFile} = $tmpFile->filename; + open (LINDAT, ">${$linDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + + $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); + $tmpFile->unlink_on_destroy( 0 ); + ${$logDataFile} = $tmpFile->filename; + open (LOGDAT, ">${$logDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + + $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); + $tmpFile->unlink_on_destroy( 0 ); + ${$ratDataFile} = $tmpFile->filename; + open (RATDAT, ">${$ratDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; my $lastProcessed = -1; @@ -633,53 +645,104 @@ my ($logProcessed, $logPending, $logFaults); my ($derivProcessed, $derivPending, $derivFaults); - my $timestamp = undef; + my (@rateProcessed, @ratePending, @rateFaults); + my $formattedTimestamp = undef; + my $middleTime = undef; + my $formattedMiddleTimestamp = undef; my $positiveJump = 0; + my @rateTimes; + my $diffPending = 0; + my $diffProcessed = 0; + my $diffFaults = 0; + my $lastDiffProcessed = 0; + $derivProcessed = 0; $linearProcessed = 0; my $someData = 0; + my $timeSep; while (my @row = $query->fetchrow_array()) { + my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row; - - # get first derivative values + $formattedTimestamp = $self->getFormattedDate($thisTimestamp); + + # if not first time in.... if (defined $lastTimestamp) { - my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); - - if ($thisProcessed >= $lastProcessed && $timeDiff){ - $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; + # times + $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); + $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp); + $formattedMiddleTimestamp = $self->getFormattedDate($middleTime); + + $diffPending = $thisPending - $lastPending; + $diffFaults = $thisFaults - $lastFaults; + + # we only count increases in processing - drops are due to cleanup etc + if ($thisProcessed > $lastProcessed) { + + $diffProcessed = ($thisProcessed - $lastProcessed) - $positiveJump; + $linearProcessed = $linearProcessed + $diffProcessed; } - else {$derivProcessed = 0;} + else { + + $diffProcessed = 0; + $linearProcessed = $linearProcessed; + } + + # calculate first derivative in units of images per hour + $derivPending = $diffPending/($timeSep/3600); + $derivFaults = $diffFaults/($timeSep/3600); + $derivProcessed = $diffProcessed/($timeSep/3600); + + # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below + if ($derivProcessed > 1000) { + $positiveJump = $thisProcessed - $lastProcessed; + } + else { + + $positiveJump = 0; + } + } + # first time in + else { + $derivPending = 0; $derivFaults = 0; - # $derivPending = abs($thisPending - $lastPending)/$timeDiff; - # $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff; - - # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below - if ($derivProcessed > 0.3) { - - $positiveJump = $thisProcessed - $lastProcessed; - #print "$derivProcessed at $thisTimestamp $positiveJump\n"; + $derivProcessed = 0; + $formattedMiddleTimestamp = $formattedTimestamp; } - else { - $positiveJump = 0; + + # deal with rate plot + push(@ratePending, $derivPending); + push(@rateFaults, $derivFaults); + push(@rateProcessed, $derivProcessed); + push(@rateTimes, $middleTime); + + # get last few values + my $size = scalar (@rateProcessed); + if ($size > $range) { + + my $counter = 0; + my $oldTime; + my $newTime = $rateTimes[$size-1]; + my $totalProcessed = 0; + my $totalPending = 0; + my $totalFaults = 0; + for (my $i=$size-1; $i>=($size - $range); $i--) { + + $totalPending = $totalPending + $ratePending[$i]; + $totalFaults = $totalFaults + $rateFaults[$i]; + $totalProcessed = $totalProcessed + $rateProcessed[$i]; + $oldTime = $rateTimes[$i]; + $counter++; } + + my $meanPending = $totalPending / $counter; + my $meanFaults = $totalFaults / $counter; + my $meanProcessed = $totalProcessed / $counter; + + my $formattedTimestamp = $self->getFormattedDate($self->getMiddleTime($oldTime, $newTime)); + print RATDAT "$formattedTimestamp $meanPending $meanFaults $meanProcessed\n"; } - else { - - $derivProcessed = 0; - $derivPending = 0; - $derivFaults = 0; - } - - # get linear values - if ($showCleanup) { - - $linearProcessed = $thisProcessed - $minProcessed; - } - elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) { - - $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed) - $positiveJump; - } + $linearPending = $thisPending; $linearFaults = $thisFaults; @@ -694,8 +757,6 @@ # print to data file - $timestamp = $self->getFormattedDate($thisTimestamp); - if ($firstDeriv) {print GNUDAT "$timestamp $derivPending $derivFaults $derivProcessed\n";} - elsif($isLog) {print GNUDAT "$timestamp $logPending $logFaults $logProcessed\n";} - else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";} + print LOGDAT "$formattedTimestamp $logPending $logFaults $logProcessed\n"; + print LINDAT "$formattedTimestamp $linearPending $linearFaults $linearProcessed\n"; if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;} @@ -705,9 +766,10 @@ $lastFaults = $thisFaults; $lastTimestamp = $thisTimestamp; - } - - close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n"; - - #return $someData; + $lastDiffProcessed = $diffProcessed; + } + + close(LINDAT) or print "* Problem closing linear gnuplot data file for plot for '$label' '$stage'\n"; + close(LOGDAT) or print "* Problem closing log gnuplot data file for plot for '$label' '$stage'\n"; + close(RATDAT) or print "* Problem closing rate gnuplot data file for plot for '$label' '$stage'\n"; } @@ -1017,5 +1079,5 @@ # # Deletes all but one row per interval from all tables for the provided date range -# TODO this is very clumsy, I just have time to thinmk of something more elegant +# TODO this is very clumsy, I just don't have time to think of something more elegant # ###########################################################################