Index: trunk/tools/czarplot.pl =================================================================== --- trunk/tools/czarplot.pl (revision 31877) +++ trunk/tools/czarplot.pl (revision 31886) @@ -26,5 +26,4 @@ my $rateInterval = undef; my $deriv = undef; -my $showCleanup = undef; my $nebulous = undef; my $savingToFile = undef; @@ -46,5 +45,4 @@ "histogram|h" => \$histogram, "nebulous|n" => \$nebulous, - "cleanup|c" => \$showCleanup, "rate|r" => \$rate, "deriv|f" => \$deriv, @@ -81,7 +79,4 @@ $analysis = 0; print "* OPTIONAL: include analysis -a (default=$analysis)\n";} -if (!$showCleanup) { - $showCleanup = 0; - print "* OPTIONAL: include cleanup in timeseries -c (default=$showCleanup)\n";} if (!$log) { $log = 0; @@ -166,12 +161,15 @@ } -if ($rate) { +if ($rate and !$timeSeries) { - # default to a rate histogram if time series not selected - if (!$timeSeries) {$histogram = 1;} - $plotter->createRatePlot($label, $stage, $begin, $end, $rateInterval, $histogram, $timeSeries); + $plotter->createRateHistogram($label, $stage, $begin, $end, $rateInterval); + exit; +} + +if (!$histogram && !$nebulous) { + $plotter->createTimeSeries($label, $stage, $begin, $end, $timeSeries, $log, $rate); exit; } -if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);} + if ($histogram) {$plotter->createHistogram($label, $begin, $end);} if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);} Index: trunk/tools/czarpoll.pl =================================================================== --- trunk/tools/czarpoll.pl (revision 31877) +++ trunk/tools/czarpoll.pl (revision 31886) @@ -252,6 +252,6 @@ chomp($label); - $plotter->createLogAndLinearTimeSeries($label, $stage, $begin, $end); - $plotter->createRatePlot($label, $stage, $begin, $end, undef, 1, 1); + $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 1, 1); + $plotter->createRateHistogram($label, $stage, $begin, $end, undef); } } @@ -263,6 +263,6 @@ my ($label) = @{$row}; - $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end); - $plotter->createRatePlot($label, undef, $begin, $end, undef, 1, 1); + $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); + $plotter->createRateHistogram($label, undef, $begin, $end, undef); $plotter->createHistogram($label, $begin, $end); } @@ -270,11 +270,11 @@ $allServerLabels = "all_".$server."_labels"; - $plotter->createLogAndLinearTimeSeries($allServerLabels, undef, $begin, $end); - $plotter->createRatePlot($allServerLabels, undef, $begin, $end, undef, 1, 1); + $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1); + $plotter->createRateHistogram($allServerLabels, undef, $begin, $end, undef); $plotter->createHistogram($allServerLabels, $begin, $end); foreach $stage (@stages) { - $plotter->createLogAndLinearTimeSeries($allServerLabels, $stage, $begin, $end); # TODO must be a neater way... - $plotter->createRatePlot($allServerLabels, $stage, $begin, $end, undef, 1, 1); + $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 1, 1); # TODO must be a neater way... + $plotter->createRateHistogram($allServerLabels, $stage, $begin, $end, undef); } } Index: trunk/tools/czartool/CzarDb.pm =================================================================== --- trunk/tools/czartool/CzarDb.pm (revision 31877) +++ trunk/tools/czartool/CzarDb.pm (revision 31886) @@ -590,13 +590,13 @@ ########################################################################### # -# Gets time series data and stores it to temp file +# Gets time series data and stores it to temp files # ########################################################################### sub createTimeSeriesData { - my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<execute) {return 0;} - my $minProcessed = scalar $query->fetchrow_array(); - if (!defined $minProcessed) {return 0;} - + my ($count) = $query->fetchrow_array(); + + # decide on a number for the running mean for rate plots - use 10% of total data points + my $range = int($count/10); $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff); - + # grab all the data for the provided time range $query = $self->{_db}->prepare(<execute; + # set up Gnuplot datafiles my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); $tmpFile->unlink_on_destroy( 0 ); - ${$dataFile} = $tmpFile->filename; - open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + ${$linDataFile} = $tmpFile->filename; + open (LINDAT, ">${$linDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + + $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); + $tmpFile->unlink_on_destroy( 0 ); + ${$logDataFile} = $tmpFile->filename; + open (LOGDAT, ">${$logDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; + + $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); + $tmpFile->unlink_on_destroy( 0 ); + ${$ratDataFile} = $tmpFile->filename; + open (RATDAT, ">${$ratDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; my $lastProcessed = -1; @@ -633,53 +645,104 @@ my ($logProcessed, $logPending, $logFaults); my ($derivProcessed, $derivPending, $derivFaults); - my $timestamp = undef; + my (@rateProcessed, @ratePending, @rateFaults); + my $formattedTimestamp = undef; + my $middleTime = undef; + my $formattedMiddleTimestamp = undef; my $positiveJump = 0; + my @rateTimes; + my $diffPending = 0; + my $diffProcessed = 0; + my $diffFaults = 0; + my $lastDiffProcessed = 0; + $derivProcessed = 0; $linearProcessed = 0; my $someData = 0; + my $timeSep; while (my @row = $query->fetchrow_array()) { + my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row; - - # get first derivative values + $formattedTimestamp = $self->getFormattedDate($thisTimestamp); + + # if not first time in.... if (defined $lastTimestamp) { - my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); - - if ($thisProcessed >= $lastProcessed && $timeDiff){ - $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; + # times + $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); + $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp); + $formattedMiddleTimestamp = $self->getFormattedDate($middleTime); + + $diffPending = $thisPending - $lastPending; + $diffFaults = $thisFaults - $lastFaults; + + # we only count increases in processing - drops are due to cleanup etc + if ($thisProcessed > $lastProcessed) { + + $diffProcessed = ($thisProcessed - $lastProcessed) - $positiveJump; + $linearProcessed = $linearProcessed + $diffProcessed; } - else {$derivProcessed = 0;} + else { + + $diffProcessed = 0; + $linearProcessed = $linearProcessed; + } + + # calculate first derivative in units of images per hour + $derivPending = $diffPending/($timeSep/3600); + $derivFaults = $diffFaults/($timeSep/3600); + $derivProcessed = $diffProcessed/($timeSep/3600); + + # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below + if ($derivProcessed > 1000) { + $positiveJump = $thisProcessed - $lastProcessed; + } + else { + + $positiveJump = 0; + } + } + # first time in + else { + $derivPending = 0; $derivFaults = 0; - # $derivPending = abs($thisPending - $lastPending)/$timeDiff; - # $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff; - - # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below - if ($derivProcessed > 0.3) { - - $positiveJump = $thisProcessed - $lastProcessed; - #print "$derivProcessed at $thisTimestamp $positiveJump\n"; + $derivProcessed = 0; + $formattedMiddleTimestamp = $formattedTimestamp; } - else { - $positiveJump = 0; + + # deal with rate plot + push(@ratePending, $derivPending); + push(@rateFaults, $derivFaults); + push(@rateProcessed, $derivProcessed); + push(@rateTimes, $middleTime); + + # get last few values + my $size = scalar (@rateProcessed); + if ($size > $range) { + + my $counter = 0; + my $oldTime; + my $newTime = $rateTimes[$size-1]; + my $totalProcessed = 0; + my $totalPending = 0; + my $totalFaults = 0; + for (my $i=$size-1; $i>=($size - $range); $i--) { + + $totalPending = $totalPending + $ratePending[$i]; + $totalFaults = $totalFaults + $rateFaults[$i]; + $totalProcessed = $totalProcessed + $rateProcessed[$i]; + $oldTime = $rateTimes[$i]; + $counter++; } + + my $meanPending = $totalPending / $counter; + my $meanFaults = $totalFaults / $counter; + my $meanProcessed = $totalProcessed / $counter; + + my $formattedTimestamp = $self->getFormattedDate($self->getMiddleTime($oldTime, $newTime)); + print RATDAT "$formattedTimestamp $meanPending $meanFaults $meanProcessed\n"; } - else { - - $derivProcessed = 0; - $derivPending = 0; - $derivFaults = 0; - } - - # get linear values - if ($showCleanup) { - - $linearProcessed = $thisProcessed - $minProcessed; - } - elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) { - - $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed) - $positiveJump; - } + $linearPending = $thisPending; $linearFaults = $thisFaults; @@ -694,8 +757,6 @@ # print to data file - $timestamp = $self->getFormattedDate($thisTimestamp); - if ($firstDeriv) {print GNUDAT "$timestamp $derivPending $derivFaults $derivProcessed\n";} - elsif($isLog) {print GNUDAT "$timestamp $logPending $logFaults $logProcessed\n";} - else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";} + print LOGDAT "$formattedTimestamp $logPending $logFaults $logProcessed\n"; + print LINDAT "$formattedTimestamp $linearPending $linearFaults $linearProcessed\n"; if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;} @@ -705,9 +766,10 @@ $lastFaults = $thisFaults; $lastTimestamp = $thisTimestamp; - } - - close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n"; - - #return $someData; + $lastDiffProcessed = $diffProcessed; + } + + close(LINDAT) or print "* Problem closing linear gnuplot data file for plot for '$label' '$stage'\n"; + close(LOGDAT) or print "* Problem closing log gnuplot data file for plot for '$label' '$stage'\n"; + close(RATDAT) or print "* Problem closing rate gnuplot data file for plot for '$label' '$stage'\n"; } @@ -1017,5 +1079,5 @@ # # Deletes all but one row per interval from all tables for the provided date range -# TODO this is very clumsy, I just have time to thinmk of something more elegant +# TODO this is very clumsy, I just don't have time to think of something more elegant # ########################################################################### Index: trunk/tools/czartool/Metrics.pm =================================================================== --- trunk/tools/czartool/Metrics.pm (revision 31877) +++ trunk/tools/czartool/Metrics.pm (revision 31886) @@ -77,9 +77,9 @@ print $htmlFile "\n"; print $htmlFile "

Summary plots (top)

\n"; - $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0); + $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 1, 1, 1); print $htmlFile "\"timeseries\"\n"; $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0); print $htmlFile "\"histogram\"\n"; - $self->{plotter}->createRatePlot("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef, 1, 1); + $self->{plotter}->createRateHistogram("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef); print $htmlFile "\"rate\"\n"; $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end}); Index: trunk/tools/czartool/Plotter.pm =================================================================== --- trunk/tools/czartool/Plotter.pm (revision 31877) +++ trunk/tools/czartool/Plotter.pm (revision 31886) @@ -97,16 +97,4 @@ my $type = $isLog ? "log" : "linear"; return $prefix . "/czarplot_" . $type . "_" . $label . "_" . $stagePart . "_$suffix.png"; -} - -########################################################################### -# -# Plots a time series for all stages for this label, both log and linear -# -########################################################################### -sub createLogAndLinearTimeSeries { - my ($self, $label, $selectedStage, $beginTime, $endTime) = @_; - - $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 1); - $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 0); } @@ -146,6 +134,6 @@ # ########################################################################### -sub createRatePlot { - my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $histogram, $timeSeries) = @_; +sub createRateHistogram { + my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_; # stages @@ -184,14 +172,6 @@ } - if ($histogram) { - - my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0); - $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval); - } - if ($timeSeries) { - - my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0); - $self->plotTimeSeries(\%gnuplotFiles, $outputFile, $label, $beginTime, $endTime, $maxX, $minX, $timeDiff, "", "Exposures processed per $interval"); - } + my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0); + $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval); # now delete temp dat files @@ -250,5 +230,5 @@ ########################################################################### sub createTimeSeries { - my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_; + my ($self, $label, $selectedStage, $beginTime, $endTime, $linear, $log, $rate) = @_; my $minX = 999999999; @@ -261,7 +241,12 @@ else {$stages = ["$selectedStage"]}; - my %gnuplotFiles; + my %linDataFiles; + my %logDataFiles; + my %ratDataFiles; my $stage = undef; - my $gnuplotFile = undef; + my $linDataFile = undef; + my $logDataFile = undef; + my $ratDataFile = undef; + foreach $stage (@{$stages}) { @@ -274,14 +259,15 @@ \$maxX, \$timeDiff, - \$gnuplotFile, - $isLog, - $showCleanup, - $deriv)) { - - $gnuplotFiles{$stage} = $gnuplotFile; + \$linDataFile, + \$logDataFile, + \$ratDataFile)) { + + $linDataFiles{$stage} = $linDataFile; + $logDataFiles{$stage} = $logDataFile; + $ratDataFiles{$stage} = $ratDataFile; } } - my $numOfPlots = keys(%gnuplotFiles); + my $numOfPlots = keys(%linDataFiles); if ($numOfPlots == 0 ) { @@ -297,21 +283,55 @@ } - my $outputFile = createImageFileName($self, $label, $selectedStage, "t", $isLog); - my $yTitle; - if ($isLog) {$yTitle = "Log(Exposures)";} - elsif ($deriv) {$yTitle = "dExposures/dTime(secs)";} - else {$yTitle = "Exposures";} - $self->plotTimeSeries( - \%gnuplotFiles, - $outputFile, - $label, - $beginTime, - $endTime, - $maxX, - $minX, - $timeDiff, - "", - $yTitle); -} + if ($linear) { + + my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 0); + + $self->plotTimeSeries( + \%linDataFiles, + $outputFile, + $label, + $beginTime, + $endTime, + $maxX, + $minX, + $timeDiff, + "", + "Exposures"); + } + + if ($log) { + + my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 1); + + $self->plotTimeSeries( + \%logDataFiles, + $outputFile, + $label, + $beginTime, + $endTime, + $maxX, + $minX, + $timeDiff, + "", + "Log(Exposures)"); + } + + if ($rate) { + + my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0); + + $self->plotTimeSeries( + \%ratDataFiles, + $outputFile, + $label, + $beginTime, + $endTime, + $maxX, + $minX, + $timeDiff, + "", + "Exposures processed per hour"); +} +} ########################################################################### @@ -468,8 +488,4 @@ my $timeFormat = undef; my $divX = undef; - #my $yTitle = undef; - #if ($isLog) {$yTitle = "Log( Exposures )";} - #elsif ($isDeriv) {$yTitle = "dExposures/dTime(secs)";} - #else {$yTitle = "Exposures";} $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); @@ -478,6 +494,4 @@ # sort out plot title - #my $title = ""; - #if ($isDeriv) {$title .= "First derivatives of "} if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}} else {$title .= "'all stages'"}