Index: branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in (revision 32851) @@ -60,4 +60,5 @@ $(DESTWWW)/czartool_getplot.php \ $(DESTWWW)/czartool_exposures.php \ +$(DESTWWW)/czartool_ipptopsps.php \ $(DESTWWW)/histogram.php \ $(DESTWWW)/show_and_delete_image.php \ Index: branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl (revision 32851) @@ -27,7 +27,7 @@ my $nebulous = undef; my $savingToFile = undef; -my $analysis = undef; my $exposureId = undef; my $log = undef; +my $psps = undef; GetOptions ( @@ -45,8 +45,8 @@ "nebulous|n" => \$nebulous, "rate|r" => \$rate, - "analysis|a" => \$analysis, "timeseries|t" => \$timeSeries, "verbose|v" => \$verbose, "log|g" => \$log, + "psps|u" => \$psps, ); @@ -58,7 +58,4 @@ print "* UNKNKOWN: option @ARGV\n"; } -if ($analysis && !$stage) { - $quit = 1; - print "* REQUIRED: choose a stage for analsis -s (default=none)\n";} if (!$histogram) { print "* OPTIONAL: plot histogram -h (default=off)\n";} @@ -71,7 +68,4 @@ if (!$rate) { print "* OPTIONAL: plot histogram of rate -r (default=off)\n";} -if (!$analysis) { - $analysis = 0; - print "* OPTIONAL: include analysis -a (default=$analysis)\n";} if (!$log) { $log = 0; @@ -97,4 +91,6 @@ print "* OPTIONAL: choose an output location -o (default=none)\n"; } +if (!$psps) { + print "* OPTIONAL: plot ippToPsps stages -u (default=off)\n";} print "*\n*******************************************************************************\n"; @@ -104,5 +100,5 @@ # default values -if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;} +if (!$rate && !$magicMask && !$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1;} if (!$verbose) {$verbose = 0;} if (!$save_temps) {$save_temps = 0;} @@ -123,4 +119,6 @@ $plotter = czartool::Plotter->new_display($config, $save_temps); } + +if ($psps) {$plotter->setIppToPspsMode();} # if a single day has been chosen @@ -177,7 +175,2 @@ } -if($analysis) { - - my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end); - if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();} -} Index: branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl (revision 32851) @@ -37,6 +37,28 @@ -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); - +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); timePoll($period); @@ -244,5 +266,29 @@ print "* Generating plots\n"; + # get ippToPsps labels + my $ippToPspsLabels = undef; + $czarDb->getCurrentIppToPspsLabels($begin, $end, \$ippToPspsLabels); + $plotter->setIppToPspsMode(); + + # create ippToPsps labels + foreach $stage (@ippToPspsStages) { + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + chomp($label); + $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1); + } + } + + # create plots for each label for all stages for ippToPsps + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); + } + + # create plots for each label for each stage + $plotter->setStandardMode(); foreach $stage (@stages) { foreach $row ( @{$rows} ) { Index: branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm (revision 32851) @@ -8,5 +8,28 @@ use File::Temp; -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); # TODO put elsewhere +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); # TODO put elsewhere + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); use base 'czartool::MySQLDb'; @@ -33,5 +56,28 @@ ########################################################################### # -# Gets current_labels table +# Gets current ippToPsps labels +# +########################################################################### +sub getCurrentIppToPspsLabels { + my ($self, $begin, $end, $labels) = @_; + + my $query = $self->{_db}->prepare(<= '$begin' + AND timestamp <= '$end' +SQL + + if (!$query->execute) {return 0;} + + ${$labels} = $query->fetchall_arrayref(); + + return 1; +} + +########################################################################### +# +# Gets current labels from the current_labels table # ########################################################################### @@ -672,4 +718,6 @@ # times $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); + if ($timeSep == 0) {$timeSep = 1;}; # HACK to prevent divide-by-zero + $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp); $formattedMiddleTimestamp = $self->getFormattedDate($middleTime); @@ -681,5 +729,5 @@ # added, and store value to subtract from linear value below. If > 1000 # processed per hour, then it must be wrong - if (($thisProcessed - $lastProcessed)/(($timeSep/3600)+1) > 1000) { + if (($thisProcessed - $lastProcessed)/($timeSep/3600) > 1000) { $positiveJump = $thisProcessed - $lastProcessed; } @@ -701,7 +749,7 @@ # calculate first derivative in units of images per hour - $derivPending = $diffPending/($timeSep/3600 +1); - $derivFaults = $diffFaults/($timeSep/3600 +1); - $derivProcessed = $diffProcessed/($timeSep/3600 +1); + $derivPending = $diffPending/($timeSep/3600); + $derivFaults = $diffFaults/($timeSep/3600); + $derivProcessed = $diffProcessed/($timeSep/3600); } @@ -1107,5 +1155,5 @@ my $stage = undef; $totalDeleted = 0; - foreach $stage (@stages) { + foreach $stage (@allStages) { if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;} @@ -1213,5 +1261,5 @@ my $stage = undef; - foreach $stage (@stages) { + foreach $stage (@allStages) { $self->optimizeTable($stage); Index: branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm (revision 32851) @@ -10,4 +10,5 @@ "addstarlap", "cleanup", + "deepstack", "detrend", "distribution", Index: branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm (revision 32851) @@ -10,5 +10,27 @@ use czartool::Config; -my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); +my @allStages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @allIppToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + ########################################################################### @@ -75,4 +97,26 @@ ########################################################################### # +# Sets plotter to use standard IPP mode, i.e. chip, cam etc +# +########################################################################### +sub setStandardMode { + my ($self) = @_; + + $self->{allStages} = \@allStages; +} + +########################################################################### +# +# Sets plotter to use ippToPsps mode, i.e. processed, loaded_to_datastore etc +# +########################################################################### +sub setIppToPspsMode { + my ($self) = @_; + + $self->{allStages} = \@allIppToPspsStages; +} + +########################################################################### +# # Some initialisation code common to all constructors # @@ -83,4 +127,5 @@ $self->{_gpc1Db} = $self->{_config}->getGpc1Instance(); $self->{_czarDb} = $self->{_config}->getCzarDbInstance(); + $self->setStandardMode(); } @@ -140,5 +185,5 @@ my $stages = undef; if (!$selectedStage) { - $stages = \@allStages; + $stages = \@{$self->{allStages}}; $selectedStage = 'all_stages'; } @@ -203,10 +248,10 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures processed per $interval\";" . + "set ylabel \"Items processed per $interval\";" . "set boxwidth 0.75;" . "plot "; my $first = 1; - foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { @@ -226,5 +271,5 @@ ########################################################################### # -# Plots a time series for all stages for this label +# Plots a time series for all or one stage for this label # ########################################################################### @@ -238,5 +283,5 @@ my $stages = undef; - if (!$selectedStage) {$stages = \@allStages;} + if (!$selectedStage) {$stages = \@{$self->{allStages}};} else {$stages = ["$selectedStage"]}; @@ -297,5 +342,5 @@ $timeDiff, "", - "Exposures"); + "Items"); } @@ -314,5 +359,5 @@ $timeDiff, "", - "Log(Exposures)"); + "Log(Items)"); } @@ -331,5 +376,5 @@ $timeDiff, "", - "Exposures per hour"); + "Items per hour"); } } @@ -351,5 +396,5 @@ my $stage = undef; my $pendingMinusFaults = undef; - foreach $stage (@allStages) { + foreach $stage (@{$self->{allStages}}) { if (!$self->{_czarDb}->countProcessedPendingAndFaults( @@ -381,5 +426,5 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures\";" . + "set ylabel \"Items\";" . "set boxwidth 0.75;" . "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . @@ -520,5 +565,6 @@ my $firstIn = 1; # loop through stages array so that they are ordered properly (not maintained in hash) - foreach my $stage (@allStages) { + #foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { if(!$gnuplotFiles->{$stage}) {next;} @@ -620,4 +666,5 @@ "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . "set style fill solid 1.00 border -1;" . + "set key center top;" . "set style histogram rowstacked;" . "set style data histograms;" . Index: branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 32851) +++ branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 32851) @@ -0,0 +1,405 @@ + 0) $selectedLabel = $labels[0]; + else $selectedLabel = "no labels"; +} +if ($selectedStage == "") { $selectedStage = "all_stages"; } +if ($plotType == "") { $plotType = "linear"; } + +// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later +if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode); + +// tell selected server to stop or run +if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer, $serverCmd); + +// tell all servers to stop or run +if ($allServerCmd) { + + foreach ($servers as &$server) serverStopRun($czardb, $server, $allServerCmd); +} + + +$debug = 0; +$table = "\n"; + + +if ($debug) { + echo "prog returned $status, and output:
"; + for ($i = 0; $i < count($distLabels); $i++) { + echo "$distLabels[$i]
"; + } +} + +$isUpdate = ($selectedMode == "update"); + +// set up the form +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +echo "
"; + +echo ""; +echo ""; +echo ""; + +echo ""; +echo ""; +echo "\n"; +echo ""; +echo ""; + +echo "
"; +echo "

Czartool

"; +echo "

IPP to PSPS Monitoring

"; +echo "
"; + # time series plot +echo "
"; + echo "
"; + # rate time series plot +echo "
"; + echo "
"; +echo "		"; +echo ""; + +#echo "
"; + +// status table at top +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; +echo "
"; +echo $table; +createTableTitle("Status and links", 3); +echo "
Time of last update$lastUpdateTime
Documentationhere
\n"; + +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; + + + +echo "
"; + + createLabelsTable($pass, + $proj, + $czardb, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + "new", + $selectedLabel, + $selectedStage, + $plotType); + +echo "
"; + +echo ""; + +menu_end(); + +########################################################################### +# +# Gets labels in use over the last 24 hours +# +########################################################################### +function getLabels($db) { + + # order by descending priority as set in gpc1 database + $sql = "SELECT DISTINCT label FROM processed WHERE timestamp >= now() - INTERVAL 1 DAY ORDER BY label"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + while ($qry->fetchInto($row)) { + + $labels[] = $row[0]; + } + + return $labels; +} + +########################################################################### +# +# Creates table for all labels showing all stages for given 'state' +# +########################################################################### +function createLabelsTable( + $pass, + $proj, + $db, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + $selectedState, + $selectedLabel, $selectedStage, $plotType) { + + $isUpdate = ($selectedMode == "update"); + global $headerColor; + global $columnHeaderColor; + + // setup table + global $table; + echo $table; + createTableTitle("Label status", 13); + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=".$selectedMode + ."&label=".$selectedLabel + ."&stage=all_stages" + ."&plottype=".$plotType; + createFormattedTableCell(($selectedStage != "all_stages"), $link, "All stages", 0, "$columnHeaderColor"); + + #echo "Label (in order of priority)"; + foreach ($stages as &$stage) { + + if ($stage == $selectedStage) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$stage + ."&plottype=".$plotType; + + createFormattedTableCell($showLink, $link, $stage, 0, "$columnHeaderColor"); +# createTableColumnHeader("Dates"); + } + + // setup default link + $defaultlink = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + + $searchState = "new"; + if ($selectedMode == "update") $searchState = "update"; + + // write rows + foreach ($labels as &$thisLabel) { + + + $distributing = false; + $publishing = false; + + // show distribution/publishing status + echo "\n"; + + // create link to label summary page for each label + if ($thisLabel == $selectedLabel) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$thisLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + createFormattedTableCell($showLink, $link, $thisLabel, 0, null); + + $str = ""; + $anyFaults = false; + + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "processed", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_ODM", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merge_worthy", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_dxlayer", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merged", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_local", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + echo "\n"; + } + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + if ($selectedLabel == "all") $showLink = 0; + else $showLink = 1; + $link = "czartool_ipptopsps.php?pass=".$pass."&proj=".$proj."&mode=".$selectedMode."&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + echo "\n"; +} + + +########################################################################### +# +# Creates a table title +# +########################################################################### +function createTableTitle($str, $colSpan) { + + global $headerColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates a table header cell +# +########################################################################### +function createTableColumnHeader($str) { + + global $columnHeaderColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates table cell with a red background if an error +# +########################################################################### +function createFormattedTableCell($showLink, $link, $str, $error, $color) { + + $fontColor = $error ? "black" : "blue"; + + if ($error) echo ""; + else if ($color) echo ""; + else echo ""; + if ($showLink) echo "$str"; + else if ($str) echo "$str"; + else echo " "; +} + + +########################################################################### +# +# Returns state and fault-count (if new) as a string +# +########################################################################### +function getStateAndFaults($db, $label, $state, $stage, &$str, &$anyFaults) { + + $anyFaults = false; + $sql = "SELECT pending, faults FROM $stage WHERE label LIKE '$label' ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + $pending = $row[0]; + $faults = $row[1]; + + if ($pending == 0) + $str = ""; + else + $str = "$pending"; + + if ($state == "new") { + + if ($faults > 0) { + + $str = $str."(".$faults.")"; + $anyFaults = true; + } + } +} + +########################################################################### +# +# Returns time of last czarDb update +# +########################################################################### +function getLastUpdateTime($db) { + + $anyFaults = false; + $sql = "SELECT timestamp FROM processed ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + return $row[0]; +} + + +?> + Index: branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 32851) @@ -27,6 +27,17 @@ menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); -$states=array("full","new","drop","wait"); -$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist", "pub"); +$stages=array( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + $servers=array( "addstar", @@ -150,5 +161,5 @@ echo "
"; echo $table; -#"\n"; + createTableTitle("Status and links", 3); echo "\n"; @@ -180,5 +191,4 @@ $pubLabels, $stages, - $states, "new", $selectedLabel, @@ -349,5 +359,4 @@ $pubLabels, $stages, - $states, $selectedState, $selectedLabel, $selectedStage, $plotType) { Index: branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat (revision 32758) +++ branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat (revision 32851) @@ -21,5 +21,6 @@ menutop | menutop | link | simple plot - chip (to remove?) | simplePlot.php menutop | menutop | link | simple plot - cam (to remove?) | simplePlotcam.php -menutop | menutop | link | czartool | czartool_labels.php +menutop | menutop | link | czartool - standard | czartool_labels.php +menutop | menutop | link | czartool - ipptopsps | czartool_ipptopsps.php menutop | menutop | link | mask stats | maskStats.php