Index: /trunk/tools/who_uses_the_cluster/who_uses_the_cluster.sh =================================================================== --- /trunk/tools/who_uses_the_cluster/who_uses_the_cluster.sh (revision 29090) +++ /trunk/tools/who_uses_the_cluster/who_uses_the_cluster.sh (revision 29090) @@ -0,0 +1,52 @@ +#!/bin/bash + +# +# This script is supposed to be run as ipp user (on ipp004) +# +# for each computer of the cluster +# get the result of the command 'top -d 3 -b -n 5 -i' +# i.e.: +# - show active processes (-i) +# - 5 times (-n 5) +# - run in batch mode (-b) +# - every 3 seconds (-d 3) +# +# If nothing is changed in this script, data can be analyzed using +# analyze_logs.pl +# + +# +# TODO: +# - make it parametered? +# - add an option for ssh timeout +# - add it to crontab +# - split hosts by pantasks server? +# + +# The hosts list on which we're sshing +HOSTS="ippdb00 ippdb01 ippdb02\ + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09\ + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19\ + ipp004 ipp005 ipp006 ipp007 ipp008 ipp009\ + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019\ + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029\ + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039\ + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049\ + ipp050 ipp051 ipp052 ipp053" + +# Basename of the files +LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_" + +# Run +for computer in $HOSTS; do + LOGFILE=$LOGFILE_BASE$computer + # Delete existing logfile + echo "" > $LOGFILE + echo "########### $computer ###########" + echo "########### $computer ###########" >> $LOGFILE + /usr/bin/ssh $computer 'setenv TERM xterm; top -d 3 -b -n 5 -i' >> $LOGFILE 2> /dev/null & +done + +# +# We should wait 15 to 20 seconds to get the results. The perl script uses any input file though. +# Index: /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.pl =================================================================== --- /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.pl (revision 29090) +++ /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.pl (revision 29090) @@ -0,0 +1,127 @@ +#!/usr/local/bin/perl -w + +# +# This script analyzes the results of the who_uses_the_system command +# + +use File::stat; + +# Basename for log files +$LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_"; +# Hosts on which the 'who_uses_the_system' command was run +@HOSTS=qw/ippdb00 ippdb01 ippdb02 + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09 + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19 + ipp004 ipp005 ipp006 ipp007 ipp008 ipp009 + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019 + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039 + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049 + ipp050 ipp051 ipp052 ipp053/; + +print "\n"; +print "Who uses the system?\n"; +print "\n"; +print "To get info, move the mouse pointer over the table cell


\n"; +print "\n"; +# Scan each logfile +foreach $computer (@HOSTS) { + $LOGFILE=$LOGFILE_BASE.$computer; + $canopen = open LOG, "<$LOGFILE"; + if ($canopen) { # The logfile exists (host is active) + # The summary table is indexed by the pid. for each pid, we + # store the average cpu/memory load, the process name, the + # user name + %summary = (); + for $line () { + chomp $line; + if ( !($line =~ /.*top.*/) && ($line =~ /^ ?[1-9].*/) ) { + # We don't want the 'top' process to be shown + $line =~ s/\s+/;/g; + $line =~ s/^;//g; + @values = split ';', $line; + $pid = $values[0]; + $username = $values[1]; + $proc =$values[8]; + $mem =$values[9]; + $procname = $values[11]; + if (!defined $summary{$pid}) { + # The pid is a new one + $summary{$pid}{"count"} = 1; + $summary{$pid}{"username"} = $username; + $summary{$pid}{"proc"} = $proc; + $summary{$pid}{"memory"} = $mem; + $summary{$pid}{"procname"} = $procname; + } else { + # The pid is a new onehas already been observed + if ($summary{$pid}{"procname"} eq $procname) { + $before = $summary{$pid}{"count"}; + $summary{$pid}{"count"}++; + $after = $summary{$pid}{"count"}; + $ratio = $before / $after; + $summary{$pid}{"proc"} = $ratio * + $summary{$pid}{"proc"} + $proc / $after; + $summary{$pid}{"memory"} = $ratio * + $summary{$pid}{"memory"} + $proc / $after; + } + } + } + } + close LOG; + # When was the logfile produced? mtime is modification time + $stats = stat($LOGFILE); + # Show results + printf("\n", + $computer, $LOGFILE, scalar localtime $stats->mtime); + $onceShown = 0; + while ( my ($pid, $value) = each(%summary) ) { + if ($value->{"count"} > 1) { + $onceShown = 1; + printf("\n"); + if ($value->{"username"} eq "ipp" or $value->{"username"} eq "apache" or $value->{"username"} eq "root") { + $color = "black"; + } else { + # The user us not a regular one... Show it in red. + $color = "red"; + } + $comment = "Whom the process belongs to"; + printf("\n", $value->{"username"}); + $comment = "Process name"; + printf("\n", $value->{"procname"}); + if ($value->{"proc"} > 100.) { + $color = "red"; + } elsif ($value->{"proc"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average cpu load"; + printf("\n", $value->{"proc"}); + if ($value->{"memory"} > 100.) { + $color = "red"; + } elsif ($value->{"memory"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average memory load"; + printf("\n", $value->{"memory"}); + $comment = "How many times the process was observed every 3 seconds during 15 seconds"; + printf("\n", $value->{"count"}); + $comment = "Process pid"; + printf("\n", $pid); + printf("\n"); + } + } + if ($onceShown == 0) { + print "\n"; + } + } else { + print "\n"; + } +} + +print "
%sFrom %s (%s)
%s%s%.1f%.1f%s%s
Nothing interesting for this one
$computerCan't open $LOGFILE
\n"; +print "\n"; +print "\n"; + Index: /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh =================================================================== --- /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh (revision 29090) +++ /trunk/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh (revision 29090) @@ -0,0 +1,7 @@ +#!/bin/bash + +INSTALLDIR=/home/panstarrs/ipp/bg_tasks +$INSTALLDIR/who_uses_the_cluster.bash +sleep 30 +/bin/rm -f ~/htdocs/ippMonitor/sc/sc.html +$INSTALLDIR/who_uses_the_cluster_analyze.pl > ~/htdocs/ippMonitor/sc/sc.html