Index: trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster.sh =================================================================== --- trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster.sh (revision 34323) +++ trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster.sh (revision 34323) @@ -0,0 +1,63 @@ +#!/bin/bash -f + +# +# This script is supposed to be run as ipp user (on ipp004) +# +# for each computer of the cluster +# get the result of the command 'top -d 3 -b -n 5 -i' +# i.e.: +# - show active processes (-i) +# - 5 times (-n 5) +# - run in batch mode (-b) +# - every 3 seconds (-d 3) +# +# If nothing is changed in this script, data can be analyzed using +# analyze_logs.pl +# + +# +# TODO: +# - make it parametered? +# - add an option for ssh timeout +# - add it to crontab +# - split hosts by pantasks server? +# + +# The hosts list on which we're sshing +HOSTS="ippdb00 ippdb01 ippdb02 ippdb03\ + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09\ + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19\ + ippc20 ippc21 ippc22 ippc23 ippc24 ippc25 ippc26 ippc27 ippc28 ippc29\ + ippc30 ippc31 ippc32 ippc33 ippc34 ippc35 ippc36 ippc37 ippc38 ippc39\ + ippc40 ippc41 ippc42 ippc43 ippc44 ippc45 ippc46 ippc47 ippc48 ippc49\ + ippc50 ippc51 ippc52 ippc53 ippc54 ippc55 ippc56 ippc57 ippc58 ippc59\ + ippc60 ippc61 ippc62 ippc63\ + ipp004 ipp005 ipp006 ipp007 ipp008 ipp009\ + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019\ + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029\ + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039\ + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049\ + ipp050 ipp051 ipp052 ipp053 ipp054 ipp055 ipp056 ipp057 ipp058 ipp059\ + ipp060 ipp061 ipp062 ipp063 ipp064 ipp065 ipp066\ + stare00 stare01 stare02 stare03 stare04\ + ippb00 ippb01 ippb02 ippb03\ + stsci00 stsci01 stsci02 stsci03 stsci04\ + stsci05 stsci06 stsci07 stsci08 stsci09" + +# Basename of the files +LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_" + +# Run +for computer in $HOSTS; do + LOGFILE=$LOGFILE_BASE$computer + # Delete existing logfile + echo "" > $LOGFILE + echo "########### $computer ###########" + echo "########### $computer ###########" >> $LOGFILE +# /usr/bin/ssh $computer 'setenv TERM xterm; top -d 3 -b -n 5 -i -c' >> $LOGFILE 2> /dev/null & + /usr/bin/ssh $computer 'setenv TERM xterm; setenv COLUMNS 1000; top -d 3 -b -n 5 -c -i' >> $LOGFILE 2> /dev/null & +done + +# +# We should wait 15 to 20 seconds to get the results. The perl script uses any input file though. +# Index: trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster_analyze.pl =================================================================== --- trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster_analyze.pl (revision 34323) +++ trunk/tools/who_is_using_the_cluster/_who_uses_the_cluster_analyze.pl (revision 34323) @@ -0,0 +1,147 @@ +#!/usr/local/bin/perl -w + +# +# This script analyzes the results of the who_uses_the_system command +# + +use File::stat; + +# Basename for log files +$LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_"; +# Hosts on which the 'who_uses_the_system' command was run +@HOSTS=qw/ipp004 ipp005 ipp006 ipp007 ipp008 ipp009 + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019 + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039 + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049 + ipp050 ipp051 ipp052 ipp053 ipp054 ipp055 ipp056 ipp057 ipp058 ipp059 + ipp060 ipp061 ipp062 ipp063 ipp064 ipp065 ipp066 + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09 + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19 + ippc20 ippc21 ippc22 ippc23 ippc24 ippc25 ippc26 ippc27 ippc28 ippc29 + ippc30 ippc31 ippc32 ippc33 ippc34 ippc35 ippc36 ippc37 ippc38 ippc39 + ippc40 ippc41 ippc42 ippc43 ippc44 ippc45 ippc46 ippc47 ippc48 ippc49 + ippc50 ippc51 ippc52 ippc53 ippc54 ippc55 ippc56 ippc57 ippc58 ippc59 + ippc60 ippc61 ippc62 ippc63 + ippdb00 ippdb01 ippdb02 ippdb03 stare00 stare01 stare02 stare03 stare04 + ippb00 ippb01 ippb02 ippb03 + stsci00 stsci01 stsci02 stsci03 stsci04 + stsci05 stsci06 stsci07 stsci08 stsci09/; + +print "\n"; +print "Who is using the system?\n"; +print "\n"; +print "This script is run from ippc18 (ipp user crontab). To get info, move the mouse pointer over the table cells to get more details. !Warning! Command lines can be truncated


\n"; +print "\n"; +# Scan each logfile +foreach $computer (@HOSTS) { + $LOGFILE=$LOGFILE_BASE.$computer; + $canopen = open LOG, "<$LOGFILE"; + if ($canopen) { # The logfile exists (host is active) + # The summary table is indexed by the pid. for each pid, we + # store the average cpu/memory load, the process name, the + # user name + %summary = (); + for $line () { + chomp $line; + if ( !($line =~ /.*top.*/) && ($line =~ /^ ?[1-9].*/) ) { + # We don't want the 'top' process to be shown + $line =~ s/\s+/;/g; + $line =~ s/^;//g; + # print "$line\n"; + @values = split ';', $line; + $pid = $values[0]; + #print $pid, "\n"; + $username = $values[1]; + $proc =$values[8]; + $mem =$values[9]; + $size = scalar(@values); + #$procname = $values[11]; + $fullprocname = ""; + #print "$procname \n"; + for ($i=11;$i<$size;$i++) { + $fullprocname = $fullprocname.$values[$i]." "; + } + if (!defined $summary{$pid}) { + # The pid is a new one + $summary{$pid}{"count"} = 1; + $summary{$pid}{"username"} = $username; + $summary{$pid}{"proc"} = $proc; + $summary{$pid}{"memory"} = $mem; + $summary{$pid}{"procname"} = $fullprocname; + } else { + # The pid has already been observed + # print "The pid has already been observed\n"; + #if ($summary{$pid}{"procname"} eq $procname) { + $before = $summary{$pid}{"count"}; + $summary{$pid}{"count"}++; + $after = $summary{$pid}{"count"}; + $ratio = $before / $after; + $summary{$pid}{"proc"} = $ratio * + $summary{$pid}{"proc"} + $proc / $after; + $summary{$pid}{"memory"} = $ratio * + $summary{$pid}{"memory"} + $proc / $after; + #} + } + } + } + close LOG; + # When was the logfile produced? mtime is modification time + $stats = stat($LOGFILE); + # Show results + printf("\n", + $LOGFILE, scalar localtime $stats->mtime, $computer); + $onceShown = 0; + while ( my ($pid, $value) = each(%summary) ) { + # print $value->{"count"} , " ", $pid, "\n"; + if ($value->{"count"} > 1) { + #print "Ok\n"; + $onceShown = 1; + printf("\n"); + if ($value->{"username"} eq "ipp" or $value->{"username"} eq "apache" or $value->{"username"} eq "root") { + $color = "black"; + } else { + # The user us not a regular one... Show it in red. + $color = "red"; + } + $comment = "Whom the process belongs to"; + printf("\n", $value->{"username"}); + $comment = "Process name"; + printf("\n", $value->{"procname"}); + if ($value->{"proc"} > 100.) { + $color = "red"; + } elsif ($value->{"proc"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average cpu load"; + printf("\n", $value->{"proc"}); + if ($value->{"memory"} > 100.) { + $color = "red"; + } elsif ($value->{"memory"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average memory load"; + printf("\n", $value->{"memory"}); + $comment = "How many times the process was observed every 3 seconds during 15 seconds"; + printf("\n", $value->{"count"}); + $comment = "Process pid"; + printf("\n", $pid); + printf("\n"); + } + } + if ($onceShown == 0) { + print "\n"; + } + } else { + print "\n"; + } +} + +print "
%s
%s%s%.1f%.1f%s%s
Nothing interesting for this one
$computerCan't open $LOGFILE
\n"; +print "\n"; +print "\n"; + Index: trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh =================================================================== --- trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh (revision 34323) +++ trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh (revision 34323) @@ -0,0 +1,12 @@ +#!/bin/bash + +INSTALLDIR=/home/panstarrs/ipp/who_uses_the_cluster +$INSTALLDIR/_who_uses_the_cluster.sh +echo "Sleeping 17s" +sleep 17 +CLUSTERMONITORDIR=~/htdocs/clusterMonitor +mkdir -p $CLUSTERMONITORDIR +/bin/rm -f $CLUSTERMONITORDIR/top.html +#/bin/mv -f $CLUSTERMONITORDIR/top.html $CLUSTERMONITORDIR/top_`date +"%Y%m%dT%H%M"`.html +$INSTALLDIR/_who_uses_the_cluster_analyze.pl > $CLUSTERMONITORDIR/top.html + Index: trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh.record =================================================================== --- trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh.record (revision 34323) +++ trunk/tools/who_is_using_the_cluster/who_uses_the_cluster_analyze.sh.record (revision 34323) @@ -0,0 +1,13 @@ +#!/bin/bash + +INSTALLDIR=/home/panstarrs/ipp/who_uses_the_cluster +$INSTALLDIR/_who_uses_the_cluster.sh +echo "Sleeping 17s" +sleep 17 +CLUSTERMONITORDIR=~/htdocs/clusterMonitor +mkdir -p $CLUSTERMONITORDIR +WHEN=`date +"%s"` +/bin/cp -f $CLUSTERMONITORDIR/top.html /home/panstarrs/ipp/cluster_use/top.$WHEN.html +/bin/gzip /home/panstarrs/ipp/cluster_use/top.$WHEN.html +/bin/rm -f $CLUSTERMONITORDIR/top.html +$INSTALLDIR/_who_uses_the_cluster_analyze.pl > $CLUSTERMONITORDIR/top.html