Index: trunk/tools/who_is_using_the_cluster/get_processes_information.py
===================================================================
--- trunk/tools/who_is_using_the_cluster/get_processes_information.py	(revision 34324)
+++ trunk/tools/who_is_using_the_cluster/get_processes_information.py	(revision 34324)
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+
+import psutil
+import os
+import pwd
+import time
+import datetime
+import subprocess
+import re
+import socket
+
+class Callable:
+    """
+    See http://code.activestate.com/recipes/52304-static-methods-aka-class-methods-in-python/
+    """
+    def __init__(self, anycallable):
+        self.__call__ = anycallable
+
+class Users:
+    """
+    Class where some information related to users is cached. More a
+    container with convenient method actually.
+    """
+    ignore = ['root', 'apache', 'bin', 'nobody', 'nagios', 'mysql', ]
+    expected = ['ipp', 'ippdor', 'ippdvo', 'iqanalysis', ]
+    always = ['ipp', 'ippdvo', ]
+    cmdline_ignore_re_progs_from_pattern = [ re.compile('.*condor_.*'), 
+                                             re.compile('.*get_processes_information.py.*'),
+                                             ]
+    singleton = dict() # index is the (numeric) uid
+    def get_user_name(uid):
+        try:
+            return Users.singleton[uid]
+        except KeyError:
+            Users.singleton[uid] = pwd.getpwuid(uid).pw_name
+            return Users.singleton[uid]
+    get_user_name = Callable(get_user_name)
+    max_age_in_seconds = 6*3600
+    min_cpu = 5.
+    min_memory = 5.
+
+class SmallProcess:
+    def __init__(self, pid, username, cmdline, cpu_percent, memory_percent, age, count = 1):
+        self.pid = pid
+        self.cmdline = cmdline
+        self.username = username
+        self.cpu_percent = cpu_percent
+        self.memory_percent = memory_percent
+        self.age = age
+        self.count = count
+
+class Processes:
+    processes = dict()
+    def add_process(process):
+        """Append the information relative to a process to the class
+        'processes' container"""
+        try:
+            Processes.processes[process.pid].append(process)
+        except KeyError:
+            Processes.processes[process.pid] = []
+            Processes.processes[process.pid].append(process)
+    add_process = Callable(add_process)
+    def statistics():
+        data = []
+        for key in sorted(Processes.processes.iterkeys()):
+            item = Processes.processes[key]
+            count = 0
+            avg_cpu_perc = 0.
+            avg_mem_perc = 0.
+            for sprocess in item:
+                avg_cpu_perc += sprocess.cpu_percent
+                avg_mem_perc += sprocess.memory_percent
+                count += 1.
+            if count > 1:
+                sp = SmallProcess(sprocess.pid, 
+                                  sprocess.username, 
+                                  sprocess.cmdline,
+                                  avg_cpu_perc / count,
+                                  avg_mem_perc / count,
+                                  sprocess.age,
+                                  count)
+                data.append(sp)
+        return data
+    statistics = Callable(statistics)
+
+class ProcessesSnapshot:
+    """
+    The container of all processes
+    """
+    def __init__(self, min_cpu = Users.min_cpu, min_memory = Users.min_memory, max_age_in_seconds = Users.max_age_in_seconds):
+        processes = dict() # index is the pid
+        ages = get_correct_processes_age()
+        for process in psutil.process_iter():
+            try:
+                age = ages[process.pid]
+            except KeyError:
+                age = 0 # Process died while we were playing...
+            try:
+                sp = SmallProcess(process.pid, 
+                                  Users.get_user_name(process.uids.real), 
+                                  process.cmdline,
+                                  process.get_cpu_percent(interval = 0), 
+                                  process.get_memory_percent(),
+                                  age)
+                add_process = False
+                if sp.cpu_percent > min_cpu or sp.memory_percent > min_memory:
+                    add_process = True
+                if sp.age > max_age_in_seconds and sp.username not in Users.ignore:
+                    add_process = True
+                for prog in Users.cmdline_ignore_re_progs_from_pattern:
+                    if prog.match(" ".join(sp.cmdline)) is not None:
+                        add_process = False
+                if sp.username in Users.always:
+                    add_process = True
+                if add_process:
+                    Processes.add_process(sp)
+            except psutil.error.NoSuchProcess:
+                pass
+
+def get_correct_processes_age():
+    p = subprocess.Popen('ps -eo pid,etime'.split(' '),
+                         stderr = subprocess.PIPE,
+                         stdout = subprocess.PIPE)
+    p.poll()
+    ages = dict()
+    for complex_line in p.stdout:
+        line = re.sub('[ \t][ \t]*', ' ', complex_line[:-1])
+        line = re.sub('^ ', '', line)
+        if line.startswith('PID'):
+            continue
+        (pid, etime) = line.split(' ')
+        if '-' in etime:
+            (days, etime) = etime.split('-')
+        else:
+            days = 0
+        age = int(days)*24*60*60
+        fields = etime.split(':')
+        if len(fields) == 3:
+            (hours, minutes, seconds) = fields
+            age += int(seconds) + 60 * (int(minutes) + 60 * int(hours))
+        elif len(fields) == 2:
+            (minutes, seconds) = fields
+            age += int(seconds) + 60 * int(minutes)
+        elif len(fields) == 2:
+            seconds = fields
+            age += int(seconds)
+        ages[int(pid)] = age
+    return ages
+
+if __name__ == '__main__':
+    nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0]
+    count = 0
+    time_interval = .5
+    iterations = 5
+    while count < iterations:
+        ProcessesSnapshot()
+        count += 1
+        time.sleep(time_interval)
+
+    output = open('/home/panstarrs/ipp/htdocs/clusterMonitor2/nodes/%s.html' % nodename, 'w')
+    # output.write('<html>\n')
+    # output.write('<head>\n')
+    # output.write('<meta http-Equiv="Cache-Control" Content="no-cache">\n')
+    # output.write('<meta http-Equiv="Pragma" Content="no-cache">\n')
+    # output.write('<meta http-Equiv="Expires" Content="0">\n')
+    # output.write('</head>\n')
+    # output.write('<body>\n')
+    now = datetime.datetime.now()
+    output.write('<a name="%s"><h2>%s</h2></a>\n' % (nodename, nodename))
+    output.write('Last refresh: %s / Time interval: %5.2f / #Iterations: %d<br/>\n'  % (now.isoformat().split('.')[0],
+                                                                                        time_interval,
+                                                                                        iterations));
+    output.write('<table class="processTable">\n')
+    output.write('<tr><th>pid (#samples)</th><th>owner</th><th>cpu%</th><th>mem%</th><th>age (hrs)</th><th>cmd line</th></tr>\n')
+    for sprocess in Processes.statistics():
+        output.write(' <tr>\n')
+        output.write('  <td class="pid" title="pid (#samples)">%d (%d)</td>\n' % (sprocess.pid, 
+                                                                                  sprocess.count))
+        if sprocess.username not in Users.expected:
+            alert_color = '#ff0000'
+        else:
+            alert_color = '#000000'
+        output.write('  <td class="owner" title="owner" style="color:%s;">%s</td>\n' % (alert_color, 
+                                                                                        sprocess.username))
+        if sprocess.cpu_percent >= Users.min_cpu:
+            alert_color = '#ff0000'
+        else:
+            alert_color = '#000000'
+        output.write('  <td class="cpu" title="CPU usage" style="color:%s;">%6.2f</td>\n' % (alert_color, 
+                                                                                             sprocess.cpu_percent) )
+        if sprocess.memory_percent >= Users.min_memory:
+            alert_color = '#ff0000'
+        else:
+            alert_color = '#000000'
+        output.write('  <td class="memory" title="memory usage" style="color:%s;">%6.2f</td>\n' % (alert_color, 
+                                                                                                   sprocess.memory_percent) )
+        if sprocess.age >= Users.max_age_in_seconds:
+            alert_color = '#ff0000'
+        else:
+            alert_color = '#000000'
+        output.write('  <td class="age" title="age (in hours") style="color:%s;">%10.2f</td>\n' % (alert_color, 
+                                                                                                   sprocess.age/3600.) )
+        output.write('  <td class="cmdline" title="command line">%s</td>\n' % (' '.join(sprocess.cmdline)))
+        output.write('</tr>\n')
+    output.write('</table>\n')
+    # output.write('</body>\n')
+    # output.write('</html>\n')
+    output.close()
Index: trunk/tools/who_is_using_the_cluster/html/header.html
===================================================================
--- trunk/tools/who_is_using_the_cluster/html/header.html	(revision 34324)
+++ trunk/tools/who_is_using_the_cluster/html/header.html	(revision 34324)
@@ -0,0 +1,66 @@
+<header>
+  <title>Who is using the cluster?</title>
+  <!-- From http://www.appnitro.com/forums/topic/auto-expandadjustable-iframe -->
+  <script language="JavaScript">
+    function calcHeight(my_iframe, url) {
+    //find the height of the internal page
+    var the_height = document.getElementById(my_iframe).contentWindow.document.body.scrollHeight;
+    //change the height of the iframe
+    document.getElementById(my_iframe).height = the_height;
+    }
+  </script>
+  <meta http-Equiv="Cache-Control" Content="no-cache">
+  <meta http-Equiv="Pragma" Content="no-cache">
+  <meta http-Equiv="Expires" Content="0">
+
+  <style type="text/css">
+    ul {
+    list-style-type:none;
+    margin:0;
+    padding:0;
+    }
+    li {
+    display:inline;
+    }
+    .processTable {
+    border:1px solid black;
+    border-collapse: collapse;
+    }
+    .pid {
+    border:1px solid black;
+    width:80px;
+    text-align:right;
+    vertical-align:top;
+    }
+    .owner {
+    border:1px solid black;
+    width:100px;
+    text-align:center;
+    vertical-align:top;
+    }
+    .cpu {
+    border:1px solid black;
+    width:80px;
+    text-align:right;
+    vertical-align:top;
+    }
+    .memory {
+    border:1px solid black;
+    width:80px;
+    text-align:right;
+    vertical-align:top;
+    }
+    .age {
+    border:1px solid black;
+    width:80px;
+    text-align:right;
+    vertical-align:top;
+    }
+    .cmdline {
+    border:1px solid black;
+    width:1000px;
+    vertical-align:top;
+    }
+  </style>
+</header>
+
Index: trunk/tools/who_is_using_the_cluster/html/help.html
===================================================================
--- trunk/tools/who_is_using_the_cluster/html/help.html	(revision 34324)
+++ trunk/tools/who_is_using_the_cluster/html/help.html	(revision 34324)
@@ -0,0 +1,37 @@
+This page shows how the cluster is being used.<p/>
+
+Every 5 minutes, a script running on ippc11 (as the ipp user. See its
+crontab) starts a process on each node of the cluster. There, five
+(i.e. the number of iterations) snapshots of the node activity are
+made at an interval of 0.5 second (i.e. time interval).<br/>
+
+Statistics about processes that have been <b>observed at least
+twice</b> are then computed. They are displayed if at least one of
+these conditions is satisfied (the corresponding column is then shown
+in red):
+<ul>
+  <li>The owner of the process is owned by an "unexpected" user
+  (i.e. by someone else
+  than <tt>root</tt>, <tt>nobody</tt>, <tt>iqanalysis</tt>...)</li>
+  <li>The owner of the process is <tt>ipp</tt> or <tt>ippdvo</tt>
+  (even if they are "expected" users), mainly because we want to check
+  if anything is still running while, e.g., we are changing the
+  tag.</li>
+  <li>The process used more than 5% of the total CPU on average</li>
+  <li>The process used more than 5% of the total memory on average</li>
+  <li>The process is older than 6 hours</li>
+</ul>
+
+The displayed columns are the following:
+<ul>
+  <li>The process ID (i.e. pid) with its number of observations</li>
+  <li>The process owner</li>
+  <li>The process cpu usage (a multi-threaded process may use more than 100%)</li>
+  <li>The process memory usage</li>
+  <li>The process age in hours</li>
+  <li>The process command line</li>
+</ul>
+
+Feel free to correct this documentation and to send me your suggestions and comments.<br/>
+
+Serge
Index: trunk/tools/who_is_using_the_cluster/html/index.php
===================================================================
--- trunk/tools/who_is_using_the_cluster/html/index.php	(revision 34324)
+++ trunk/tools/who_is_using_the_cluster/html/index.php	(revision 34324)
@@ -0,0 +1,70 @@
+<?php
+
+print("<html>\n");
+
+print(file_get_contents("header.html"));
+
+print("<body>\n");
+
+$NODES = array( "ipp004", "ipp005", "ipp006", "ipp007", "ipp008", "ipp009", 
+		"ipp010", "ipp011", "ipp012", "ipp013", "ipp014", "ipp015", "ipp016", "ipp017", "ipp018", "ipp019", 
+		"ipp020", "ipp021", "ipp023", "ipp024", "ipp025", "ipp026", "ipp027", "ipp028", "ipp029", 
+		"ipp030", "ipp031", "ipp032", "ipp033", "ipp034", "ipp035", "ipp036", "ipp037", "ipp038", "ipp039", 
+		"ipp040", "ipp041", "ipp042", "ipp043", "ipp044", "ipp045", "ipp046", "ipp047", "ipp048", "ipp049", 
+		"ipp050", "ipp051", "ipp052", "ipp053", "ipp054", "ipp055", "ipp056", "ipp057", "ipp058", "ipp059", 
+		"ipp060", "ipp061", "ipp062", "ipp063", "ipp064", "ipp065", "ipp066", 
+		"ippc01", "ippc02", "ippc03", "ippc04", "ippc05", "ippc06", "ippc07", "ippc08", "ippc09", 
+		"ippc10", "ippc11", "ippc12", "ippc13", "ippc14", "ippc15", "ippc16", "ippc17", "ippc18", "ippc19", 
+		"ippc20", "ippc21", "ippc22", "ippc23", "ippc24", "ippc25", "ippc26", "ippc27", "ippc28", "ippc29", 
+		"ippc30", "ippc31", "ippc32", "ippc33", "ippc34", "ippc35", "ippc36", "ippc37", "ippc38", "ippc39", 
+		"ippc40", "ippc41", "ippc42", "ippc43", "ippc44", "ippc45", "ippc46", "ippc47", "ippc48", "ippc49", 
+		"ippc50", "ippc51", "ippc52", "ippc53", "ippc54", "ippc55", "ippc56", "ippc57", "ippc58", "ippc59", 
+		"ippc60", "ippc61", "ippc62", "ippc63", 
+		"stare00", "stare01", "stare02", "stare03", "stare04", 
+		"ippdb00", "ippdb01", "ippdb02", "ippdb03", 
+		"stsci00", "stsci01", "stsci02", "stsci03", "stsci04", 
+		"stsci05", "stsci06", "stsci07", "stsci08", "stsci09",
+		);
+
+print("Run from ippc11. See ipp crontab");
+print("</p>\n");
+
+print("<table><tr><td>\n");
+print("<table>\n  <tr>\n");
+$current_index = 0;
+$previous_node_index = 10;
+foreach ($NODES as $node) {
+  $node_index = intval(substr($node, -1));
+  if ($current_index > $node_index) {
+    $current_index = 0;
+    print("  </tr>\n");
+    print("  <tr>\n");
+  }
+  while ($current_index < $node_index) {
+    print("    <td></td>\n");
+    $current_index += 1;
+  }
+  print("    <td><a href=\"#".$node."\">".$node."</a></td>\n");
+  $current_index += 1;
+  if ($current_index >= 10) {
+    $current_index = 0;
+    print("  </tr>\n");
+    print("  <tr>\n");
+  }
+}
+print("  </tr>\n</table>\n");
+print("</td><td>&nbsp;</td><td>\n");
+print("Some help goes here");
+print("</td></tr>\n</table>\n");
+
+print("</p>\n");
+
+foreach ($NODES as $node) {
+  print(file_get_contents("nodes/$node.html"));
+  print("<p/>\n");
+}
+
+print("</body>\n");
+
+print("</html>\n");
+?>
Index: trunk/tools/who_is_using_the_cluster/update_all_nodes.py
===================================================================
--- trunk/tools/who_is_using_the_cluster/update_all_nodes.py	(revision 34324)
+++ trunk/tools/who_is_using_the_cluster/update_all_nodes.py	(revision 34324)
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+log = open('/home/panstarrs/ippm/operations/who_is_using_the_cluster/update_all_nodes.log', 'w')
+
+import sys
+sys.path.append('/home/panstarrs/ippm/common/lib64/python2.5/site-packages')
+sys.path.append('/home/panstarrs/ippm/common/lib/python2.5/site-packages')
+
+try:
+    import threading
+    # Importing paramiko displays the following warning:
+#/home/panstarrs/ippm/common/lib/python2.5/site-packages/pycrypto-2.5-py2.5-linux-x86_64.egg/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.
+#  _warn("Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.", PowmInsecureWarning)
+#   I ignore it
+    import warnings
+    warnings.simplefilter("ignore")
+    import paramiko
+#   And restore the warning defaults
+    warnings.simplefilter("always")
+    import socket
+except Exception, e:
+    log.write(str(e) + '\n')
+    log.close()
+    sys.exit(1)
+
+nodes = [  "ipp004", "ipp005", "ipp006", "ipp007", "ipp008", "ipp009", 
+           "ipp010", "ipp011", "ipp012", "ipp013", "ipp014", "ipp015", "ipp016", "ipp017", "ipp018", "ipp019", 
+           "ipp020", "ipp021", "ipp023", "ipp024", "ipp025", "ipp026", "ipp027", "ipp028", "ipp029", 
+           "ipp030", "ipp031", "ipp032", "ipp033", "ipp034", "ipp035", "ipp036", "ipp037", "ipp038", "ipp039", 
+           "ipp040", "ipp041", "ipp042", "ipp043", "ipp044", "ipp045", "ipp046", "ipp047", "ipp048", "ipp049", 
+           "ipp050", "ipp051", "ipp052", "ipp053", "ipp054", "ipp055", "ipp056", "ipp057", "ipp058", "ipp059", 
+           "ipp060", "ipp061", "ipp062", "ipp063", "ipp064", "ipp065", "ipp066", 
+           "ippc01", "ippc02", "ippc03", "ippc04", "ippc05", "ippc06", "ippc07", "ippc08", "ippc09", 
+           "ippc10", "ippc11", "ippc12", "ippc13", "ippc14", "ippc15", "ippc16", "ippc17", "ippc18", "ippc19", 
+           "ippc20", "ippc21", "ippc22", "ippc23", "ippc24", "ippc25", "ippc26", "ippc27", "ippc28", "ippc29", 
+           "ippc30", "ippc31", "ippc32", "ippc33", "ippc34", "ippc35", "ippc36", "ippc37", "ippc38", "ippc39", 
+           "ippc40", "ippc41", "ippc42", "ippc43", "ippc44", "ippc45", "ippc46", "ippc47", "ippc48", "ippc49", 
+           "ippc50", "ippc51", "ippc52", "ippc53", "ippc54", "ippc55", "ippc56", "ippc57", "ippc58", "ippc59", 
+           "ippc60", "ippc61", "ippc62", "ippc63", 
+           "stare00", "stare01", "stare02", "stare03", "stare04", 
+           "ippdb00", "ippdb01", "ippdb02", "ippdb03", 
+           "stsci00", "stsci01", "stsci02", "stsci03", "stsci04", 
+           "stsci05", "stsci06", "stsci07", "stsci08", "stsci09",
+           ]
+
+class SshThread(threading.Thread):
+    def __init__(self, node):
+        threading.Thread.__init__(self)
+        self.node = node
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.node)
+    def run(self):
+        stdin, stdout, stderr = self.ssh.exec_command('/home/panstarrs/ippm/operations/who_is_using_the_cluster/get_processes_information.py')
+        for line in stderr:
+            print 'Error for %s: %s' % ( self.node, line)
+
+if __name__=='__main__':
+    ssh_threads = []
+    for node in nodes:
+        try:
+            ssh_threads.append(SshThread(node))
+        except socket.error, e:
+            print 'Exception caught for node %s' % node
+            print '  %s' % e
+    for t in ssh_threads:
+        t.start()
+    for t in ssh_threads:
+        t.join()