Index: trunk/ippMonitor/raw/czartool_labels.php
===================================================================
--- trunk/ippMonitor/raw/czartool_labels.php	(revision 42794)
+++ trunk/ippMonitor/raw/czartool_labels.php	(revision 42858)
@@ -266,9 +266,10 @@
     $start = microtime(true);
     echo "<tr>";
-    createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\"><font color=\"blue\">here</a>)", 2);
+    createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\"><font color=\"blue\">here</a>)", 3);
       
     echo "<tr>";
     createTableColumnHeader("Database");
-    createTableColumnHeader("Status (sec behind master)");
+    createTableColumnHeader("disk (TB)");
+    createTableColumnHeader("Status");
     // gpc dbs
     showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $LABEL_GPC1);
@@ -287,5 +288,5 @@
     
     // end time
-    echo "<tr><th colspan=2>";
+    echo "<tr><th colspan=3>";
     $total_time = round((microtime(true) - $start), 3);
     echo "loading in $total_time seconds.";
@@ -375,5 +376,5 @@
 echo getSpaces($czardb);
 # storage plot
- echo getHosts($czardb);
+echo getHosts($czardb);
 
 # timer for main page end
@@ -1252,9 +1253,9 @@
     echo $table;
     echo "<tr>";
-    createTableTitle("Pantasks server status", 3);
+    createTableTitle("Pantasks server status", 4);
     echo "<tr>";
 
     // Define column headers in an array for flexibility
-    $columnHeaders = array("Server", "Alive?", "Running?");
+    $columnHeaders = array("Server", "Host", "Alive?", "Running?");
     foreach ($columnHeaders as $header) {
         createTableColumnHeader($header);
@@ -1263,5 +1264,5 @@
 
     foreach ($servers as &$server) {
-        getServerStatus($db,$proj, $server, $alive, $running);
+        getServerStatus($db,$proj, $server, $alive, $running, $host);
 
         echo "<tr>\n";
@@ -1275,9 +1276,13 @@
             ."&stage=".$selectedStage
             ."&plottype=".$plotType;
+       // ganglia
+       $link2 = "http://ganglia.pan-starrs.ifa.hawaii.edu/?m=load_one&r=hour&s=descending&c=IPP+Production&h=".$host.".ifa.hawaii.edu&sh=1&hc=7&z=small";
 
         // Display server row cells
         createFormattedTableCell(1, $link, $server, 0, 0);
+        createFormattedTableCell(1, $link2, $host, 0 , 0);
         createFormattedTableCell(0, "", $alive ? "yes" : "NO", !$alive, 0);
         createFormattedTableCell(0, "", $running ? "yes" : "NO", !$running, 0);
+
 
         echo "</tr>\n";
@@ -1328,8 +1333,8 @@
 #
 ###########################################################################
-function getServerStatus($db,$proj, $server, &$alive, &$running) {
+function getServerStatus($db,$proj, $server, &$alive, &$running, &$host) {
 
 #    $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
-    $sql = "SELECT alive, running FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
+    $sql = "SELECT alive, running, host FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
     if($debug){echo "$sql<br>";}
 
@@ -1340,4 +1345,5 @@
     $alive = $row[0];
     $running = $row[1];
+    $host = $row[2];
 }
 
@@ -1385,61 +1391,121 @@
 ###########################################################################
 function showReplicationsStatus($replHost, $replUser, $replPassword, $replDatabaseName) {
-    #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
+    //print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
+    global $czardb;
+
+    // Prepare storage info early, https://panstarrs.atlassian.net/browse/IPP-2338
+    $storageInfo = '';
+    $hostPrefixParts = explode(' ', $replDatabaseName);
+    $hostPrefix = addslashes($hostPrefixParts[0]);
+
+    $sql = sprintf(
+        "SELECT host, FORMAT(total,1), FORMAT(available,2), FORMAT(used,2), writable, readable, FORMAT(used/total*100,1)
+         FROM hosts
+         WHERE host LIKE '%s%%' order by available desc
+         LIMIT 1",
+         $hostPrefix
+    );
+
+    $qry = $czardb->query($sql);
+    if (PEAR::isError($qry)) {
+        $storageInfo = "<td style=\"background-color: red; color: white;\">Storage query error</td>";
+    } else {
+        if ($qry->fetchInto($row)) {
+            list($host, $total, $available, $used, $writable, $readable, $ratio) = $row;
+            $storageInfo = "<td>$ratio% of $total</td>";
+        } else {
+            $storageInfo = "<td>No host storage data</td>";
+        }
+    }
+
     $dbRepl = DB::connect("mysql://$replUser:$replPassword@$replHost");
     if (PEAR::isError($dbRepl)) {
-        echo "<tr><td>$replDatabaseName</td><td bgcolor=\"red\">MySQL DB connection error - check configuration</td></tr>";
-        //die("MySQL DB connection error: Check the configuration for $replDatabaseName");
-        //die("");
+        echo "<tr><td>$replDatabaseName</td>$storageInfo<td style=\"background-color: red; color: white;\">MySQL DB connection error - check configuration</td></tr>";
+        return -1; // Don't stop the rest of the script
+    }
+
+    // Query replication status
+    $res = $dbRepl->query('SHOW SLAVE STATUS');
+    if (PEAR::isError($res)) {
+        echo "<tr><td>$replDatabaseName</td>$storageInfo<td style=\"background-color: red; color: white;\">MySQL query error - invalid user or privileges</td></tr>";
+        $dbRepl->disconnect();
         return -1;
-
-    }
-
-    # note: mysql user must have either SUPER or REPLICATION CLIENT privileges on the database 
-    $res = $dbRepl->query('SHOW SLAVE STATUS');
-    if (dberror($res)) {
-        echo "<tr><td>$replDatabaseName</td><td bgcolor=\"red\">MySQL query error - invalid user? check configuration</td></tr>";
-        //die("MySQL DB connection error: Check the configuration for $replDatabaseName");
-        //die("");
-        return -1;
-    }
-
-    while ($res->fetchInto($row)) {
-        # Have a look for Last_Errno in http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html
-        $errorStatusInMySql = $row[18];
-	$sec_behind = $row[32];
-	if ($sec_behind===NULL) {
-	    $sec_behind = 99999999;
-	    $errorStatusInMySql = 999999999;
-	}   
-        $replStatus = ($errorStatusInMySql==0?"OK ($sec_behind)":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
-        echo "<tr><td>$replDatabaseName</td><td>$replStatus</td></tr>";
-        if ($errorStatusInMySql!=0) {
-            echo "<tr><td colspan=\"2\">Connect to $replDatabaseName replication host, execute 'SHOW SLAVE STATUS', and fix the problem.<br/>";
-            echo "Information <a href=\"http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html\">here</a></td></tr>";
+    }
+
+    if (!$res->fetchInto($row)) {
+        echo "<tr><td>$replDatabaseName</td>$storageInfo<td>No replication data available</td></tr>";
+        $dbRepl->disconnect();
+        return 0;
+    }
+
+    $errorStatusInMySql = $row[18];
+    $sec_behind = $row[32];
+
+    if ($sec_behind === NULL) {
+        $sec_behind = 99999999;
+        $errorStatusInMySql = 999999999;
+    }
+
+    $replStatus = ($errorStatusInMySql == 0)
+        ? "OK ($sec_behind sec behind)"
+        : "<span style=\"background-color: yellow; color: red;\">PROBLEM</span>";
+
+    echo "<tr><td>$replDatabaseName</td>$storageInfo";
+
+    if ($errorStatusInMySql != 0) {
+        echo "<tr><td colspan=\"5\">Connect to <strong>$replDatabaseName</strong>, run <code>SHOW SLAVE STATUS</code>, and resolve the issue. ";
+        echo "See <a href=\"https://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html\" target=\"_blank\">MySQL Docs</a>.</td></tr>";
+    }
+
+    echo "<td>$replStatus</td>";
+
+    $dbRepl->disconnect();
+
+}
+
+###########################################################################
+#
+# Shows the status of the various replication mysql servers
+#
+###########################################################################
+function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
+    //print "<br>$Host, $User, $Password, $DatabaseName<br/>";
+    global $czardb;
+    // Prepare storage info early, https://panstarrs.atlassian.net/browse/IPP-2338
+    $storageInfo = '';
+    $hostPrefixParts = explode(' ', $DatabaseName);
+    $hostPrefix = addslashes($hostPrefixParts[0]);
+
+    $sql = sprintf(
+        "SELECT host, FORMAT(total,1), FORMAT(available,2), FORMAT(used,2), writable, readable, FORMAT(used/total*100,1)
+         FROM hosts
+         WHERE host LIKE '%s%%' order by available desc
+         LIMIT 1",
+         $hostPrefix
+    );
+
+    $qry = $czardb->query($sql);
+    if (PEAR::isError($qry)) {
+        $storageInfo = "<td style=\"background-color: red; color: white;\">Storage query error</td>";
+    } else {
+        if ($qry->fetchInto($row)) {
+            list($host, $total, $available, $used, $writable, $readable, $ratio) = $row;
+            $storageInfo = "<td>$ratio% of $total </td>";
+        } else {
+            $storageInfo = "<td>No host storage data</td>";
         }
     }
-    $dbRepl->disconnect();
-}
-
-###########################################################################
-#
-# Shows the status of the various replication mysql servers
-#
-###########################################################################
-function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
-    #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
+
     $db = DB::connect("mysql://$User:$Password@$Host");
     if (PEAR::isError($db)) {
-        echo "<tr><td>$DatabaseName</td><td bgcolor=\"red\">MySQL DB connection error</td></tr>";
-        //die("MySQL DB connection error: Check the configuration for $DatabaseName");
-        //die("");
+        echo "<tr><td>$DatabaseName</td>$storageInfo<td>$Status</td>";
         return -1;
-
     }
     $Status = ($errorStatusInMySql==0?"OK":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
-    echo "<tr><td>$DatabaseName</td><td>$Status</td></tr>";
+    echo "<tr><td>$DatabaseName</td>$storageInfo<td>$Status</td>";
 
     $db->disconnect();
 }
+
 
 ###########################################################################
@@ -1664,5 +1730,5 @@
     echo "     'free', {type: 'string', role: 'style'}, {type: 'string', role: 'tooltip', 'p': {'html': true}}],";
 
-    $sql = "SELECT host, format(total,1), format(available,2), format(used,2), writable, readable, format(used/total*100,2) FROM hosts order by host";
+    $sql = "SELECT host, format(total,1), format(available,2), format(used,2), writable, readable, format(used/total*100,2) FROM hosts where host like 'ipp1%' or host like 'ippb%' order by host";
     if ($debug) {echo "$sql<br>";}
     $qry = $db->query($sql);