IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42858 for trunk


Ignore:
Timestamp:
May 13, 2025, 1:13:09 PM (14 months ago)
Author:
cclin33
Message:

add disk usage in Database status section

Location:
trunk/ippMonitor
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippMonitor/czartool/czarpoll.pl

    r42798 r42858  
    229229
    230230    my $server = undef;
    231     my $alive = undef;
    232     my $running = undef;
     231
     232    my ($alive, $running);
     233
    233234    foreach $server (@{$servers}) {
     235        # For gpc1
    234236        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
    235237        $czarDb->updateServerStatus($server,'gpc1', $alive, $running);
     238        # For gpc2
    236239        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
    237240        $czarDb->updateServerStatus($server,'gpc2', $alive, $running);
     241
    238242   }
    239243}
    … …  
    245249###########################################################################
    246250sub updateServerStatusLive {
    247     print "* Checking all pantasks servers\n";
     251    print "* Checking all pantasks live servers\n";
    248252
    249253    my $servers = $pantasks->getServerList();
    250254
    251255    my $server = undef;
    252     my $alive = undef;
    253     my $running = undef;
     256    my ($alive, $running, $pan_host);
     257
    254258    foreach $server (@{$servers}) {
    255         $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
    256         $czarDb->updateServerStatusLive($server,'gpc1', $alive, $running);
    257         $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
    258         $czarDb->updateServerStatusLive($server,'gpc2', $alive, $running);
     259        # For gpc1
     260        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running, \$pan_host);
     261        $czarDb->updateServerStatusLive($server,'gpc1', $alive, $running, $pan_host);
     262        print "Server: $server | Telescope: gpc1 | Alive: $alive | Running: $running | Host: " . (defined $pan_host ? $pan_host : "undef") . "\n";
     263        # For gpc2
     264        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running, \$pan_host);
     265        $czarDb->updateServerStatusLive($server,'gpc2', $alive, $running, $pan_host);
     266        print "Server: $server | Telescope: gpc2 | Alive: $alive | Running: $running | Host: " . (defined $pan_host ? $pan_host : "undef") . "\n";
    259267   }
    260268}
    … …  
    637645
    638646            # Convert KB to TB
    639             my $total_tb     = sprintf("%.1f", $blocks    / 1024 / 1024 / 1024);
    640             my $used_tb      = sprintf("%.1f", $used      / 1024 / 1024 / 1024);
    641             my $available_tb = sprintf("%.1f", $available / 1024 / 1024 / 1024);
     647            my $total_tb     = sprintf("%.3f", $blocks    / 1024 / 1024 / 1024 );
     648            my $used_tb      = sprintf("%.3f", $used      / 1024 / 1024 / 1024 );
     649            my $available_tb = sprintf("%.3f", $available / 1024 / 1024 / 1024 );
    642650
    643651            my $used_percent      = sprintf("%.3f", $used / $blocks * 100);
    … …  
    650658
    651659            # Example: ippdb01.0(3.5T)
    652             my $host_label = "$host.$suffix(${total_tb}T)";
    653             $czarDb->updateHost($host_label, 100, $available_percent, $used_percent, $readable, $writable);
    654         }
    655     }
    656 }
     660            #my $host_label = "$host.$suffix(${total_tb}T)";
     661            my $host_label = "$host.$suffix";
     662            $czarDb->updateHost($host_label, $total_tb, $available_tb, $used_tb, $readable, $writable);
     663        }
     664    }
     665}

  • trunk/ippMonitor/czartool/czartool/CzarDb.pm

    r41125 r42858  
    175175###########################################################################
    176176sub updateServerStatusLive {
    177     my ($self, $server,$telescope, $alive, $running) = @_;
     177    my ($self, $server,$telescope, $alive, $running, $pan_host) = @_;
    178178
    179179    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    184184    $query = $self->{_db}->prepare(<<SQL);
    185185    INSERT INTO live_servers
    186         (server,telescope, alive, running)
     186        (server,telescope, alive, running, host)
    187187        VALUES
    188         ('$server','$telescope', $alive, $running);
     188        ('$server','$telescope', $alive, $running, '$pan_host');
    189189SQL
    190190       $query->execute;

  • trunk/ippMonitor/czartool/czartool/Pantasks.pm

    r42794 r42858  
    179179###########################################################################
    180180sub getServerStatus {
    181     my ($self, $server,$telescope, $alive, $running) = @_;
    182 
    183     my $ipphome;
    184 
    185     if ($telescope eq 'gpc1') {
    186        $ipphome = $ipphome1;
    187     }
    188     elsif ($telescope eq 'gpc2') {
    189        $ipphome = $ipphome2;
    190     }
    191     else {
    192         print "WARNING: Must specify a telescope for connecting to the right pantask\n";
     181    my ($self, $server,$telescope, $alive, $running, $pan_host) = @_;
     182
     183    my $ipphome;
     184
     185    if ($telescope eq 'gpc1') {
     186       $ipphome = $ipphome1;
     187    }
     188    elsif ($telescope eq 'gpc2') {
     189       $ipphome = $ipphome2;
     190    }
     191    else {
     192        print "WARNING: Must specify a telescope for connecting to the right pantask\n";
     193    }
     194
     195    my $pan_host_cmd = "grep \"PANTASKS_SERVER \" $ipphome/$server/ptolemy.rc 2>/dev/null | awk '{print \$2}'";
     196    ${$pan_host} = `$pan_host_cmd`;
     197    chomp(${$pan_host});
     198   
     199    #print "DEBUG: pan_host for server $server ($telescope) is: " . (defined $pan_host ? $pan_host : "undef") . "\n";
     200
     201    if ($? != 0 || ${$pan_host} eq '') {
     202        ${$pan_host} = undef;
    193203    }
    194204

  • trunk/ippMonitor/raw/czartool_labels.php

    r42794 r42858  
    266266    $start = microtime(true);
    267267    echo "<tr>";
    268     createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\"><font color=\"blue\">here</a>)", 2);
     268    createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\"><font color=\"blue\">here</a>)", 3);
    269269     
    270270    echo "<tr>";
    271271    createTableColumnHeader("Database");
    272     createTableColumnHeader("Status (sec behind master)");
     272    createTableColumnHeader("disk (TB)");
     273    createTableColumnHeader("Status");
    273274    // gpc dbs
    274275    showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $LABEL_GPC1);
    … …  
    287288   
    288289    // end time
    289     echo "<tr><th colspan=2>";
     290    echo "<tr><th colspan=3>";
    290291    $total_time = round((microtime(true) - $start), 3);
    291292    echo "loading in $total_time seconds.";
    … …  
    375376echo getSpaces($czardb);
    376377# storage plot
    377  echo getHosts($czardb);
     378echo getHosts($czardb);
    378379
    379380# timer for main page end
    … …  
    12521253    echo $table;
    12531254    echo "<tr>";
    1254     createTableTitle("Pantasks server status", 3);
     1255    createTableTitle("Pantasks server status", 4);
    12551256    echo "<tr>";
    12561257
    12571258    // Define column headers in an array for flexibility
    1258     $columnHeaders = array("Server", "Alive?", "Running?");
     1259    $columnHeaders = array("Server", "Host", "Alive?", "Running?");
    12591260    foreach ($columnHeaders as $header) {
    12601261        createTableColumnHeader($header);
    … …  
    12631264
    12641265    foreach ($servers as &$server) {
    1265         getServerStatus($db,$proj, $server, $alive, $running);
     1266        getServerStatus($db,$proj, $server, $alive, $running, $host);
    12661267
    12671268        echo "<tr>\n";
    … …  
    12751276            ."&stage=".$selectedStage
    12761277            ."&plottype=".$plotType;
     1278       // ganglia
     1279       $link2 = "http://ganglia.pan-starrs.ifa.hawaii.edu/?m=load_one&r=hour&s=descending&c=IPP+Production&h=".$host.".ifa.hawaii.edu&sh=1&hc=7&z=small";
    12771280
    12781281        // Display server row cells
    12791282        createFormattedTableCell(1, $link, $server, 0, 0);
     1283        createFormattedTableCell(1, $link2, $host, 0 , 0);
    12801284        createFormattedTableCell(0, "", $alive ? "yes" : "NO", !$alive, 0);
    12811285        createFormattedTableCell(0, "", $running ? "yes" : "NO", !$running, 0);
     1286
    12821287
    12831288        echo "</tr>\n";
    … …  
    13281333#
    13291334###########################################################################
    1330 function getServerStatus($db,$proj, $server, &$alive, &$running) {
     1335function getServerStatus($db,$proj, $server, &$alive, &$running, &$host) {
    13311336
    13321337#    $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
    1333     $sql = "SELECT alive, running FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
     1338    $sql = "SELECT alive, running, host FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
    13341339    if($debug){echo "$sql<br>";}
    13351340
    … …  
    13401345    $alive = $row[0];
    13411346    $running = $row[1];
     1347    $host = $row[2];
    13421348}
    13431349
    … …  
    13851391###########################################################################
    13861392function showReplicationsStatus($replHost, $replUser, $replPassword, $replDatabaseName) {
    1387     #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
     1393    //print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
     1394    global $czardb;
     1395
     1396    // Prepare storage info early, https://panstarrs.atlassian.net/browse/IPP-2338
     1397    $storageInfo = '';
     1398    $hostPrefixParts = explode(' ', $replDatabaseName);
     1399    $hostPrefix = addslashes($hostPrefixParts[0]);
     1400
     1401    $sql = sprintf(
     1402        "SELECT host, FORMAT(total,1), FORMAT(available,2), FORMAT(used,2), writable, readable, FORMAT(used/total*100,1)
     1403         FROM hosts
     1404         WHERE host LIKE '%s%%' order by available desc
     1405         LIMIT 1",
     1406         $hostPrefix
     1407    );
     1408
     1409    $qry = $czardb->query($sql);
     1410    if (PEAR::isError($qry)) {
     1411        $storageInfo = "<td style=\"background-color: red; color: white;\">Storage query error</td>";
     1412    } else {
     1413        if ($qry->fetchInto($row)) {
     1414            list($host, $total, $available, $used, $writable, $readable, $ratio) = $row;
     1415            $storageInfo = "<td>$ratio% of $total</td>";
     1416        } else {
     1417            $storageInfo = "<td>No host storage data</td>";
     1418        }
     1419    }
     1420
    13881421    $dbRepl = DB::connect("mysql://$replUser:$replPassword@$replHost");
    13891422    if (PEAR::isError($dbRepl)) {
    1390         echo "<tr><td>$replDatabaseName</td><td bgcolor=\"red\">MySQL DB connection error - check configuration</td></tr>";
    1391         //die("MySQL DB connection error: Check the configuration for $replDatabaseName");
    1392         //die("");
     1423        echo "<tr><td>$replDatabaseName</td>$storageInfo<td style=\"background-color: red; color: white;\">MySQL DB connection error - check configuration</td></tr>";
     1424        return -1; // Don't stop the rest of the script
     1425    }
     1426
     1427    // Query replication status
     1428    $res = $dbRepl->query('SHOW SLAVE STATUS');
     1429    if (PEAR::isError($res)) {
     1430        echo "<tr><td>$replDatabaseName</td>$storageInfo<td style=\"background-color: red; color: white;\">MySQL query error - invalid user or privileges</td></tr>";
     1431        $dbRepl->disconnect();
    13931432        return -1;
    1394 
    1395     }
    1396 
    1397     # note: mysql user must have either SUPER or REPLICATION CLIENT privileges on the database
    1398     $res = $dbRepl->query('SHOW SLAVE STATUS');
    1399     if (dberror($res)) {
    1400         echo "<tr><td>$replDatabaseName</td><td bgcolor=\"red\">MySQL query error - invalid user? check configuration</td></tr>";
    1401         //die("MySQL DB connection error: Check the configuration for $replDatabaseName");
    1402         //die("");
    1403         return -1;
    1404     }
    1405 
    1406     while ($res->fetchInto($row)) {
    1407         # Have a look for Last_Errno in http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html
    1408         $errorStatusInMySql = $row[18];
    1409         $sec_behind = $row[32];
    1410         if ($sec_behind===NULL) {
    1411             $sec_behind = 99999999;
    1412             $errorStatusInMySql = 999999999;
    1413         }   
    1414         $replStatus = ($errorStatusInMySql==0?"OK ($sec_behind)":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
    1415         echo "<tr><td>$replDatabaseName</td><td>$replStatus</td></tr>";
    1416         if ($errorStatusInMySql!=0) {
    1417             echo "<tr><td colspan=\"2\">Connect to $replDatabaseName replication host, execute 'SHOW SLAVE STATUS', and fix the problem.<br/>";
    1418             echo "Information <a href=\"http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html\">here</a></td></tr>";
     1433    }
     1434
     1435    if (!$res->fetchInto($row)) {
     1436        echo "<tr><td>$replDatabaseName</td>$storageInfo<td>No replication data available</td></tr>";
     1437        $dbRepl->disconnect();
     1438        return 0;
     1439    }
     1440
     1441    $errorStatusInMySql = $row[18];
     1442    $sec_behind = $row[32];
     1443
     1444    if ($sec_behind === NULL) {
     1445        $sec_behind = 99999999;
     1446        $errorStatusInMySql = 999999999;
     1447    }
     1448
     1449    $replStatus = ($errorStatusInMySql == 0)
     1450        ? "OK ($sec_behind sec behind)"
     1451        : "<span style=\"background-color: yellow; color: red;\">PROBLEM</span>";
     1452
     1453    echo "<tr><td>$replDatabaseName</td>$storageInfo";
     1454
     1455    if ($errorStatusInMySql != 0) {
     1456        echo "<tr><td colspan=\"5\">Connect to <strong>$replDatabaseName</strong>, run <code>SHOW SLAVE STATUS</code>, and resolve the issue. ";
     1457        echo "See <a href=\"https://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html\" target=\"_blank\">MySQL Docs</a>.</td></tr>";
     1458    }
     1459
     1460    echo "<td>$replStatus</td>";
     1461
     1462    $dbRepl->disconnect();
     1463
     1464}
     1465
     1466###########################################################################
     1467#
     1468# Shows the status of the various replication mysql servers
     1469#
     1470###########################################################################
     1471function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
     1472    //print "<br>$Host, $User, $Password, $DatabaseName<br/>";
     1473    global $czardb;
     1474    // Prepare storage info early, https://panstarrs.atlassian.net/browse/IPP-2338
     1475    $storageInfo = '';
     1476    $hostPrefixParts = explode(' ', $DatabaseName);
     1477    $hostPrefix = addslashes($hostPrefixParts[0]);
     1478
     1479    $sql = sprintf(
     1480        "SELECT host, FORMAT(total,1), FORMAT(available,2), FORMAT(used,2), writable, readable, FORMAT(used/total*100,1)
     1481         FROM hosts
     1482         WHERE host LIKE '%s%%' order by available desc
     1483         LIMIT 1",
     1484         $hostPrefix
     1485    );
     1486
     1487    $qry = $czardb->query($sql);
     1488    if (PEAR::isError($qry)) {
     1489        $storageInfo = "<td style=\"background-color: red; color: white;\">Storage query error</td>";
     1490    } else {
     1491        if ($qry->fetchInto($row)) {
     1492            list($host, $total, $available, $used, $writable, $readable, $ratio) = $row;
     1493            $storageInfo = "<td>$ratio% of $total </td>";
     1494        } else {
     1495            $storageInfo = "<td>No host storage data</td>";
    14191496        }
    14201497    }
    1421     $dbRepl->disconnect();
    1422 }
    1423 
    1424 ###########################################################################
    1425 #
    1426 # Shows the status of the various replication mysql servers
    1427 #
    1428 ###########################################################################
    1429 function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
    1430     #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
     1498
    14311499    $db = DB::connect("mysql://$User:$Password@$Host");
    14321500    if (PEAR::isError($db)) {
    1433         echo "<tr><td>$DatabaseName</td><td bgcolor=\"red\">MySQL DB connection error</td></tr>";
    1434         //die("MySQL DB connection error: Check the configuration for $DatabaseName");
    1435         //die("");
     1501        echo "<tr><td>$DatabaseName</td>$storageInfo<td>$Status</td>";
    14361502        return -1;
    1437 
    14381503    }
    14391504    $Status = ($errorStatusInMySql==0?"OK":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
    1440     echo "<tr><td>$DatabaseName</td><td>$Status</td></tr>";
     1505    echo "<tr><td>$DatabaseName</td>$storageInfo<td>$Status</td>";
    14411506
    14421507    $db->disconnect();
    14431508}
     1509
    14441510
    14451511###########################################################################
    … …  
    16641730    echo "     'free', {type: 'string', role: 'style'}, {type: 'string', role: 'tooltip', 'p': {'html': true}}],";
    16651731
    1666     $sql = "SELECT host, format(total,1), format(available,2), format(used,2), writable, readable, format(used/total*100,2) FROM hosts order by host";
     1732    $sql = "SELECT host, format(total,1), format(available,2), format(used,2), writable, readable, format(used/total*100,2) FROM hosts where host like 'ipp1%' or host like 'ippb%' order by host";
    16671733    if ($debug) {echo "$sql<br>";}
    16681734    $qry = $db->query($sql);
Note: See TracChangeset for help on using the changeset viewer.