IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 23, 2019, 2:24:01 PM (7 years ago)
Author:
tdeboer
Message:

updates to ippMonitor and czartool to handle gpc2 labels and improve czartool

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippMonitor/raw/czartool_labels.php

    r40427 r40840  
    9090        # "publishing",
    9191        "registration",
    92         "replication",
    9392        "stack",
    9493        # "deepstack",
    … …  
    139138
    140139// $labels is generated with sequential integer key values
    141 $labels = getLabels($czardb, $selectedMode);
    142 $distLabelsIndex = getLabels($czardb, "distribution");
     140$labels = getLabels($czardb, $proj, $selectedMode);
     141$distLabelsIndex = getLabels($czardb, $proj, "distribution");
    143142for ($i = 0; $i < count($distLabelsIndex); $i++) {
    144143  $distLabelsExist[$distLabelsIndex[$i]] = true;
    … …  
    195194echo "<td style=width:100px;text-align:top;\">";
    196195 # time series plot
    197 echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=$plotType\"><br>";
     196echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
     197# log time series plot
     198echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=t&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=log\"><br>";
    198199 # rate time series plot
    199200echo "<br><img src=\"czartool_getplot.php?mode=$selectedMode&type=rt&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
    200  # rate stacked histo plot
    201 # echo "<img src=\"czartool_getplot.php?mode=$selectedMode&type=rh&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
    202201 # histogram plot
    203202# echo "<img src=\"czartool_getplot.php?mode=$selectedMode&type=h&label=$selectedLabel&proj=$proj&stage=$selectedStage&plottype=linear\"><br>";
    … …  
    218217
    219218// status table at top
    220 $nsStatus = getNightlyScienceStatus($czardb);
     219$nsStatus = getNightlyScienceStatus($czardb, $proj);
    221220$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
    222221$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
    … …  
    232231echo "<tr><td>Documentation</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\"><font color=\"blue\">here</font></a></td></tr>";
    233232echo "<tr><td>Cluster load</td><td><a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&hc=6&c=IPP%2520Production\"><font color=\"blue\">here</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=cpu_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">cpu_report</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=mem_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">mem_report</font></a></td></tr>";
    234 echo "<tr><td>Who's using the cluster?</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/ippMonitor/clusterMonitor/index.php\"><font color=\"blue\">here</font></a></td></tr>";
    235 echo "<tr><td>Files monitor</td><td><a href=\"http://ippmonitor.ipp.ifa.hawaii.edu/diskMonitor\"><font color=\"blue\">here</font></a></td></tr>";
    236 echo "<tr><td>IPP metrics</td><td><a href=\"http://ipp0022.ifa.hawaii.edu/ps1sc/ippMetrics\"><font color=\"blue\">here</font></a></td></tr>";
    237 echo "<tr><td>Czar log pages</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_CzarLogs\"><font color=\"blue\">All</font></a>&nbsp;&nbsp;&nbsp;";
    238 echo "<a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_Czarlog_".(date("Ymd")-date("w")+"1")."\">This week</a>";
    239 echo "</td></tr>";
     233echo "<tr><td>IPP metrics</td><td><a href=\"https://ps1wiki.ifa.hawaii.edu/data/metrics/index.html\"><font color=\"blue\">here</font></a></td></tr>";
     234echo "<tr><td>Czar log pages</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/pages/viewrecentblogposts.action?key=IPPCZAR\"><font color=\"blue\">here</font></a></td></tr>";
    240235echo "<tr><td>Exposure summary</td><td><a href=\"czartool_exposures.php?pass=$pass&proj=$proj\"><font color=\"blue\">here</font></a></td></tr>";
     236echo "<tr><td>Confluence</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/display/IPPCZAR/IPP-Czaring+Home\"><font color=\"blue\">here</font></a></td></tr>";
     237echo "<tr><td>JIRA</td><td><a href=\"https://jira.ipp.ifa.hawaii.edu/secure/Dashboard.jspa\"><font color=\"blue\">here</font></a></td></tr>";
    241238
    242239$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
    … …  
    266263      createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "<br>";
    267264      createSummitDataTable($projectdb); echo "<br>"; // this is slow because of the join between summitExp and rawExp
    268       createDatesTable($czardb); echo "<br>";
    269 
    270       # This is awful
     265      createChunkDataTable($projectdb); echo "<br>";
     266      createDatesTable($czardb, $proj); echo "<br>";
     267
    271268      echo $table;
    272269      echo "<tr>";
    273       createTableTitle("Database replication status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
     270      createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
    274271
    275272      echo "<tr>";
    276273      createTableColumnHeader("Database");
    277274      createTableColumnHeader("Status (sec behind master)");
     275      showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1);
    278276      showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
    279277      showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY);
    280278
     279      echo "<tr>";
     280
     281      showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS);
    281282      showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
    282283      showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY);
     284      showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY);
    283285
    284286#     some mysql replication slaves to add when they are returned to service
    285287#     showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP);
    286 #     showReplicationsStatus($REPL_HOST_IPPADMIN, $REPL_USER_IPPADMIN, $REPL_PASSWORD_IPPADMIN, $REPL_DBNAME_IPPADMIN);  --- this one is in GPC1
    287288#     showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP);
    288289      echo "</table>";
     290
     291      echo $table;
     292      echo "<tr>";
     293      createTableTitle("Critical machine status", 2);
     294
     295      echo "<tr>";
     296      createTableColumnHeader("Hostname");
     297      createTableColumnHeader("Status");
     298      showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY);
     299      showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY);
     300
     301#      showPingStatus($HOST_APACHE1, $DBNAME_APACHE1);
     302      showPingStatus($HOST_APACHE2, $DBNAME_APACHE2);
     303      showPingStatus($HOST_APACHE3, $DBNAME_APACHE3);
     304      showPingStatus($HOST_APACHE4, $DBNAME_APACHE4);
     305      showPingStatus($HOST_APACHE5, $DBNAME_APACHE5);
     306      showPingStatus($HOST_APACHE6, $DBNAME_APACHE6);
     307      showPingStatus($HOST_PROXY, $DBNAME_PROXY);
     308      showPingStatus($HOST_SVN, $DBNAME_SVN);
     309      showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE);
     310
     311      echo "</table>";
     312
    289313echo "</table>";
    290314menu_end();
    … …  
    401425###########################################################################
    402426#
     427# Creates a table of chunk progress
     428#
     429###########################################################################
     430function createChunkDataTable($projectdb) {
     431
     432    #First find the reference exposure ID to speed up the subsequent queries
     433    $date = gmdate("Y-m-d");
     434    $sql = "SELECT MAX(exp_id) FROM rawExp WHERE dateobs >= date_sub('$date', interval 20 day) AND dateobs <= date_sub('$date', interval 10 day)";
     435    $qry = $projectdb->query($sql);
     436    if (dberror($qry)) {
     437        print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n";
     438        $refExpID = 0;
     439    }
     440    $qry->fetchInto($refExpID);
     441
     442    #Find the number of chunks
     443    $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM";
     444    $query .= "   (";
     445    $query .= "     SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit";
     446    $query .= "     FROM rawExp";
     447    $query .= "     WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0]";
     448    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'";
     449    $query .= "     AND comment LIKE '%visit 1%'";
     450    $query .= "     GROUP BY chunk";
     451    $query .= "   ) AS groupchunk";
     452    $query .= "   LEFT JOIN";
     453    $query .= "   (";
     454    $query .= "     SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM ";
     455    $query .= "       (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits";
     456    $query .= "         FROM rawExp";
     457    $query .= "         WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
     458    $query .= "         AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'";
     459    $query .= "         GROUP BY object, filter, chunk";
     460    $query .= "       ) AS groupobj";
     461    $query .= "     WHERE groupobj.Nvisits = 4 ";
     462    $query .= "     GROUP BY groupobj.chunk";
     463    $query .= "   ) AS group4";
     464    $query .= "   on groupchunk.chunk=group4.chunk ORDER BY dateobs";
     465
     466    #$sql = "SELECT DISTINCT substr(comment, 1, position(' ' in comment)) FROM rawExp WHERE dateobs LIKE '$date%' AND exp_id > $refExpID AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit 1%'";
     467
     468    $qry = $projectdb->query($query);
     469    if (dberror($qry)) {
     470        echo "<b>error reading rawExp table</b><br>\n";
     471        echo "<br><small><b> table query : $query </b></small><br>\n";
     472        menu_end();
     473    }
     474
     475    // set up the table
     476    global $table;
     477    echo $table;
     478    echo "<tr>";
     479    createTableTitle("Status of last night's chunks", 5);
     480    echo "<tr>";
     481    createTableColumnHeader("Chunk");
     482    createTableColumnHeader("Nobs_1stvisit");
     483    createTableColumnHeader("Nobs_4visits");
     484    createTableColumnHeader("Ndiffs_published");
     485    createTableColumnHeader("Completion");
     486
     487    $msg = "No science images taken since $date";
     488
     489    // list the results
     490    while ($qry->fetchInto($row)) {
     491        $Nfirstvisits = 0;
     492        $Nfourvisits = 0;
     493
     494        $chunkname = $row[0];
     495        if($row[1] > 0) $Nfirstvisits = $row[1];
     496        if($row[2] > 0) $Nfourvisits = $row[2];
     497
     498        $query2 = "     SELECT warp_id FROM ";
     499        $query2 .= "           warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
     500        $query2 .= "           WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
     501        $query2 .= "           AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'";
     502        $qry2 = $projectdb->query($query2);
     503
     504        $Ndiffs = 0;
     505        while ($qry2->fetchInto($row2)) {
     506            $query3 = "     SELECT diff_id FROM ";
     507            $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)";
     508            $query3 .= "           WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id";
     509            $qry3 = $projectdb->query($query3);
     510            while ($qry3->fetchInto($row3)) {
     511                if ($row3[0] !=0) $Ndiffs ++;
     512            }
     513
     514            $query3 = "     SELECT diff_id FROM ";
     515            $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)";
     516            $query3 .= "           WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id";
     517            $qry3 = $projectdb->query($query3);
     518            while ($qry3->fetchInto($row3)) {
     519                if ($row3[0] !=0) $Ndiffs ++;
     520            }
     521        }
     522
     523        if ($Ndiffs != ($Nfirstvisits*4)) $complete = "chunk not done"; 
     524        if ($Ndiffs == ($Nfirstvisits*4)) $complete = "chunk done";       
     525
     526        if ($Nfirstvisits != $Nfourvisits) {
     527            $Ndiffs .= "/?  ";
     528        }
     529        if ($Nfirstvisits == $Nfourvisits) {
     530            $Ndiffs .= "/";
     531            $Ndiffs .= $Nfirstvisits*4;
     532        }
     533
     534        echo "<tr>";
     535
     536        echo "<td>$chunkname</td>";
     537        echo "<td>$Nfirstvisits</td>";
     538        echo "<td>$Nfourvisits</td>";
     539        echo "<td>$Ndiffs</td>";
     540        echo "<td>$complete</td>";
     541
     542        echo "</tr>";
     543    }
     544    echo "</table>\n";
     545
     546}
     547
     548###########################################################################
     549#
    403550# Gets labels for this server
    404551#
    405552###########################################################################
    406 function getLabels($db, $server) {
     553function getLabels($db, $proj, $server) {
    407554
    408555    # order by descending priority as set in project database
    409     $sql = "SELECT label FROM current_labels WHERE server LIKE '$server' ORDER BY priority DESC, label";
     556    $sql = "SELECT label FROM science_labels WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY priority DESC, label";
    410557    if ($debug) {echo "$sql<br>";}
    411558
    … …  
    547694    foreach ($stages as &$stage) {
    548695
    549         $reverting = getRevertStatus($db, $stage);
     696        $reverting = getRevertStatus($db, $proj, $stage);
    550697        if(!$reverting) {$label =  "";}
    551698        if($reverting) {$label = "reverting";}
    … …  
    608755#
    609756###########################################################################
    610 function setRevertStatus($db, $stage, $reverting) {
    611 
    612     $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'";
     757function setRevertStatus($db, $proj, $stage, $reverting) {
     758
     759    $sql = "UPDATE science_reverts SET reverting = $reverting WHERE stage LIKE '$stage' AND telescope LIKE '$proj'";
    613760    if ($debug) {echo "$sql<br>";}
    614761
    … …  
    622769#
    623770###########################################################################
    624 function getNightlyScienceStatus($db) {
    625 
    626     $sql = "SELECT status FROM nightlyscience";
     771function getNightlyScienceStatus($db, $proj) {
     772
     773    $sql = "SELECT status FROM science_nightlyscience WHERE telescope LIKE '$proj'";
    627774    if ($debug) {echo "$sql<br>";}
    628775
    … …  
    639786#
    640787###########################################################################
    641 function turnRevertsOnOff($db, $stage, $mode) {
     788function turnRevertsOnOff($db, $proj, $stage, $mode) {
    642789
    643790    exec("czartool_revert.pl -t $stage -o $mode", $response, $status);
    … …  
    646793    else if ($response[0] == "on") $currentRevertMode = 1;
    647794
    648     setRevertStatus($db, $stage, $currentRevertMode);
     795    setRevertStatus($db, $proj, $stage, $currentRevertMode);
    649796}
    650797
    … …  
    654801#
    655802###########################################################################
    656 function getRevertStatus($db, $stage) {
     803function getRevertStatus($db, $proj, $stage) {
    657804
    658805    $anyFaults = false;
    659806
    660     $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'";
     807    $sql = "SELECT reverting FROM science_reverts WHERE stage LIKE '$stage' AND telescope LIKE '$proj'";
    661808    if ($debug) {echo "$sql<br>";}
    662809
    … …  
    706853#
    707854###########################################################################
    708 function createDatesTable($db) {
     855function createDatesTable($db, $proj) {
    709856
    710857    global $table;
    … …  
    717864    echo "</tr>\n";
    718865
    719     createServerDateRow($db, "stdscience");
    720     createServerDateRow($db, "registration");
     866    createServerDateRow($db, $proj, "stdscience");
     867    createServerDateRow($db, $proj, "registration");
    721868
    722869    echo "</table>\n";
    … …  
    729876#
    730877###########################################################################
    731 function createServerDateRow($db, $server) {
    732 
    733     $sql = "SELECT date FROM server_dates WHERE server LIKE '$server' ORDER BY date";
     878function createServerDateRow($db, $proj, $server) {
     879
     880    $sql = "SELECT date FROM science_server_dates WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY date";
    734881    if ($debug) {echo "$sql<br>";}
    735882
    … …  
    9821129}
    9831130
     1131###########################################################################
     1132#
     1133# Shows the status of the various replication mysql servers
     1134#
     1135###########################################################################
     1136function showDatabaseStatus($Host, $User, $Password, $DatabaseName) {
     1137    #print "<br>$replHost, $replUser, $replPassword, $replDatabaseName<br/>";
     1138    $db = DB::connect("mysql://$User:$Password@$Host");
     1139    if (PEAR::isError($db)) {
     1140        echo "<tr><td>$DatabaseName</td><td bgcolor=\"red\">MySQL DB connection error</td></tr>";
     1141        //die("MySQL DB connection error: Check the configuration for $DatabaseName");
     1142        //die("");
     1143        return -1;
     1144
     1145    }
     1146    $Status = ($errorStatusInMySql==0?"OK":"<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">PROBLEM</font>");
     1147    echo "<tr><td>$DatabaseName</td><td>$Status</td></tr>";
     1148
     1149    $db->disconnect();
     1150}
     1151
     1152###########################################################################
     1153#
     1154# Shows the status of a machine using ping
     1155#
     1156###########################################################################
     1157function showPingStatus($Host, $DatabaseName) {
     1158    exec("ping -c 1 $Host", $res, $rval);
     1159    if ($rval == 0){
     1160      $PingStatus= "UP";
     1161    }else{
     1162      $PingStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">DOWN</font>";
     1163    }
     1164    echo "<tr><td>$DatabaseName</td><td>$PingStatus</td></tr>";
     1165}
     1166
    9841167?>
    9851168
Note: See TracChangeset for help on using the changeset viewer.