Index: trunk/ippMonitor/raw/czartool_labels.php =================================================================== --- trunk/ippMonitor/raw/czartool_labels.php (revision 40427) +++ trunk/ippMonitor/raw/czartool_labels.php (revision 40840) @@ -90,5 +90,4 @@ # "publishing", "registration", - "replication", "stack", # "deepstack", @@ -139,6 +138,6 @@ // $labels is generated with sequential integer key values -$labels = getLabels($czardb, $selectedMode); -$distLabelsIndex = getLabels($czardb, "distribution"); +$labels = getLabels($czardb, $proj, $selectedMode); +$distLabelsIndex = getLabels($czardb, $proj, "distribution"); for ($i = 0; $i < count($distLabelsIndex); $i++) { $distLabelsExist[$distLabelsIndex[$i]] = true; @@ -195,9 +194,9 @@ echo ""; # time series plot -echo "

"; +echo "

"; +# log time series plot +echo "

"; # rate time series plot echo "

"; - # rate stacked histo plot -# echo "
"; # histogram plot # echo "
"; @@ -218,5 +217,5 @@ // status table at top -$nsStatus = getNightlyScienceStatus($czardb); +$nsStatus = getNightlyScienceStatus($czardb, $proj); $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; @@ -232,11 +231,9 @@ echo "Documentationhere"; echo "Cluster loadhere cpu_report mem_report"; -echo "Who's using the cluster?here"; -echo "Files monitorhere"; -echo "IPP metricshere"; -echo "Czar log pagesAll   "; -echo "This week"; -echo ""; +echo "IPP metricshere"; +echo "Czar log pageshere"; echo "Exposure summaryhere"; +echo "Confluencehere"; +echo "JIRAhere"; $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; @@ -266,25 +263,52 @@ createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "
"; createSummitDataTable($projectdb); echo "
"; // this is slow because of the join between summitExp and rawExp - createDatesTable($czardb); echo "
"; - - # This is awful + createChunkDataTable($projectdb); echo "
"; + createDatesTable($czardb, $proj); echo "
"; + echo $table; echo ""; - createTableTitle("Database replication status (update fix log here)", 2); + createTableTitle("Database status (update fix log here)", 2); echo ""; createTableColumnHeader("Database"); createTableColumnHeader("Status (sec behind master)"); + showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1); showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY); + echo ""; + + showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS); showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY); + showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY); # some mysql replication slaves to add when they are returned to service # showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP); -# showReplicationsStatus($REPL_HOST_IPPADMIN, $REPL_USER_IPPADMIN, $REPL_PASSWORD_IPPADMIN, $REPL_DBNAME_IPPADMIN); --- this one is in GPC1 # showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP); echo ""; + + echo $table; + echo ""; + createTableTitle("Critical machine status", 2); + + echo ""; + createTableColumnHeader("Hostname"); + createTableColumnHeader("Status"); + showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY); + showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY); + +# showPingStatus($HOST_APACHE1, $DBNAME_APACHE1); + showPingStatus($HOST_APACHE2, $DBNAME_APACHE2); + showPingStatus($HOST_APACHE3, $DBNAME_APACHE3); + showPingStatus($HOST_APACHE4, $DBNAME_APACHE4); + showPingStatus($HOST_APACHE5, $DBNAME_APACHE5); + showPingStatus($HOST_APACHE6, $DBNAME_APACHE6); + showPingStatus($HOST_PROXY, $DBNAME_PROXY); + showPingStatus($HOST_SVN, $DBNAME_SVN); + showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE); + + echo ""; + echo ""; menu_end(); @@ -401,11 +425,134 @@ ########################################################################### # +# Creates a table of chunk progress +# +########################################################################### +function createChunkDataTable($projectdb) { + + #First find the reference exposure ID to speed up the subsequent queries + $date = gmdate("Y-m-d"); + $sql = "SELECT MAX(exp_id) FROM rawExp WHERE dateobs >= date_sub('$date', interval 20 day) AND dateobs <= date_sub('$date', interval 10 day)"; + $qry = $projectdb->query($sql); + if (dberror($qry)) { + print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n"; + $refExpID = 0; + } + $qry->fetchInto($refExpID); + + #Find the number of chunks + $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM"; + $query .= " ("; + $query .= " SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0]"; + $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND comment LIKE '%visit 1%'"; + $query .= " GROUP BY chunk"; + $query .= " ) AS groupchunk"; + $query .= " LEFT JOIN"; + $query .= " ("; + $query .= " SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'"; + $query .= " GROUP BY object, filter, chunk"; + $query .= " ) AS groupobj"; + $query .= " WHERE groupobj.Nvisits = 4 "; + $query .= " GROUP BY groupobj.chunk"; + $query .= " ) AS group4"; + $query .= " on groupchunk.chunk=group4.chunk ORDER BY dateobs"; + + #$sql = "SELECT DISTINCT substr(comment, 1, position(' ' in comment)) FROM rawExp WHERE dateobs LIKE '$date%' AND exp_id > $refExpID AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit 1%'"; + + $qry = $projectdb->query($query); + if (dberror($qry)) { + echo "error reading rawExp table
\n"; + echo "
table query : $query
\n"; + menu_end(); + } + + // set up the table + global $table; + echo $table; + echo ""; + createTableTitle("Status of last night's chunks", 5); + echo ""; + createTableColumnHeader("Chunk"); + createTableColumnHeader("Nobs_1stvisit"); + createTableColumnHeader("Nobs_4visits"); + createTableColumnHeader("Ndiffs_published"); + createTableColumnHeader("Completion"); + + $msg = "No science images taken since $date"; + + // list the results + while ($qry->fetchInto($row)) { + $Nfirstvisits = 0; + $Nfourvisits = 0; + + $chunkname = $row[0]; + if($row[1] > 0) $Nfirstvisits = $row[1]; + if($row[2] > 0) $Nfourvisits = $row[2]; + + $query2 = " SELECT warp_id FROM "; + $query2 .= " warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query2 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'"; + $qry2 = $projectdb->query($query2); + + $Ndiffs = 0; + while ($qry2->fetchInto($row2)) { + $query3 = " SELECT diff_id FROM "; + $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)"; + $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; + $qry3 = $projectdb->query($query3); + while ($qry3->fetchInto($row3)) { + if ($row3[0] !=0) $Ndiffs ++; + } + + $query3 = " SELECT diff_id FROM "; + $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)"; + $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; + $qry3 = $projectdb->query($query3); + while ($qry3->fetchInto($row3)) { + if ($row3[0] !=0) $Ndiffs ++; + } + } + + if ($Ndiffs != ($Nfirstvisits*4)) $complete = "chunk not done"; + if ($Ndiffs == ($Nfirstvisits*4)) $complete = "chunk done"; + + if ($Nfirstvisits != $Nfourvisits) { + $Ndiffs .= "/? "; + } + if ($Nfirstvisits == $Nfourvisits) { + $Ndiffs .= "/"; + $Ndiffs .= $Nfirstvisits*4; + } + + echo ""; + + echo "$chunkname"; + echo "$Nfirstvisits"; + echo "$Nfourvisits"; + echo "$Ndiffs"; + echo "$complete"; + + echo ""; + } + echo "\n"; + +} + +########################################################################### +# # Gets labels for this server # ########################################################################### -function getLabels($db, $server) { +function getLabels($db, $proj, $server) { # order by descending priority as set in project database - $sql = "SELECT label FROM current_labels WHERE server LIKE '$server' ORDER BY priority DESC, label"; + $sql = "SELECT label FROM science_labels WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY priority DESC, label"; if ($debug) {echo "$sql
";} @@ -547,5 +694,5 @@ foreach ($stages as &$stage) { - $reverting = getRevertStatus($db, $stage); + $reverting = getRevertStatus($db, $proj, $stage); if(!$reverting) {$label = "";} if($reverting) {$label = "reverting";} @@ -608,7 +755,7 @@ # ########################################################################### -function setRevertStatus($db, $stage, $reverting) { - - $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'"; +function setRevertStatus($db, $proj, $stage, $reverting) { + + $sql = "UPDATE science_reverts SET reverting = $reverting WHERE stage LIKE '$stage' AND telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -622,7 +769,7 @@ # ########################################################################### -function getNightlyScienceStatus($db) { - - $sql = "SELECT status FROM nightlyscience"; +function getNightlyScienceStatus($db, $proj) { + + $sql = "SELECT status FROM science_nightlyscience WHERE telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -639,5 +786,5 @@ # ########################################################################### -function turnRevertsOnOff($db, $stage, $mode) { +function turnRevertsOnOff($db, $proj, $stage, $mode) { exec("czartool_revert.pl -t $stage -o $mode", $response, $status); @@ -646,5 +793,5 @@ else if ($response[0] == "on") $currentRevertMode = 1; - setRevertStatus($db, $stage, $currentRevertMode); + setRevertStatus($db, $proj, $stage, $currentRevertMode); } @@ -654,9 +801,9 @@ # ########################################################################### -function getRevertStatus($db, $stage) { +function getRevertStatus($db, $proj, $stage) { $anyFaults = false; - $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'"; + $sql = "SELECT reverting FROM science_reverts WHERE stage LIKE '$stage' AND telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -706,5 +853,5 @@ # ########################################################################### -function createDatesTable($db) { +function createDatesTable($db, $proj) { global $table; @@ -717,6 +864,6 @@ echo "\n"; - createServerDateRow($db, "stdscience"); - createServerDateRow($db, "registration"); + createServerDateRow($db, $proj, "stdscience"); + createServerDateRow($db, $proj, "registration"); echo "\n"; @@ -729,7 +876,7 @@ # ########################################################################### -function createServerDateRow($db, $server) { - - $sql = "SELECT date FROM server_dates WHERE server LIKE '$server' ORDER BY date"; +function createServerDateRow($db, $proj, $server) { + + $sql = "SELECT date FROM science_server_dates WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY date"; if ($debug) {echo "$sql
";} @@ -982,4 +1129,40 @@ } +########################################################################### +# +# Shows the status of the various replication mysql servers +# +########################################################################### +function showDatabaseStatus($Host, $User, $Password, $DatabaseName) { + #print "
$replHost, $replUser, $replPassword, $replDatabaseName
"; + $db = DB::connect("mysql://$User:$Password@$Host"); + if (PEAR::isError($db)) { + echo "$DatabaseNameMySQL DB connection error"; + //die("MySQL DB connection error: Check the configuration for $DatabaseName"); + //die(""); + return -1; + + } + $Status = ($errorStatusInMySql==0?"OK":"PROBLEM"); + echo "$DatabaseName$Status"; + + $db->disconnect(); +} + +########################################################################### +# +# Shows the status of a machine using ping +# +########################################################################### +function showPingStatus($Host, $DatabaseName) { + exec("ping -c 1 $Host", $res, $rval); + if ($rval == 0){ + $PingStatus= "UP"; + }else{ + $PingStatus = "DOWN"; + } + echo "$DatabaseName$PingStatus"; +} + ?>