IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41125 for trunk/ippMonitor


Ignore:
Timestamp:
Nov 14, 2019, 2:50:20 PM (7 years ago)
Author:
ippitc
Message:

improve loading speed

Location:
trunk/ippMonitor
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippMonitor/czartool/czarDbSchema.sql

    r40055 r41125  
    404404
    405405--
     406-- Table structure for table `science_labels` (from current_labels)
     407--
     408
     409DROP TABLE IF EXISTS `science_labels`;
     410SET @saved_cs_client     = @@character_set_client;
     411SET character_set_client = utf8;
     412CREATE TABLE `science_labels` (
     413  `server` varchar(128) default NULL,
     414  `label` varchar(128) default NULL,
     415  `telescope` varchar(128) NOT NULL,
     416  `priority` int(11) NOT NULL default '0',
     417  KEY `science_labels` (`server`,`label`, `telescope`)
     418) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     419SET character_set_client = @saved_cs_client;
     420
     421--
     422-- Table structure for table `science_nightlyscience` (from nightlyscience)
     423--
     424
     425DROP TABLE IF EXISTS `science_nightlyscience`;
     426SET @saved_cs_client     = @@character_set_client;
     427SET character_set_client = utf8;
     428CREATE TABLE `science_nightlyscience` (
     429  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     430  `status` varchar(128) default 'NONE',
     431  `telescope` varchar(128) NOT NULL
     432) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     433SET character_set_client = @saved_cs_client;
     434
     435--
     436-- Table structure for table `science_reverts` (from reverts)
     437--
     438
     439DROP TABLE IF EXISTS `science_reverts`;
     440SET @saved_cs_client     = @@character_set_client;
     441SET character_set_client = utf8;
     442CREATE TABLE `science_reverts` (
     443  `stage` varchar(128) default NULL,
     444  `telescope` varchar(128) NOT NULL,
     445  `reverting` tinyint(4) default NULL
     446) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     447SET character_set_client = @saved_cs_client;
     448
     449--
     450-- Table structure for table `science_server_dates` (from server_dates)
     451--
     452
     453DROP TABLE IF EXISTS `science_server_dates`;
     454SET @saved_cs_client     = @@character_set_client;
     455SET character_set_client = utf8;
     456CREATE TABLE `science_server_dates` (
     457  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     458  `server` varchar(64) default 'NONE',
     459  `telescope` varchar(128) NOT NULL,
     460  `date` varchar(64) default 'NONE'
     461) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     462SET character_set_client = @saved_cs_client;
     463
     464--
     465-- Table structure for table `science_servers` (from servers)
     466--
     467
     468DROP TABLE IF EXISTS `science_servers`;
     469SET @saved_cs_client     = @@character_set_client;
     470SET character_set_client = utf8;
     471CREATE TABLE `science_servers` (
     472  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     473  `server` varchar(128) default NULL,
     474  `telescope` varchar(128) NOT NULL,
     475  `alive` tinyint(4) default NULL,
     476  `running` tinyint(4) default NULL
     477) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     478SET character_set_client = @saved_cs_client;
     479
     480--
     481-- Table structure for table `live_servers` (from servers)
     482--
     483
     484DROP TABLE IF EXISTS `live_servers`;
     485SET @saved_cs_client     = @@character_set_client;
     486SET character_set_client = utf8;
     487CREATE TABLE `live_servers` (
     488  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     489  `server` varchar(128) default NULL,
     490  `telescope` varchar(128) NOT NULL,
     491  `alive` tinyint(4) default NULL,
     492  `running` tinyint(4) default NULL
     493) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     494SET character_set_client = @saved_cs_client;
     495
     496--
    406497-- Table structure for table `server_dates`
    407498--

  • trunk/ippMonitor/czartool/czarpoll.pl

    r40879 r41125  
    236236        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
    237237        $czarDb->updateServerStatus($server,'gpc2', $alive, $running);
     238   }
     239}
     240
     241###########################################################################
     242#
     243# Updates pantasks server status TODO should really get info for all servers at once
     244#
     245###########################################################################
     246sub updateServerStatusLive {
     247    print "* Checking all pantasks servers\n";
     248
     249    my $servers = $pantasks->getServerList();
     250
     251    my $server = undef;
     252    my $alive = undef;
     253    my $running = undef;
     254    foreach $server (@{$servers}) {
     255        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
     256        $czarDb->updateServerStatusLive($server,'gpc1', $alive, $running);
     257        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
     258        $czarDb->updateServerStatusLive($server,'gpc2', $alive, $running);
    238259   }
    239260}
    … …  
    272293    my $newDayTime = $config->getMetricsStartTime();
    273294    my $lastDayDailyTasks = "2010-01-01";
    274 
     295    my $checkneb = 1; # 1 for nebulous, 0 for localhost test
    275296    # main polling loop
    276297    while (1) {
     298
    277299        my $iterationStartTime = time();
    278 
     300 
    279301        #get the time of last czarpoll update
    280302        $polldate = $czarDb->getPollDate();
    … …  
    287309        if($timediff > 120.) {
    288310            print "There is no recent Czarpoll iteration\n";
    289             print "*Updating database entries*\n";
     311            print "* Updating database entries*\n";
    290312            $dbupdate = 1;
    291313        } else {
    292314            print "There is a recent Czarpoll iteration already made ($timediff sec ago)\n";
    293             print "*Skipping database queries and entries*\n";
     315            print "* Skipping database queries and entries\n";
    294316            $dbupdate = 0;
    295317        }
    … …  
    326348                #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB.
    327349                #Therefore, it makes no sense to run the cleanup+optimising twice
    328                 #print "* Performing database cleanup\n";
    329                 #$czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
    330                 #print "* Optimizing the database\n";
    331                 #$czarDb->optimize();
     350                print "* Performing database cleanup\n";
     351                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
     352                print "* Optimizing the database\n";
     353                $czarDb->optimize();
    332354                $lastDayDailyTasks = $yesterday;
    333355            }
    … …  
    344366
    345367        # check nebulous
    346         print "* Checking Nebulous\n";
    347         if($dbupdate == 1) {$nebulous->updateClusterSpaceInfo();}
    348         $plotter0->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end);
    349         $plotter0->plotDiskUsageHistogram();
    350         $plotter0->plotDiskUsageHistogramLarge();
     368        if ($checkneb == 1) {
     369            print "* Checking Nebulous\n";
     370            if($dbupdate == 1) {$nebulous->updateClusterSpaceInfo();}
     371            $plotter0->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end);
     372            $plotter0->plotDiskUsageHistogram();
     373            $plotter0->plotDiskUsageHistogramLarge();
     374        }
     375        else {
     376            print "* Locahost, so no checking nebulous\n";
     377        }
     378
     379        # check pantasks status
    351380        if($dbupdate == 1) {updateServerStatus();}
     381        if($dbupdate == 1) {updateServerStatusLive();}
    352382
    353383        # check pantasks dates

  • trunk/ippMonitor/czartool/czartool/CzarDb.pm

    r40878 r41125  
    166166SQL
    167167
     168       $query->execute;
     169}
     170
     171###########################################################################
     172#
     173# Updates server table
     174#
     175###########################################################################
     176sub updateServerStatusLive {
     177    my ($self, $server,$telescope, $alive, $running) = @_;
     178
     179    my $query = $self->{_db}->prepare(<<SQL);
     180    DELETE FROM live_servers where timestamp < CURRENT_TIMESTAMP - INTERVAL 12 hour;
     181SQL
     182       $query->execute;
     183
     184    $query = $self->{_db}->prepare(<<SQL);
     185    INSERT INTO live_servers
     186        (server,telescope, alive, running)
     187        VALUES
     188        ('$server','$telescope', $alive, $running);
     189SQL
    168190       $query->execute;
    169191}

  • trunk/ippMonitor/raw/czartool_labels.php

    r40936 r41125  
    214214echo "<td style=height:200px;width:400px;text-align:top;\">";
    215215
    216 #echo "<br>";
     216/////////////////////////////////////////
     217// timer begin for status and links
     218/////////////////////////////////////////
     219$time = microtime();
     220$time = explode(' ', $time);
     221$time = $time[1] + $time[0];
     222$start = $time;
    217223
    218224// status table at top
    219225$nsStatus = getNightlyScienceStatus($czardb, $proj);
    220226$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
    221 $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
     227$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel .
     228        "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
     229
    222230echo "<br>";
    223231echo $table;
    224 
    225 createTableTitle("Status and links", 3);
     232createTableTitle("Status and links", 2);
    226233echo "<tr>\n";
    227 echo "<tr><td>Time of last update</td><td>$lastUpdateTime</td></tr>";
    228 echo "<tr><td>ippMonitor SVN version</td><td>"; include 'version.php'; echo "</td></tr>";
    229 echo "<tr><td>Current nightly science status</td><td>$nsStatus</td></tr>";
    230 echo "<tr><td>Postage stamp server status</td><td><a href=\"http://pstamp.ipp.ifa.hawaii.edu/status.php\"><font color=\"blue\">here</font></a></td></tr>";
    231 echo "<tr><td>Documentation</td><td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\"><font color=\"blue\">here</font></a></td></tr>";
    232 echo "<tr><td>Cluster load</td><td><a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&hc=6&c=IPP%2520Production\"><font color=\"blue\">here</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=cpu_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">cpu_report</font></a> <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=mem_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\"><font color=\"blue\">mem_report</font></a></td></tr>";
    233 echo "<tr><td>IPP metrics</td><td><a href=\"https://ps1wiki.ifa.hawaii.edu/data/metrics/index.html\"><font color=\"blue\">here</font></a></td></tr>";
    234 echo "<tr><td>Czar log pages</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/pages/viewrecentblogposts.action?key=IPPCZAR\"><font color=\"blue\">here</font></a></td></tr>";
    235 echo "<tr><td>Exposure summary</td><td><a href=\"czartool_exposures.php?pass=$pass&proj=$proj\"><font color=\"blue\">here</font></a></td></tr>";
    236 echo "<tr><td>Confluence</td><td><a href=\"https://confluence.ipp.ifa.hawaii.edu/display/IPPCZAR/IPP-Czaring+Home\"><font color=\"blue\">here</font></a></td></tr>";
    237 echo "<tr><td>JIRA</td><td><a href=\"https://jira.ipp.ifa.hawaii.edu/secure/Dashboard.jspa\"><font color=\"blue\">here</font></a></td></tr>";
     234echo "<tr><td>Time of last update</td>
     235          <td>$lastUpdateTime</td></tr>";
     236echo "<tr><td>ippMonitor SVN version</td>
     237          <td>"; include 'version.php'; echo "</td></tr>";
     238echo "<tr><td>Current nightly science status</td>
     239          <td>$nsStatus</td></tr>";
     240echo "<tr><td>Postage stamp server status</td>
     241          <td><a href=\"http://pstamp.ipp.ifa.hawaii.edu/status.php\" target=new><font color=\"blue\">here</font></a></td></tr>";
     242echo "<tr><td>Documentation</td>
     243          <td><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\" target=new><font color=\"blue\">here</font></a></td></tr>";
     244echo "<tr><td>Cluster load</td>
     245          <td><a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&hc=6&c=IPP%2520Production\">
     246              <font color=\"blue\">here</font></a>
     247              <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=cpu_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\">
     248              <font color=\"blue\">cpu_report</font></a>
     249              <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?m=mem_report&r=hour&s=by%2520name&c=IPP+Production&h=&sh=1&hc=8&z=small\">
     250              <font color=\"blue\">mem_report</font></a></td></tr>";
     251echo "<tr><td>IPP metrics</td>
     252          <td><a href=\"https://ps1wiki.ifa.hawaii.edu/data/metrics/index.html\"><font color=\"blue\" target=new>here</font></a></td></tr>";
     253echo "<tr><td>Czar log pages</td>
     254          <td><a href=\"https://confluence.ipp.ifa.hawaii.edu/pages/viewrecentblogposts.action?key=IPPCZAR\" target=new><font color=\"blue\">here</font></a></td></tr>";
     255echo "<tr><td>Exposure summary</td>
     256          <td><a href=\"czartool_exposures.php?pass=$pass&proj=$proj\"><font color=\"blue\">here</font></a></td></tr>";
     257echo "<tr><td>Confluence</td>
     258          <td><a href=\"https://confluence.ipp.ifa.hawaii.edu/display/IPPCZAR/IPP-Czaring+Home\" target=new><font color=\"blue\">here</font></a></td></tr>";
     259echo "<tr><td>JIRA</td>
     260          <td><a href=\"https://jira.ipp.ifa.hawaii.edu/secure/Dashboard.jspa\" target=new><font color=\"blue\">here</font></a></td></tr>";
     261echo "<tr><td>data store</td>
     262          <td><a href=\"https://svn.ifa.hawaii.edu/\" target=new><font color=\"blue\">here</font></a></td></tr>";
    238263
    239264$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
    240 $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
    241 
    242 echo "</table>\n";
    243 
    244 echo "<br>";
    245 
    246        createLabelsTable($pass,
    247          $proj,
    248          $czardb,
    249          $selectedMode,
    250          $labels,
    251          $distLabelsExist,
    252          $pubLabelsExist,
    253          $stages,
    254          "new",
    255          $selectedLabel,
    256          $selectedStage,
    257          $plotType);
    258 
    259       if ($selectedMode == "update") {echo "<br>"; createPStampDataTable();}
    260       echo "<br>";
    261 
    262       # XXX EAM : 2017.09.17 : debugging slowness
    263       createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "<br>";
    264       createSummitDataTable($projectdb); echo "<br>"; // this is slow because of the join between summitExp and rawExp
    265       createChunkDataTable($projectdb); echo "<br>";
    266       createDatesTable($czardb, $proj); echo "<br>";
    267 
    268       echo $table;
    269       echo "<tr>";
    270       createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
    271 
    272       echo "<tr>";
    273       createTableColumnHeader("Database");
    274       createTableColumnHeader("Status (sec behind master)");
    275       showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1);
    276       showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
    277       showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY);
    278 
    279       echo "<tr>";
    280 
    281       showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS);
    282       showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
    283       showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY);
    284       showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY);
    285 
    286 #     some mysql replication slaves to add when they are returned to service
    287 #     showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP);
    288 #     showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP);
    289       echo "</table>";
    290 
    291       echo $table;
    292       echo "<tr>";
    293       createTableTitle("Critical machine status", 2);
    294 
    295       echo "<tr>";
    296       createTableColumnHeader("Hostname");
    297       createTableColumnHeader("Status");
    298       showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY);
    299       showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY);
    300 
    301 #      showPingStatus($HOST_APACHE1, $DBNAME_APACHE1);
    302       showPingStatus($HOST_APACHE2, $DBNAME_APACHE2);
    303       showPingStatus($HOST_APACHE3, $DBNAME_APACHE3);
    304       showPingStatus($HOST_APACHE4, $DBNAME_APACHE4);
    305       showPingStatus($HOST_APACHE5, $DBNAME_APACHE5);
    306       showPingStatus($HOST_APACHE6, $DBNAME_APACHE6);
    307       showPingStatus($HOST_APACHE7, $DBNAME_APACHE7);
    308       showPingStatus($HOST_PROXY, $DBNAME_PROXY);
    309       showPingStatus($HOST_SVN, $DBNAME_SVN);
    310       showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE);
    311 
    312       echo "</table>";
    313 
    314 echo "</table>";
     265$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode .
     266        "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
     267
     268// timer finished
     269echo "<tr><th colspan=4>";
     270$time = microtime();
     271$time = explode(' ', $time);
     272$time = $time[1] + $time[0];
     273$finish = $time;
     274$total_time = round(($finish - $start), 3);
     275echo "loading in $total_time seconds.";
     276echo "</th></tr>";
     277echo "</table><br>";
     278
     279createLabelsTable($pass, $proj, $czardb, $selectedMode, $labels,
     280  $distLabelsExist, $pubLabelsExist, $stages, "new", $selectedLabel,
     281  $selectedStage, $plotType);
     282if ($selectedMode == "update") {echo "<br>"; createPStampDataTable();}
     283
     284# XXX EAM : 2017.09.17 : debugging slowness
     285# CCL : 2019.11.09 : debugging slowness
     286//createSummitDataTable($projectdb); // this is slow because of the join between summitExp and rawExp, but not the major one
     287createSummitDataTable2($projectdb); // this is slow because of the join between summitExp and rawExp, but not the major one
     288
     289createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType);
     290// this was the major slowness part due to large table in czardb
     291
     292createChunkDataTable($projectdb);  // this is another part for slowness during nightly processing
     293
     294createDatesTable($czardb, $proj);
     295
     296/////////////////////////////////////////
     297// timer begin for Database status
     298/////////////////////////////////////////
     299$time = microtime();
     300$time = explode(' ', $time);
     301$time = $time[1] + $time[0];
     302$start = $time;
     303
     304// status table at top
     305  echo $table;
     306  echo "<tr>";
     307  createTableTitle("Database status (update fix log <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Replication_Issues\">here</a>)", 2);
     308 
     309  echo "<tr>";
     310  createTableColumnHeader("Database");
     311  createTableColumnHeader("Status (sec behind master)");
     312  showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1);
     313  showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
     314  showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY);
     315 
     316  echo "<tr>";
     317  showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS);
     318  showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
     319  showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY);
     320  showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY);
     321 
     322//  some mysql replication slaves to add when they are returned to service
     323//  showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP);
     324//  showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP);
     325
     326// timer finished
     327echo "<tr><th colspan=2>";
     328$time = microtime();
     329$time = explode(' ', $time);
     330$time = $time[1] + $time[0];
     331$finish = $time;
     332$total_time = round(($finish - $start), 3);
     333echo "loading in $total_time seconds.";
     334echo "</th></tr>";
     335echo "</table><br>";
     336
     337/////////////////////////////////////////
     338// timer begin for critical machines
     339/////////////////////////////////////////
     340$time = microtime();
     341$time = explode(' ', $time);
     342$time = $time[1] + $time[0];
     343$start = $time;
     344
     345// status table at top
     346  $PingStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">DOWN</font>";
     347  echo $table;
     348  echo "<tr>";
     349  createTableTitle("Critical machine status", 3);
     350
     351  echo "<tr>";
     352  createTableColumnHeader("Hostname");
     353  createTableColumnHeader("Status");
     354  createTableColumnHeader("Loading time");
     355  showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY);
     356  showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY);
     357  showPingStatus($HOST_APACHE1, $DBNAME_APACHE1);
     358  showPingStatus($HOST_APACHE2, $DBNAME_APACHE2);
     359  showPingStatus($HOST_APACHE3, $DBNAME_APACHE3);
     360  showPingStatus($HOST_APACHE4, $DBNAME_APACHE4);
     361  showPingStatus($HOST_APACHE5, $DBNAME_APACHE5);
     362  showPingStatus($HOST_APACHE6, $DBNAME_APACHE6);
     363  showPingStatus($HOST_APACHE7, $DBNAME_APACHE7);
     364  showPingStatus($HOST_APACHE8, $DBNAME_APACHE8);
     365//     echo "<td>ippc70 (apache node) <td> $PingStatus <td> 0 seconds ";  // comment out if ippc70 is back
     366  showPingStatus($HOST_PROXY, $DBNAME_PROXY);
     367  showPingStatus($HOST_SVN, $DBNAME_SVN);
     368  showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE);
     369
     370// timer finished
     371echo "<tr><th colspan=3>";
     372$time = microtime();
     373$time = explode(' ', $time);
     374$time = $time[1] + $time[0];
     375$finish = $time;
     376$total_time = round(($finish - $start), 3);
     377echo "loading in $total_time seconds.";
     378echo "</th></tr>";
     379echo "</table><br>";
     380   
     381
    315382menu_end();
    316383
     384
     385#############################################################################
     386#############################  Functions Below  #############################
     387#############################################################################
    317388
    318389###########################################################################
    … …  
    361432}
    362433
    363 ###########################################################################
    364 #
    365 # Creates a table of summitExp against rawExp
    366 #
    367 ###########################################################################
    368 function createSummitDataTable($projectdb) {
    369 
    370     $date = date("Y-m-d");
    371     $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'";
    372 
    373     $qry = $projectdb->query($sql);
    374     if (dberror($qry)) {
    375         echo "<b>error reading newExp table</b><br>\n";
    376         echo "<br><small><b> table query : $sql </b></small><br>\n";
    377         menu_end();
    378     }
    379 
    380     // set up the table
    381     global $table;
    382     echo $table;
    383     echo "<tr>";
    384     createTableTitle("Status of last night's data", 4);
    385     echo "<tr>";
    386     createTableColumnHeader("Exposure type");
    387     createTableColumnHeader("At summit");
    388     createTableColumnHeader("Registered at MHPCC");
    389     createTableColumnHeader("At summit (Dropped)");
    390 
    391     $msg = "No science images taken since $date";
    392 
    393     // list the results
    394     while ($qry->fetchInto($expType)) {
    395 
    396         $sql = "SELECT COUNT(*) FROM summitExp left join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND exp_type = '$expType[0]' and (state != 'drop' or state is null) and fault != 1042";
    397 #       $sql = "SELECT COUNT(*) FROM summitExp WHERE dateobs > '$date' AND exp_type = '$expType[0]'";
    398         $qry2 = $projectdb->query($sql);
    399         $qry2->fetchInto($summit);
    400 
    401 
    402         $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name WHERE summitExp.dateobs > '$date' AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'";
    403         $qry2 = $projectdb->query($sql);
    404         $qry2->fetchInto($mhpcc);
    405 
    406         $sql = "SELECT COUNT(*) FROM summitExp join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND exp_type = '$expType[0]' and state = 'drop'";
    407         $qry2 = $projectdb->query($sql);
    408         $qry2->fetchInto($dropped);
    409 
    410 
    411         #Query with 30 minutes delay to check during the night if any exposures are missing.
    412         $datehour = gmdate("Y-m-d H:i:s");
    413         $sql = "SELECT COUNT(*) FROM summitExp left join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND dateobs <= date_sub('$datehour', interval 30 minute) AND exp_type = '$expType[0]' and (state != 'drop' or state is null) and fault != 1042";
    414         $qry2 = $projectdb->query($sql);
    415         $qry2->fetchInto($summitdelay);
    416 
    417         $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name WHERE summitExp.dateobs > '$date' AND summitExp.dateobs <= date_sub('$datehour', interval 30 minute) AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'";
    418         $qry2 = $projectdb->query($sql);
    419         $qry2->fetchInto($mhpccdelay);
    420 
    421         echo "<tr>";
    422 
    423         $DLdiff = $summit[0] - $mhpcc[0];
    424         if ($DLdiff == 0) $error = 0;
    425         else $error = 1;
    426 
    427         $delayDLdiff = $summitdelay[0] - $mhpccdelay[0];
    428         if ($delayDLdiff == 0) $errordelay = 0;
    429         else $errordelay = 1;
    430 
    431         echo "<td>$expType[0]</td>";
    432         echo "<td>$summit[0]</td>";
    433         #createFormattedTableCell("", "", $mhpcc[0], $error, 0);
    434         if ($delayDLdiff==0) {
    435            createFormattedTableCell("", "", $mhpcc[0], $error, 0);
    436         } else {
    437            $DLStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">$mhpcc[0] ($delayDLdiff)</font>";
    438            echo "<td>$DLStatus</td>";
    439         }
    440 
    441         echo "<td>$dropped[0]</td>";
    442 
    443         echo "</tr>";
    444     }
    445 
    446     echo "</table>\n";
    447 
    448 }
    449434
    450435###########################################################################
    … …  
    455440function createChunkDataTable($projectdb) {
    456441
     442    // timer start
     443    $time = microtime();
     444    $time = explode(' ', $time);
     445    $time = $time[1] + $time[0];
     446    $start = $time;
     447
     448    // function begin
    457449    #First find the reference exposure ID to speed up the subsequent queries
    458450    $date = gmdate("Y-m-d");
    … …  
    592584        echo "</tr>";
    593585    }
    594     echo "</table>\n";
     586
     587
     588    // timer finished
     589    echo "<tr><th colspan=5>";
     590    $time = microtime();
     591    $time = explode(' ', $time);
     592    $time = $time[1] + $time[0];
     593    $finish = $time;
     594    $total_time = round(($finish - $start), 3);
     595    echo "loading in $total_time seconds.";
     596    $start= $finish;
     597    echo "</th></tr>";
     598    echo "</table></br>";
    595599
    596600}
    … …  
    634638        $selectedLabel, $selectedStage, $plotType) {
    635639
     640    // timer start
     641    $time = microtime();
     642    $time = explode(' ', $time);
     643    $time = $time[1] + $time[0];
     644    $start = $time;
     645
     646    // function begin
    636647    $isUpdate = ($selectedMode == "update");
    637648    global $headerColor;
    … …  
    757768
    758769    echo "</tr>\n";
    759     echo "</table>\n";
     770
     771    // timer finished
     772    echo "<tr><th colspan=11>";
     773    $time = microtime();
     774    $time = explode(' ', $time);
     775    $time = $time[1] + $time[0];
     776    $finish = $time;
     777    $total_time = round(($finish - $start), 3);
     778    echo "loading in $total_time seconds.";
     779    $start= $finish;
     780    echo "</th></tr>";
     781    echo "</table><br>";
    760782}
    761783
    … …  
    905927function createDatesTable($db, $proj) {
    906928
     929    // timer start
     930    $time = microtime();
     931    $time = explode(' ', $time);
     932    $time = $time[1] + $time[0];
     933    $start = $time;
     934
     935    // function begin
    907936    global $table;
    908937    echo $table;
    … …  
    917946    createServerDateRow($db, $proj, "registration");
    918947
    919     echo "</table>\n";
     948    // timer finished
     949    echo "<tr><th colspan=11>";
     950    $time = microtime();
     951    $time = explode(' ', $time);
     952    $time = $time[1] + $time[0];
     953    $finish = $time;
     954    $total_time = round(($finish - $start), 3);
     955    echo "loading in $total_time seconds.";
     956    $start= $finish;
     957    echo "</th></tr>";
     958    echo "</table><br>";
     959
    920960}
    921961
    … …  
    959999function createServersTable($pass, $proj, $selectedMode, $db, $servers, $selectedLabel, $selectedStage, $plotType) {
    9601000
     1001    // timer start
     1002    $time = microtime();
     1003    $time = explode(' ', $time);
     1004    $time = $time[1] + $time[0];
     1005    $start = $time;
     1006
     1007    // function begin
    9611008    // set up table columns
    9621009    global $table;
    … …  
    10451092    }
    10461093
    1047     echo "</table>\n";
     1094    // timer finished
     1095    echo "<tr><th colspan=5>";
     1096    $time = microtime();
     1097    $time = explode(' ', $time);
     1098    $time = $time[1] + $time[0];
     1099    $finish = $time;
     1100    $total_time = round(($finish - $start), 3);
     1101    echo "loading in $total_time seconds.";
     1102    $start= $finish;
     1103    echo "</th></tr>";
     1104    echo "</table><br>\n";
    10481105}
    10491106
    … …  
    10871144function getServerStatus($db,$proj, $server, &$alive, &$running) {
    10881145
    1089     $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
     1146#    $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
     1147    $sql = "SELECT alive, running FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1";
    10901148    if($debug){echo "$sql<br>";}
    10911149
    … …  
    12061264###########################################################################
    12071265function showPingStatus($Host, $DatabaseName) {
     1266    // timer start
     1267    $time = microtime();
     1268    $time = explode(' ', $time);
     1269    $time = $time[1] + $time[0];
     1270    $start = $time;
     1271
     1272    // function begin
    12081273    exec("ping -c 1 $Host", $res, $rval);
    12091274    if ($rval == 0){
    … …  
    12121277      $PingStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">DOWN</font>";
    12131278    }
    1214     echo "<tr><td>$DatabaseName</td><td>$PingStatus</td></tr>";
    1215 }
    1216 
     1279    echo "<tr><td>$DatabaseName</td>";
     1280
     1281    echo "</td><td>$PingStatus</td>";
     1282    // timer finished
     1283    echo "<td>";
     1284    $time = microtime();
     1285    $time = explode(' ', $time);
     1286    $time = $time[1] + $time[0];
     1287    $finish = $time;
     1288    $total_time = round(($finish - $start), 3);
     1289    echo "$total_time seconds";
     1290    $start= $finish;
     1291    echo "</td></tr>";
     1292}
     1293
     1294
     1295###########################################################################
     1296#
     1297# Creates a table of summitExp against rawExp
     1298#
     1299###########################################################################
     1300function createSummitDataTable($projectdb) {
     1301
     1302    // timer start
     1303    $time = microtime();
     1304    $time = explode(' ', $time);
     1305    $time = $time[1] + $time[0];
     1306    $start = $time;
     1307
     1308    // function begin
     1309    $date = date("Y-m-d");
     1310    $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'";
     1311
     1312    $qry = $projectdb->query($sql);
     1313    if (dberror($qry)) {
     1314        echo "<b>error reading newExp table</b><br>\n";
     1315        echo "<br><small><b> table query : $sql </b></small><br>\n";
     1316        menu_end();
     1317    }
     1318
     1319    // set up the table
     1320    global $table;
     1321    echo $table;
     1322    echo "<tr>";
     1323    createTableTitle("Data status > $date", 4);
     1324    echo "<tr>";
     1325    createTableColumnHeader("Exposure type");
     1326    createTableColumnHeader("At summit");
     1327    createTableColumnHeader("Registered at MHPCC");
     1328    createTableColumnHeader("Summit (Drop/Null)");
     1329
     1330    $msg = "No science images taken since $date";
     1331
     1332    // list the results
     1333    while ($qry->fetchInto($expType)) {
     1334
     1335        $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id)
     1336                WHERE dateobs > '$date' AND exp_type = '$expType[0]'";
     1337             //   AND state != 'drop' AND state is not null AND fault != 1042";
     1338        $qry2 = $projectdb->query($sql);
     1339        $qry2->fetchInto($summit);
     1340
     1341        $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name
     1342                WHERE summitExp.dateobs > '$date' AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'";
     1343        $qry2 = $projectdb->query($sql);
     1344        $qry2->fetchInto($mhpcc);
     1345
     1346        $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id)
     1347                WHERE dateobs > '$date' AND exp_type = '$expType[0]' AND (state = 'drop' OR state is null)";
     1348        $qry2 = $projectdb->query($sql);
     1349        $qry2->fetchInto($dropped);
     1350
     1351
     1352        #Query with 30 minutes delay to check during the night if any exposures are missing.
     1353        $datehour = gmdate("Y-m-d H:i:s");
     1354        $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id)
     1355                WHERE dateobs > '$date' AND dateobs <= date_sub('$datehour', interval 30 minute)
     1356                AND exp_type = '$expType[0]' AND (state != 'drop' OR state is null) and fault != 1042";
     1357
     1358        $qry2 = $projectdb->query($sql);
     1359        $qry2->fetchInto($summitdelay);
     1360
     1361        $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name
     1362                WHERE summitExp.dateobs > '$date' AND summitExp.dateobs <= date_sub('$datehour', interval 30 minute)
     1363                AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'";
     1364        $qry2 = $projectdb->query($sql);
     1365        $qry2->fetchInto($mhpccdelay);
     1366
     1367        echo "<tr>";
     1368
     1369        $DLdiff = $summit[0] - $mhpcc[0];
     1370        if ($DLdiff == 0) $error = 0;
     1371        else $error = 1;
     1372
     1373        $delayDLdiff = $summitdelay[0] - $mhpccdelay[0];
     1374        if ($delayDLdiff == 0) $errordelay = 0;
     1375        else $errordelay = 1;
     1376
     1377        echo "<td>$expType[0]</td>";
     1378        echo "<td>$summit[0]</td>";
     1379        if ($delayDLdiff==0) {
     1380           createFormattedTableCell("", "", $mhpcc[0], $error, 0);
     1381        } else {
     1382           $DLStatus = "<font style=\"BACKGROUND-COLOR: yellow\"  color=\"red\">$mhpcc[0] ($delayDLdiff)</font>";
     1383           echo "<td>$DLStatus</td>";
     1384        }
     1385
     1386        echo "<td>$dropped[0]</td>";
     1387
     1388        echo "</tr>";
     1389    }
     1390
     1391    // timer finished
     1392    echo "<tr><th colspan=4>";
     1393    $time = microtime();
     1394    $time = explode(' ', $time);
     1395    $time = $time[1] + $time[0];
     1396    $finish = $time;
     1397    $total_time = round(($finish - $start), 3);
     1398    echo "loading in $total_time seconds.";
     1399    $start= $finish;
     1400    echo "</th></tr>";
     1401    echo "</table><br>";
     1402}
     1403
     1404
     1405function createSummitDataTable2($projectdb) {
     1406
     1407    // timer start
     1408    $time = microtime();
     1409    $time = explode(' ', $time);
     1410    $time = $time[1] + $time[0];
     1411    $start = $time;
     1412
     1413    // function begin
     1414    $date = date("Y-m-d");
     1415    // set up the table
     1416    global $table;
     1417    global $columnHeaderColor;
     1418    echo $table;
     1419    echo "<tr>";
     1420    createTableTitle("Data status > $date", 4);
     1421    echo "<tr>";
     1422    createTableColumnHeader("Exp_type at summit");
     1423    createTableColumnHeader("Status at summit");
     1424    createTableColumnHeader("Status at mhpcc");
     1425    createTableColumnHeader("Number");
     1426    $sql = "SELECT summitExp.exp_type,pzDownloadExp.state,rawExp.state,COUNT(*) FROM summitExp
     1427            LEFT JOIN pzDownloadExp USING (summit_id) LEFT JOIN rawExp on summitExp.exp_name = rawExp.exp_name
     1428            WHERE summitExp.dateobs > '$date' group by summitExp.exp_type,pzDownloadExp.state,rawExp.state";
     1429    $qry = $projectdb->query($sql);
     1430    while ($qry->fetchInto($tmp)) {
     1431        if ($tmp[1] == NULL) {
     1432          echo  "<tr><td>$tmp[0]<td> null <td> null <td bgcolor=\"yellow\"> <font color=\"red\">$tmp[3]";
     1433        }
     1434        else {
     1435          if ($tmp[1] == 'run') {
     1436            echo  "<tr><td>$tmp[0]<td> loading <td> null <td bgcolor=$columnHeaderColor> <font color=\"red\">$tmp[3]";
     1437          }
     1438          else {
     1439            if ($tmp[2] == NULL) {
     1440              echo  "<tr><td>$tmp[0] <td> $tmp[1] <td> loading <td bgcolor=$columnHeaderColor> <font color=\"blue\">$tmp[3]";
     1441            }
     1442            else {
     1443              echo  "<tr><td>$tmp[0]<td> $tmp[1] <td> downloaded <td> $tmp[3]";
     1444            }
     1445          }
     1446        }
     1447    }
     1448
     1449    // timer finished
     1450    echo "<tr><th colspan=4>";
     1451    $time = microtime();
     1452    $time = explode(' ', $time);
     1453    $time = $time[1] + $time[0];
     1454    $finish = $time;
     1455    $total_time = round(($finish - $start), 3);
     1456    echo "loading in $total_time seconds.";
     1457    $start= $finish;
     1458    echo "</th></tr>";
     1459    echo "</table><br>";
     1460}
    12171461?>
    12181462
Note: See TracChangeset for help on using the changeset viewer.