Index: trunk/ippMonitor/Makefile.in =================================================================== --- trunk/ippMonitor/Makefile.in (revision 40834) +++ trunk/ippMonitor/Makefile.in (revision 40840) @@ -165,4 +165,9 @@ $(DESTWWW)/warpFailedSkyfiles.php \ $(DESTWWW)/failedWarpSkyfiles.php \ +$(DESTWWW)/fullforceFailedSkyfile.php \ +$(DESTWWW)/fullforceProcessedSkyfile.php \ +$(DESTWWW)/fullforceRun.php \ +$(DESTWWW)/fullforceSummary.php \ +$(DESTWWW)/fullforce_failure.php \ $(DESTWWW)/failedFullForceSkyfiles.php \ $(DESTWWW)/diffSummary.php \ @@ -172,4 +177,5 @@ $(DESTWWW)/diffProcessedSkyfile.php \ $(DESTWWW)/diffProcessedSkyfile_Images.php \ +$(DESTWWW)/diffFailedSkyfile.php \ $(DESTWWW)/failedDiffSkyfile.php \ $(DESTWWW)/stackRun.php \ Index: trunk/ippMonitor/config.dat.in =================================================================== --- trunk/ippMonitor/config.dat.in (revision 40834) +++ trunk/ippMonitor/config.dat.in (revision 40840) @@ -1,6 +1,19 @@ +// host and user information for the main processing database (contains, e.g., gpc1, gpc2, etc) $DBHOST = ""; $DBUSER = ""; $DBPASS = ""; + + +// location of nebulous apache interface machine +$NEB_SERVER = ""; + +// Czartool configuration variables +// NOTE: These should match the configuration file used by the czartool perl modules: czartool/czarconfig.xml +$CZARDBNAME = ""; +$CZARDBHOST = ""; + +// location of plots produced by czarpoll.pl +$CZARPLOTDIR = ""; // Temporary files belonging to DELETION_USER and older than @@ -9,33 +22,84 @@ $DELETION_USER = ""; -$DELETION_DELAY = 15; +$DELETION_DELAY =5; -$REPL_HOST_IPPADMIN = ""; -$REPL_USER_IPPADMIN = ""; -$REPL_PASSWORD_IPPADMIN = ""; -$REPL_DBNAME_IPPADMIN = "" ; +$HOST_NEBULOUS = ""; +$USER_NEBULOUS = ""; +$PASSWORD_NEBULOUS = ""; +$DBNAME_NEBULOUS = ""; -$REPL_HOST_NEBULOUS = ""; -$REPL_USER_NEBULOUS = ""; +$REPL_HOST_NEBULOUS = ""; +$REPL_USER_NEBULOUS = ""; $REPL_PASSWORD_NEBULOUS = ""; -$REPL_DBNAME_NEBULOUS = ""; +$REPL_DBNAME_NEBULOUS = ""; -$REPL_HOST_PSTAMP = ""; -$REPL_USER_PSTAMP = ""; +$REPL_HOST_NEBULOUS_SECONDARY = ""; +$REPL_USER_NEBULOUS_SECONDARY = ""; +$REPL_PASSWORD_NEBULOUS_SECONDARY = ""; +$REPL_DBNAME_NEBULOUS_SECONDARY = ""; + +$REPL_HOST_PSTAMP = ""; +$REPL_USER_PSTAMP = ""; $REPL_PASSWORD_PSTAMP = ""; -$REPL_DBNAME_PSTAMP = ""; +$REPL_DBNAME_PSTAMP = ""; -$REPL_HOST_GPC1 = ""; -$REPL_USER_GPC1 = ""; +$REPL_HOST_ISP = ""; +$REPL_USER_ISP = ""; +$REPL_PASSWORD_ISP = ""; +$REPL_DBNAME_ISP = ""; + +$HOST_GPC1 = ""; +$USER_GPC1 = ""; +$PASSWORD_GPC1 = ""; +$DBNAME_GPC1 = ""; + +$REPL_HOST_GPC1 = ""; +$REPL_USER_GPC1 = ""; $REPL_PASSWORD_GPC1 = ""; -$REPL_DBNAME_GPC1 = ""; +$REPL_DBNAME_GPC1 = ""; -$REPL_HOST_ISP = ""; -$REPL_USER_ISP = ""; -$REPL_PASSWORD_ISP = ""; -$REPL_DBNAME_ISP = ""; +$REPL_HOST_GPC1_SECONDARY = ""; +$REPL_USER_GPC1_SECONDARY = ""; +$REPL_PASSWORD_GPC1_SECONDARY = ""; +$REPL_DBNAME_GPC1_SECONDARY = ""; -$REPL_HOST_NEBULOUS_SECONDARY = ""; -$REPL_USER_NEBULOUS_SECONDARY = ""; -$REPL_PASSWORD_NEBULOUS_SECONDARY = ""; -$REPL_DBNAME_NEBULOUS_SECONDARY = ""; +$REPL_HOST_DATASTORE_SECONDARY = ""; +$REPL_USER_DATASTORE_SECONDARY = ""; +$REPL_PASSWORD_DATASTORE_SECONDARY = ""; +$REPL_DBNAME_DATASTORE_SECONDARY = ""; + +//Important machines that should be checked for accessibiltiy +$HOST_GATEWAY = ""; +$DBNAME_GATEWAY = ""; + +$HOST_GATEWAY_SECONDARY = ""; +$DBNAME_GATEWAY_SECONDARY = ""; + +$HOST_APACHE1 = ""; +$DBNAME_APACHE1 = ""; + +$HOST_APACHE2 = ""; +$DBNAME_APACHE2 = ""; + +$HOST_APACHE3 = ""; +$DBNAME_APACHE3 = ""; + +$HOST_APACHE4 = ""; +$DBNAME_APACHE4 = ""; + +$HOST_APACHE5 = ""; +$DBNAME_APACHE5 = ""; + +$HOST_APACHE6 = ""; +$DBNAME_APACHE6 = ""; + +$HOST_PROXY = ""; +$DBNAME_PROXY = ""; + +$HOST_SVN = ""; +$DBNAME_SVN = ""; + +$HOST_CONFLUENCE = ""; +$DBNAME_CONFLUENCE = ""; + + Index: trunk/ippMonitor/czartool/czarconfig.xml.in =================================================================== --- trunk/ippMonitor/czartool/czarconfig.xml.in (revision 40834) +++ trunk/ippMonitor/czartool/czarconfig.xml.in (revision 40840) @@ -10,4 +10,5 @@ 8 PATH/TO/czartool_plots + PATH/TO/gnuplot Index: trunk/ippMonitor/czartool/czarpoll.pl =================================================================== --- trunk/ippMonitor/czartool/czarpoll.pl (revision 40834) +++ trunk/ippMonitor/czartool/czarpoll.pl (revision 40840) @@ -102,12 +102,18 @@ foreach $server (@servers) { - my @dates = @{$pantasks->getDates($server)}; + my @dates = @{$pantasks->getDates($server,'gpc1')}; if (@dates) { - - $czarDb->updateServerDates($server, \@dates); + $czarDb->updateServerDates($server, \@dates,'gpc1'); } else { - - print "WARNING: No dates to update for '$server'\n"; + print "WARNING: No dates to update for '$server' in gpc1\n"; + } + + my @dates2 = @{$pantasks->getDates($server,'gpc2')}; + if (@dates2) { + $czarDb->updateServerDates($server, \@dates2,'gpc2'); + } + else { + print "WARNING: No dates to update for '$server' in gpc2\n"; } } @@ -126,22 +132,80 @@ print "* Updating labels\n"; - # my @servers = ("stdscience", "distribution", "publishing", "update"); my @servers = ("stdscience", "distribution"); my @allLabels = (); - + my @allLabels2 = (); + my @fields = (); + + #first grab the labels fom the relevant pantasks my $server = undef; foreach $server (@servers) { - my @labels = @{$pantasks->getLabels($server)}; + my @labels = @{$pantasks->getLabels($server,'gpc1')}; push (@allLabels, @labels); - } + + my @labels2 = @{$pantasks->getLabels($server,'gpc2')}; + push (@allLabels2, @labels2); + } + + ## adding extra labels from external file + my $filename = 'extra_labels.dat'; + if (open(my $fh, '<:encoding(UTF-8)', $filename)) { + my $header = <$fh>; + $header = <$fh>; + while (my $row = <$fh>) { + chomp $row; + @fields = split(' ', $row); + if ($fields[1] eq 'gpc1') { + push (@allLabels, $fields[0]); + } + if ($fields[1] eq 'gpc2') { + push (@allLabels2, $fields[0]); + } + } + } else { + warn "No extra labels found in '$filename' $!"; + } + + # Now, grab the priorities for each label + my @priorities = (); + my $row = undef; + my $label = undef; + my $priority = undef; + foreach $label ( @allLabels ) { + $priority = $gpc1Db->getPriority($label); + push(@priorities, $priority); + } + + #also grab gpc2 priorities from the gpc2 Label table + my @priorities2 = (); + my $priority2 = undef; + foreach $label ( @allLabels2 ) { + $priority2 = $gpc2Db->getPriority($label); + push(@priorities2, $priority2); + } + + #sort labels by priority + my @idx = sort {$priorities[$a] <=> $priorities[$b]} 0 ..$#priorities; + @allLabels = @allLabels[@idx]; + @priorities = @priorities[@idx]; + + my @idx2 = sort {$priorities2[$a] <=> $priorities2[$b]} 0 ..$#priorities2; + @allLabels2 = @allLabels2[@idx2]; + @priorities2 = @priorities2[@idx2]; # store them all in stdscience: $server = "stdscience"; if (@allLabels) { - $czarDb->updateCurrentLabels($server, \@allLabels); + $czarDb->updateCurrentLabels($server, \@allLabels,'gpc1', \@priorities); } else { - print "WARNING: No labels to update for '$server'\n"; - } + print "WARNING: No labels to update for gpc1 for '$server'\n"; + } + + if (@allLabels2) { + $czarDb->updateCurrentLabels($server, \@allLabels2,'gpc2', \@priorities2); + } else { + print "WARNING: No labels to update for gpc2 for '$server'\n"; + } + return(\@allLabels,\@allLabels2); } @@ -160,8 +224,9 @@ my $running = undef; foreach $server (@{$servers}) { - - $pantasks->getServerStatus($server, \$alive, \$running); - $czarDb->updateServerStatus($server, $alive, $running); - } + $pantasks->getServerStatus($server,'gpc1', \$alive, \$running); + $czarDb->updateServerStatus($server,'gpc1', $alive, $running); + $pantasks->getServerStatus($server,'gpc2', \$alive, \$running); + $czarDb->updateServerStatus($server,'gpc2', $alive, $running); + } } @@ -182,4 +247,5 @@ my $str = undef; my $labels = undef; + my $labels2 = undef; my $updateLabels = undef; my $row = undef; @@ -187,4 +253,5 @@ my $end = undef; my $priority = undef; + my $priority2 = undef; my $newState = undef; my $nsStatus = undef; @@ -227,8 +294,10 @@ # now cleanup tables from yesterday and optimize - print "* Performing database cleanup\n"; - $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval()); - print "* Optimizing the database\n"; - $czarDb->optimize(); + #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB. + #Therefore, it makes no sense to run the cleanup+optimising twice + #print "* Performing database cleanup\n"; + #$czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval()); + #print "* Optimizing the database\n"; + #$czarDb->optimize(); $lastDayDailyTasks = $yesterday; } @@ -237,6 +306,8 @@ # check nightly science status print "* Checking nightly science status\n"; - if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";} - $czarDb->updateNightlyScience($nsStatus); + if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc1')) {$nsStatus = "Unknown";} + $czarDb->updateNightlyScience($nsStatus,'gpc1'); + if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc2')) {$nsStatus = "Unknown";} + $czarDb->updateNightlyScience($nsStatus,'gpc2'); # check nebulous @@ -252,5 +323,7 @@ # check labels - updateLabels(); + my ($labels, $labels2) = updateLabels(); + my @labels = @$labels; + my @labels2 = @$labels2; # servers to check @@ -265,20 +338,10 @@ # deal with stdscience labels - if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;} my $size = @{$labels}; if($size > 0) { - - # get priority - foreach $row ( @{$labels} ) { - my ($label) = @{$row}; - # for now use priorities from gpc1 database - $priority = $gpc1Db->getPriority($label); - $czarDb->setLabelPriority($label, $priority); - } - - updateAllStages($thisServer, $newState, $labels, $begin, $end); + updateAllStages($thisServer, $newState, $labels,$labels2, $begin, $end); if ($thisServer eq "stdscience") { createPlots($thisServer, $plotter1, $labels, $begin, $end); - createPlots($thisServer, $plotter2, $labels, $begin, $end); + createPlots($thisServer, $plotter2, $labels2, $begin, $end); } } @@ -376,5 +439,4 @@ # # $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); - # #TdB 20190411: Make the all stages plots for the nightlyscience labels only # if ($label =~ m/nightlyscience/) { # $plotter->createTimeSeries($label, undef, $begin, $end, 1, 0, 1); @@ -382,11 +444,10 @@ #} - $allServerLabels = "all_".$server."_labels"; - - $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 0, 1); - - #TdB 20190411: We are turning off these plots for the time being. They take up a lot of time and are rarely looked at. If need be, can be generated on the fly using czartool. + $allServerLabels = $server."_labels"; + + $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1); + foreach $stage (@stages) { - $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 1); # TODO must be a neater way... + $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 0); # TODO must be a neater way... } } @@ -398,5 +459,5 @@ ########################################################################### sub updateAllStages { - my ($labelServer, $newState, $rows, $begin, $end) = @_; + my ($labelServer, $newState, $rows, $rows2, $begin, $end) = @_; print "* Updating stage data\n"; @@ -419,6 +480,9 @@ foreach $stage (@stages) { $server = $pantasks->getServerForThisStage($stage); - $pantasks->getRevertStatus($stage, \$reverting); - $czarDb->updateRevertStatus($stage, $reverting); + $pantasks->getRevertStatus($stage,'gpc1', \$reverting); + $czarDb->updateRevertStatus($stage,'gpc1', $reverting); + + $pantasks->getRevertStatus($stage,'gpc2', \$reverting); + $czarDb->updateRevertStatus($stage,'gpc2', $reverting); print "* Checking labels for $stage stage\n"; @@ -427,17 +491,16 @@ if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) { ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage); - $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $new, $full, $faults); + $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $new, $full, $faults); ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage); - $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $new, $full, $faults); + $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $new, $full, $faults); next; } - + #count the number of exposures under each label, and their faults $totalNew1=$totalFaults1=$totalFull1=0; - $totalNew2=$totalFaults2=$totalFull2=0; foreach $row ( @{$rows} ) { - my ($label) = @{$row}; + my ($label) = $row; chomp($label); @@ -447,5 +510,4 @@ $full = $gpc1Db->countExposures($label, $stage, "full"); $faults = $gpc1Db->countFaults($label, $stage, $newState); - #printf("%s %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults); $totalNew1 += $new; $totalFull1 += $full; @@ -453,9 +515,16 @@ $czarDb->insertNewTimeData($stage, $label, 'gpc1', $new, $full, $faults); - + } + + $totalNew2=$totalFaults2=$totalFull2=0; + foreach $row ( @{$rows2} ) { + my ($label) = $row; + + chomp($label); + + # Move this to a function and pass in the database $new = $gpc2Db->countExposures($label, $stage, $newState); $full = $gpc2Db->countExposures($label, $stage, "full"); $faults = $gpc2Db->countFaults($label, $stage, $newState); - #printf("%s %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults); $totalNew2 += $new; $totalFull2 += $full; @@ -465,7 +534,7 @@ } - $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1); - $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2); - } -} - + $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1); + $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2); + } +} + Index: trunk/ippMonitor/czartool/czartool.pl =================================================================== --- trunk/ippMonitor/czartool/czartool.pl (revision 40834) +++ trunk/ippMonitor/czartool/czartool.pl (revision 40840) @@ -15,10 +15,11 @@ my $save_temps = 1; my $interval = undef; +my $telescope = "gpc1"; GetOptions ( "dbname|d=s" => \$czarDbName, "interval|i=s" => \$interval, + "telescope|t=s" => \$telescope, ); - my @states = ("full", "new", "drop", "wait"); @@ -26,4 +27,5 @@ my $czarDb = $config->getCzarDbInstance(); my $gpc1Db = $config->getGpc1Instance(); +my $gpc2Db = $config->getGpc2Instance(); my $pantasks = new czartool::Pantasks(); @@ -55,9 +57,9 @@ sub promptPoll { - @stdscienceLabels = @{$pantasks->getLabels("stdscience")}; - @distributionLabels = @{$pantasks->getLabels("distribution")}; - # @publishingLabels = @{$pantasks->getLabels("publishing")}; + @stdscienceLabels = @{$pantasks->getLabels("stdscience",$telescope)}; + @distributionLabels = @{$pantasks->getLabels("distribution",$telescope)}; + # @publishingLabels = @{$pantasks->getLabels("publishing", $telescope)}; @publishingLabels = @stdscienceLabels; - checkAllLabels("new"); + checkAllLabels("new", $telescope); printInstructions(); @@ -68,7 +70,7 @@ if ($key eq "s") {checkServers();} - elsif ($key eq "l") {checkAllLabels("new");} - elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience");} - elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1]);} + elsif ($key eq "l") {checkAllLabels("new", $telescope);} + elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience",$telescope);} + elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1], $telescope);} printInstructions(); @@ -139,4 +141,5 @@ ########################################################################### sub labelDetails { + my ($telescope) = @_; print "\nPlease enter the label name, or number from table above: "; @@ -144,8 +147,8 @@ chomp($input); if ($input =~ m/^([0-9]|[1-9][0-9]|[1-9][0-9][0-9])$/) { - return checkOneLabel($stdscienceLabels[$input]); - } - - checkOneLabel($input); + return checkOneLabel($stdscienceLabels[$input], $telescope); + } + + checkOneLabel($input, $telescope); } @@ -156,5 +159,5 @@ ########################################################################### sub checkOneLabel { - my ($label) = @_; + my ($label, $telescope) = @_; chomp($label); @@ -188,5 +191,5 @@ foreach $stage (@stages) { - printf("|%14s", getStateAndFaultsString($label, $state, $stage)); + printf("|%14s", getStateAndFaultsString($label, $state, $stage, $telescope)); } @@ -209,5 +212,5 @@ ########################################################################### sub checkAllLabels { - my ($state) = @_; + my ($state, $telescope) = @_; #print time, $/; my $stage; @@ -259,5 +262,5 @@ foreach $stage (@stages) { - printf("|%14s", getStateAndFaultsString($stdsLabel, $state, $stage)); + printf("|%14s", getStateAndFaultsString($stdsLabel, $state, $stage, $telescope)); } @@ -280,11 +283,24 @@ ########################################################################### sub getStateAndFaultsString { - my ($label, $state, $stage) = @_; - - my $new = $gpc1Db->countExposures($label, $stage, $state); - - #if ($state ne "new") {return $new;} - - my $faults = $gpc1Db->countFaults($label, $stage, $state); + my ($label, $state, $stage, $telescope) = @_; + + my $new = undef; + my $faults = undef; + + if ($telescope eq 'gpc1') { + $new = $gpc1Db->countExposures($label, $stage, $state); + #if ($state ne "new") {return $new;} + + $faults = $gpc1Db->countFaults($label, $stage, $state); + } + elsif ($telescope eq 'gpc2') { + $new = $gpc2Db->countExposures($label, $stage, $state); + #if ($state ne "new") {return $new;} + + $faults = $gpc2Db->countFaults($label, $stage, $state); + } + else { + print "WARNING: Must specify a telescope for connecting to the right pantask\n"; + } if ($faults < 1) {return $new;} Index: trunk/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- trunk/ippMonitor/czartool/czartool/CzarDb.pm (revision 40834) +++ trunk/ippMonitor/czartool/czartool/CzarDb.pm (revision 40840) @@ -137,11 +137,11 @@ ########################################################################### sub updateServerStatus { - my ($self, $server, $alive, $running) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<execute; -} - -########################################################################### -# -# Sets priority for this label -# -########################################################################### -sub setLabelPriority { - my ($self, $label, $priority) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<&1`; my $line; @@ -149,5 +172,17 @@ ########################################################################### sub getServerStatus { - my ($self, $server, $alive, $running) = @_; + my ($self, $server,$telescope, $alive, $running) = @_; + + my $ipphome; + + if ($telescope eq 'gpc1') { + $ipphome = "/data/ippc64.1/ippitc"; + } + elsif ($telescope eq 'gpc2') { + $ipphome = "/data/ippc18.0/home/ippps2"; + } + else { + print "WARNING: Must specify a telescope for connecting to the right pantask\n"; + } my @cmdOut = `echo "status ; quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; @@ -170,5 +205,16 @@ ########################################################################### sub getServerCurrentStatus { - my ($self, $server) = @_; + my ($self, $server,$telescope) = @_; + + my $ipphome; + if ($telescope eq 'gpc1') { + $ipphome = "/data/ippc64.1/ippitc"; + } + elsif ($telescope eq 'gpc2') { + $ipphome = "/data/ippc18.0/home/ippps2"; + } + else { + print "WARNING: Must specify a telescope for connecting to the right pantask\n"; + } my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; @@ -189,5 +235,16 @@ ########################################################################### sub getRevertStatus { - my ($self, $stage, $reverting) = @_; + my ($self, $stage,$telescope, $reverting) = @_; + + my $ipphome; + if ($telescope eq 'gpc1') { + $ipphome = "/data/ippc64.1/ippitc"; + } + elsif ($telescope eq 'gpc2') { + $ipphome = "/data/ippc18.0/home/ippps2"; + } + else { + print "WARNING: Must specify a telescope for connecting to the right pantask\n"; + } my $server = $self->getServerForThisStage($stage); @@ -214,9 +271,20 @@ ########################################################################### sub getDates { - my ($self, $server) = @_; + my ($self, $server,$telescope) = @_; my @dates; my $prefix; + my $ipphome; + + if ($telescope eq 'gpc1') { + $ipphome = "/data/ippc64.1/ippitc"; + } + elsif ($telescope eq 'gpc2') { + $ipphome = "/data/ippc18.0/home/ippps2"; + } + else { + print "WARNING: Must specify a telescope for connecting to the right pantask\n"; + } if ($server eq "stdscience") {$prefix = "ns";} Index: trunk/ippMonitor/raw/czartool_getplot.php =================================================================== --- trunk/ippMonitor/raw/czartool_getplot.php (revision 40834) +++ trunk/ippMonitor/raw/czartool_getplot.php (revision 40840) @@ -22,5 +22,5 @@ $path = $CZARPLOTDIR; -if ($label == "all") $label = "all_".$mode."_labels"; +if ($label == "all") $label = $mode."_labels"; if ($type=="t" || $type=="r" || $type=="rt" || $type=="rh") Index: trunk/ippMonitor/raw/czartool_labels.php =================================================================== --- trunk/ippMonitor/raw/czartool_labels.php (revision 40834) +++ trunk/ippMonitor/raw/czartool_labels.php (revision 40840) @@ -90,5 +90,4 @@ # "publishing", "registration", - "replication", "stack", # "deepstack", @@ -139,6 +138,6 @@ // $labels is generated with sequential integer key values -$labels = getLabels($czardb, $selectedMode); -$distLabelsIndex = getLabels($czardb, "distribution"); +$labels = getLabels($czardb, $proj, $selectedMode); +$distLabelsIndex = getLabels($czardb, $proj, "distribution"); for ($i = 0; $i < count($distLabelsIndex); $i++) { $distLabelsExist[$distLabelsIndex[$i]] = true; @@ -195,9 +194,9 @@ echo ""; # time series plot -echo "

"; +echo "

"; +# log time series plot +echo "

"; # rate time series plot echo "

"; - # rate stacked histo plot -# echo "
"; # histogram plot # echo "
"; @@ -218,5 +217,5 @@ // status table at top -$nsStatus = getNightlyScienceStatus($czardb); +$nsStatus = getNightlyScienceStatus($czardb, $proj); $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; @@ -232,11 +231,9 @@ echo "Documentationhere"; echo "Cluster loadhere cpu_report mem_report"; -echo "Who's using the cluster?here"; -echo "Files monitorhere"; -echo "IPP metricshere"; -echo "Czar log pagesAll   "; -echo "This week"; -echo ""; +echo "IPP metricshere"; +echo "Czar log pageshere"; echo "Exposure summaryhere"; +echo "Confluencehere"; +echo "JIRAhere"; $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; @@ -266,25 +263,52 @@ createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "
"; createSummitDataTable($projectdb); echo "
"; // this is slow because of the join between summitExp and rawExp - createDatesTable($czardb); echo "
"; - - # This is awful + createChunkDataTable($projectdb); echo "
"; + createDatesTable($czardb, $proj); echo "
"; + echo $table; echo ""; - createTableTitle("Database replication status (update fix log here)", 2); + createTableTitle("Database status (update fix log here)", 2); echo ""; createTableColumnHeader("Database"); createTableColumnHeader("Status (sec behind master)"); + showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1); showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY); + echo ""; + + showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS); showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY); + showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY); # some mysql replication slaves to add when they are returned to service # showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP); -# showReplicationsStatus($REPL_HOST_IPPADMIN, $REPL_USER_IPPADMIN, $REPL_PASSWORD_IPPADMIN, $REPL_DBNAME_IPPADMIN); --- this one is in GPC1 # showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP); echo ""; + + echo $table; + echo ""; + createTableTitle("Critical machine status", 2); + + echo ""; + createTableColumnHeader("Hostname"); + createTableColumnHeader("Status"); + showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY); + showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY); + +# showPingStatus($HOST_APACHE1, $DBNAME_APACHE1); + showPingStatus($HOST_APACHE2, $DBNAME_APACHE2); + showPingStatus($HOST_APACHE3, $DBNAME_APACHE3); + showPingStatus($HOST_APACHE4, $DBNAME_APACHE4); + showPingStatus($HOST_APACHE5, $DBNAME_APACHE5); + showPingStatus($HOST_APACHE6, $DBNAME_APACHE6); + showPingStatus($HOST_PROXY, $DBNAME_PROXY); + showPingStatus($HOST_SVN, $DBNAME_SVN); + showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE); + + echo ""; + echo ""; menu_end(); @@ -401,11 +425,134 @@ ########################################################################### # +# Creates a table of chunk progress +# +########################################################################### +function createChunkDataTable($projectdb) { + + #First find the reference exposure ID to speed up the subsequent queries + $date = gmdate("Y-m-d"); + $sql = "SELECT MAX(exp_id) FROM rawExp WHERE dateobs >= date_sub('$date', interval 20 day) AND dateobs <= date_sub('$date', interval 10 day)"; + $qry = $projectdb->query($sql); + if (dberror($qry)) { + print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n"; + $refExpID = 0; + } + $qry->fetchInto($refExpID); + + #Find the number of chunks + $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM"; + $query .= " ("; + $query .= " SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0]"; + $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND comment LIKE '%visit 1%'"; + $query .= " GROUP BY chunk"; + $query .= " ) AS groupchunk"; + $query .= " LEFT JOIN"; + $query .= " ("; + $query .= " SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'"; + $query .= " GROUP BY object, filter, chunk"; + $query .= " ) AS groupobj"; + $query .= " WHERE groupobj.Nvisits = 4 "; + $query .= " GROUP BY groupobj.chunk"; + $query .= " ) AS group4"; + $query .= " on groupchunk.chunk=group4.chunk ORDER BY dateobs"; + + #$sql = "SELECT DISTINCT substr(comment, 1, position(' ' in comment)) FROM rawExp WHERE dateobs LIKE '$date%' AND exp_id > $refExpID AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit 1%'"; + + $qry = $projectdb->query($query); + if (dberror($qry)) { + echo "error reading rawExp table
\n"; + echo "
table query : $query
\n"; + menu_end(); + } + + // set up the table + global $table; + echo $table; + echo ""; + createTableTitle("Status of last night's chunks", 5); + echo ""; + createTableColumnHeader("Chunk"); + createTableColumnHeader("Nobs_1stvisit"); + createTableColumnHeader("Nobs_4visits"); + createTableColumnHeader("Ndiffs_published"); + createTableColumnHeader("Completion"); + + $msg = "No science images taken since $date"; + + // list the results + while ($qry->fetchInto($row)) { + $Nfirstvisits = 0; + $Nfourvisits = 0; + + $chunkname = $row[0]; + if($row[1] > 0) $Nfirstvisits = $row[1]; + if($row[2] > 0) $Nfourvisits = $row[2]; + + $query2 = " SELECT warp_id FROM "; + $query2 .= " warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query2 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'"; + $qry2 = $projectdb->query($query2); + + $Ndiffs = 0; + while ($qry2->fetchInto($row2)) { + $query3 = " SELECT diff_id FROM "; + $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)"; + $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; + $qry3 = $projectdb->query($query3); + while ($qry3->fetchInto($row3)) { + if ($row3[0] !=0) $Ndiffs ++; + } + + $query3 = " SELECT diff_id FROM "; + $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)"; + $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; + $qry3 = $projectdb->query($query3); + while ($qry3->fetchInto($row3)) { + if ($row3[0] !=0) $Ndiffs ++; + } + } + + if ($Ndiffs != ($Nfirstvisits*4)) $complete = "chunk not done"; + if ($Ndiffs == ($Nfirstvisits*4)) $complete = "chunk done"; + + if ($Nfirstvisits != $Nfourvisits) { + $Ndiffs .= "/? "; + } + if ($Nfirstvisits == $Nfourvisits) { + $Ndiffs .= "/"; + $Ndiffs .= $Nfirstvisits*4; + } + + echo ""; + + echo "$chunkname"; + echo "$Nfirstvisits"; + echo "$Nfourvisits"; + echo "$Ndiffs"; + echo "$complete"; + + echo ""; + } + echo "\n"; + +} + +########################################################################### +# # Gets labels for this server # ########################################################################### -function getLabels($db, $server) { +function getLabels($db, $proj, $server) { # order by descending priority as set in project database - $sql = "SELECT label FROM current_labels WHERE server LIKE '$server' ORDER BY priority DESC, label"; + $sql = "SELECT label FROM science_labels WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY priority DESC, label"; if ($debug) {echo "$sql
";} @@ -547,5 +694,5 @@ foreach ($stages as &$stage) { - $reverting = getRevertStatus($db, $stage); + $reverting = getRevertStatus($db, $proj, $stage); if(!$reverting) {$label = "";} if($reverting) {$label = "reverting";} @@ -608,7 +755,7 @@ # ########################################################################### -function setRevertStatus($db, $stage, $reverting) { - - $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'"; +function setRevertStatus($db, $proj, $stage, $reverting) { + + $sql = "UPDATE science_reverts SET reverting = $reverting WHERE stage LIKE '$stage' AND telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -622,7 +769,7 @@ # ########################################################################### -function getNightlyScienceStatus($db) { - - $sql = "SELECT status FROM nightlyscience"; +function getNightlyScienceStatus($db, $proj) { + + $sql = "SELECT status FROM science_nightlyscience WHERE telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -639,5 +786,5 @@ # ########################################################################### -function turnRevertsOnOff($db, $stage, $mode) { +function turnRevertsOnOff($db, $proj, $stage, $mode) { exec("czartool_revert.pl -t $stage -o $mode", $response, $status); @@ -646,5 +793,5 @@ else if ($response[0] == "on") $currentRevertMode = 1; - setRevertStatus($db, $stage, $currentRevertMode); + setRevertStatus($db, $proj, $stage, $currentRevertMode); } @@ -654,9 +801,9 @@ # ########################################################################### -function getRevertStatus($db, $stage) { +function getRevertStatus($db, $proj, $stage) { $anyFaults = false; - $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'"; + $sql = "SELECT reverting FROM science_reverts WHERE stage LIKE '$stage' AND telescope LIKE '$proj'"; if ($debug) {echo "$sql
";} @@ -706,5 +853,5 @@ # ########################################################################### -function createDatesTable($db) { +function createDatesTable($db, $proj) { global $table; @@ -717,6 +864,6 @@ echo "\n"; - createServerDateRow($db, "stdscience"); - createServerDateRow($db, "registration"); + createServerDateRow($db, $proj, "stdscience"); + createServerDateRow($db, $proj, "registration"); echo "\n"; @@ -729,7 +876,7 @@ # ########################################################################### -function createServerDateRow($db, $server) { - - $sql = "SELECT date FROM server_dates WHERE server LIKE '$server' ORDER BY date"; +function createServerDateRow($db, $proj, $server) { + + $sql = "SELECT date FROM science_server_dates WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY date"; if ($debug) {echo "$sql
";} @@ -982,4 +1129,40 @@ } +########################################################################### +# +# Shows the status of the various replication mysql servers +# +########################################################################### +function showDatabaseStatus($Host, $User, $Password, $DatabaseName) { + #print "
$replHost, $replUser, $replPassword, $replDatabaseName
"; + $db = DB::connect("mysql://$User:$Password@$Host"); + if (PEAR::isError($db)) { + echo "$DatabaseNameMySQL DB connection error"; + //die("MySQL DB connection error: Check the configuration for $DatabaseName"); + //die(""); + return -1; + + } + $Status = ($errorStatusInMySql==0?"OK":"PROBLEM"); + echo "$DatabaseName$Status"; + + $db->disconnect(); +} + +########################################################################### +# +# Shows the status of a machine using ping +# +########################################################################### +function showPingStatus($Host, $DatabaseName) { + exec("ping -c 1 $Host", $res, $rval); + if ($rval == 0){ + $PingStatus= "UP"; + }else{ + $PingStatus = "DOWN"; + } + echo "$DatabaseName$PingStatus"; +} + ?> Index: trunk/ippMonitor/raw/ipp.czar.dat =================================================================== --- trunk/ippMonitor/raw/ipp.czar.dat (revision 40834) +++ trunk/ippMonitor/raw/ipp.czar.dat (revision 40840) @@ -3,5 +3,4 @@ menutop | menutop | plain |   | -menulink | menuselect | link | czartool - old | czartool_labels_old.php menulink | menuselect | link | czartool | czartool_labels.php menulink | menuselect | link | mask stats | maskStats.php @@ -10,4 +9,4 @@ menulink | menuselect | link | Exposures Status | exposureStatus.php menulink | menuselect | link | MOPS Exposures Status | mopsStatus.php -menulink | menuselect | link | GPC1 MySql ProcessList | gpc1MysqlProcessList.php +menulink | menuselect | link | MySql ProcessList | gpc1MysqlProcessList.php Index: trunk/ippMonitor/raw/ipp.menu.dat =================================================================== --- trunk/ippMonitor/raw/ipp.menu.dat (revision 40834) +++ trunk/ippMonitor/raw/ipp.menu.dat (revision 40840) @@ -23,4 +23,5 @@ menutop | menutop | link | Stack Steps | ipp.stack.php menutop | menutop | link | Static Sky Steps | ipp.sky.php +menutop | menutop | link | Full Force Steps | ipp.ffs.php menutop | menutop | link | Distribution | ipp.dist.php menutop | menutop | link | Large Area Processing | ipp.lap.php Index: trunk/ippMonitor/raw/ipp.sky.dat =================================================================== --- trunk/ippMonitor/raw/ipp.sky.dat (revision 40834) +++ trunk/ippMonitor/raw/ipp.sky.dat (revision 40840) @@ -16,9 +16,2 @@ menulink | menuselect | link | Skycal Failed | skycal_failure.php -menutop | menutop | plain |   | -menulink | menuselect | link | Static Sky Summary | staticskySummary.php -menulink | menuselect | link | Static Sky Run | staticskyRun.php -menulink | menuselect | link | Static Sky Input Skyfile | staticskyInputSkyfile.php -menulink | menuselect | link | Static Sky Processed Skyfile | staticskyProcessedSkyfile.php -menulink | menuselect | link | Static Sky Failed Skyfile | staticskyFailedSkyfile.php -menulink | menuselect | link | Static Sky Failed | failedStaticsky.php