Index: trunk/ippMonitor/czartool/czarDbSchema.sql =================================================================== --- trunk/ippMonitor/czartool/czarDbSchema.sql (revision 40017) +++ trunk/ippMonitor/czartool/czarDbSchema.sql (revision 40055) @@ -520,4 +520,80 @@ ) ENGINE=InnoDB DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; + + +-- +-- Table structure for table `summitExp` +-- + +DROP TABLE IF EXISTS `summitExp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `summitExp` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `dbname` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `summitExpIndex` (`timestamp`,`label`, `dbname`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `downloadExp` +-- + +DROP TABLE IF EXISTS `downloadExp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `downloadExp` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `dbname` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `downloadExpIndex` (`timestamp`,`label`, `dbname`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `newExp` +-- + +DROP TABLE IF EXISTS `newExp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `newExp` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `dbname` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `newExpIndex` (`timestamp`,`label`, `dbname`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `rawExp` +-- + +DROP TABLE IF EXISTS `rawExp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `rawExp` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `dbname` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `rawExpIndex` (`timestamp`,`label`, `dbname`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + + + /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; @@ -529,2 +605,3 @@ /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + Index: trunk/ippMonitor/czartool/czarpoll.pl =================================================================== --- trunk/ippMonitor/czartool/czarpoll.pl (revision 40017) +++ trunk/ippMonitor/czartool/czarpoll.pl (revision 40055) @@ -63,5 +63,11 @@ "fullforce", "dist", - "pub"); + "pub", + "summitExp", + "downloadExp", + "newExp", + "rawExp"); + + my @ippToPspsStages = ( @@ -274,4 +280,12 @@ } + my @trashFiles = ; + if ($#trashFiles != -1) { + print "* WARNING: Cleaning " . ($#trashFiles + 1) . " files that were not auto cleaned.\n"; + foreach my $tFile (@trashFiles) { + unlink($tFile); + } + } + my $iterationTime = time() - $iterationStartTime; @@ -350,4 +364,6 @@ # skip update labels for now. The plots are not relevant. next if $label =~ 'ps_ud'; + # Skip individual label tables for raw-like stages. +# if ($stage =~ /Exp/) { next; } $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); @@ -390,5 +406,4 @@ foreach $stage (@stages) { - $server = $pantasks->getServerForThisStage($stage); $pantasks->getRevertStatus($stage, \$reverting); @@ -396,4 +411,16 @@ print "* Checking labels for $stage stage\n"; + + # These have no labels to check, so just get them out of the way fast + if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) { + ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage); + $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $new, $full, $faults); + + ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage); + $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $new, $full, $faults); + + next; + } + $totalNew1=$totalFaults1=$totalFull1=0; Index: trunk/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- trunk/ippMonitor/czartool/czartool/CzarDb.pm (revision 40017) +++ trunk/ippMonitor/czartool/czartool/CzarDb.pm (revision 40055) @@ -17,5 +17,9 @@ "dist", "pub", - "chipbackground"); # TODO put elsewhere + "chipbackground", + "summitExp", + "downloadExp", + "newExp", + "rawExp"); # TODO put elsewhere my @ippToPspsStages = ( Index: trunk/ippMonitor/czartool/czartool/Gpc1Db.pm =================================================================== --- trunk/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 40017) +++ trunk/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 40055) @@ -7,4 +7,5 @@ use warnings; use strict; +use POSIX qw/strftime/; use czartool::MySQLDb; @@ -34,4 +35,9 @@ elsif ($stage eq "pub") {${$joinTable}="publishDone";} elsif ($stage eq "chipbackground") {${$id}="chip_bg_id"; ${$joinTable}="chipBackgroundImfile";} + elsif ($stage eq 'summitExp') { ${$id} = 'summit_id'; ${$joinTable} = 'summitImfile'; } + elsif ($stage eq 'downloadExp') { ${$id} = 'summit_id'; ${$joinTable} = 'pzDownloadImfile'; } + elsif ($stage eq 'newExp') { ${$id} = 'exp_id'; ${$joinTable} = 'newImfile'; } + elsif ($stage eq 'rawExp') { ${$id} = 'exp_id'; ${$joinTable} = 'rawImfile'; } + else { print "* ERROR: could not find joinTable and ID for '$stage' stage\n"; @@ -63,5 +69,8 @@ elsif ($stage eq "pub" ) {return "publishRun";} elsif ($stage eq "chipbackground" ) {return "chipBackgroundRun";} - + elsif ($stage eq 'summitExp' ) { return 'summitExp'; } + elsif ($stage eq 'downloadExp' ) { return 'pzDownloadExp'; } + elsif ($stage eq 'newExp' ) { return 'newExp'; } + elsif ($stage eq 'rawExp' ) { return 'rawExp'; } return "ERROR"; } @@ -95,4 +104,79 @@ return scalar $query->fetchrow_array(); } + +### +# General purpose way to get early stage results. These all have different join requirements than the later stages. +# +sub countRawExposures { + my ($self, $date, $stage) = @_; + + my $query; + # Ignore the date input, because it's localtime, and not UTC, but dateobs is entirely UTC. + $date = strftime('%Y-%m-%d', gmtime); + + if ($stage eq 'summitExp') { + # summitExp is populated directly, and contains the fault information + $query = $self->{_db}->prepare(< '${date}T00:00:00' + AND dateobs < '${date}T23:59:59' +SQL + } + elsif ($stage eq 'downloadExp') { + # pzDownloadExp has the state, but the faults are in pzDownloadImfile. dateobs needs to be grabbed from summitExp still. + $query = $self->{_db}->prepare(< '${date}T00:00:00' + AND dateobs < '${date}T23:59:59' + GROUP BY summit_id +SQL + } + elsif ($stage eq 'newExp') { + # This could skip pzDownloadExp. This could probably skip everything, as this is autopopulated when the exposure completes downloading. + $query = $self->{_db}->prepare(< '${date}T00:00:00' + AND dateobs < '${date}T23:59:59' + GROUP BY exp_id +SQL + } + elsif ($stage eq 'rawExp') { + # The rawExp entry doesn't exist until all the rawImfiles are populated, + $query = $self->{_db}->prepare(< '${date}T00:00:00' + AND summitExp.dateobs < '${date}T23:59:59' + GROUP BY exp_id +SQL + } + $query->execute; + my $result = $query->fetchall_arrayref(); + + my ($new,$full,$faults) = (0,0,0); + + foreach my $row (@{ $result }) { + my ($id,$state,$fault) = @{ $row }; + unless (defined($fault)) { $fault = 0; } + if ($fault eq '') { $fault = 0; } + if ($fault != 0) { $faults ++ ; next; } + unless (defined($state)) { $new++; next; } + if (($state eq 'new' )||($state eq 'run' )||($state eq 'reg' )) { $new ++; next; } + if (($state eq 'stop')||($state eq 'full')||($state eq 'drop')) { $full ++; next; } + + # Shouldn't get here. + print STDERR "countRawExposures recieved bad line: STAGE: $stage ID: $id STATE: $state FAULT: $fault\n"; + } + + return($new,$full,$faults); +} + ########################################################################### Index: trunk/ippMonitor/czartool/czartool/Pantasks.pm =================================================================== --- trunk/ippMonitor/czartool/czartool/Pantasks.pm (revision 40017) +++ trunk/ippMonitor/czartool/czartool/Pantasks.pm (revision 40055) @@ -24,4 +24,6 @@ ); +my $ipphome = "/data/ippc64.1/ippitc"; + ########################################################################### # @@ -92,5 +94,6 @@ my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day); - my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c ~ipp/stdscience/ptolemy.rc 2>&1`; + print "stdscience : $ipphome/stdscience/ptolemy.rc\n"; + my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c $ipphome/stdscience/ptolemy.rc 2>&1`; my $line; foreach $line (@cmdOut) { @@ -117,9 +120,10 @@ my @labels; - my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + print "stdscience : $ipphome/stdscience/ptolemy.rc\n"; + my @cmdOut = `echo "show.labels;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; my $line; my $passedHeader=0; foreach $line (@cmdOut) { - + print "labels: $line"; chomp($line); if (!$self->outputOk($line)) {return \@labels;} @@ -145,5 +149,5 @@ my ($self, $server, $alive, $running) = @_; - my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + my @cmdOut = `echo "status ; quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; my $line; ${$alive} = 0; @@ -166,5 +170,5 @@ my ($self, $server) = @_; - my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; my $line; my $passedHeader=0; @@ -189,5 +193,5 @@ if ($stage eq "cam") {$stage = "camera";} - my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; my $line; foreach $line (@cmdOut) { @@ -217,5 +221,5 @@ elsif ($server eq "registration") {$prefix = "register";} - my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`; my $line; my $passedHeader=0; Index: trunk/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- trunk/ippMonitor/czartool/czartool/Plotter.pm (revision 40017) +++ trunk/ippMonitor/czartool/czartool/Plotter.pm (revision 40055) @@ -20,5 +20,9 @@ "fullforce", "dist", - "pub" + "pub", + 'summitExp', + 'downloadExp', + 'newExp', + 'rawExp' ); @@ -245,5 +249,7 @@ # now delete temp dat files - foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});} + foreach my $stage (keys %gnuplotFiles) { + unlink($gnuplotFiles{$stage}); + } return 1; @@ -442,6 +448,7 @@ use FileHandle; GP->autoflush(1); - - if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";} + + + if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";"; } print GP "set term $self->{_outputFormat};" . @@ -573,5 +580,4 @@ use FileHandle; GP->autoflush(1); - if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";} print GP @@ -616,5 +622,9 @@ # now delete temp dat files - foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});} + + foreach my $stage (keys %$gnuplotFiles) { + my $ff = $gnuplotFiles->{$stage}; + unlink($gnuplotFiles->{$stage}); + } }