IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40055


Ignore:
Timestamp:
May 24, 2017, 2:55:55 PM (9 years ago)
Author:
watersc1
Message:

Update to czartool to populate summit/download/new/raw Exp tables, and plot those results with the other stages. Fixes bug that allowed plot temp files to remain undeleted

Location:
trunk/ippMonitor/czartool
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippMonitor/czartool/czarDbSchema.sql

    r38261 r40055  
    520520) ENGINE=InnoDB DEFAULT CHARSET=latin1;
    521521SET character_set_client = @saved_cs_client;
     522
     523
     524--
     525-- Table structure for table `summitExp`
     526--
     527
     528DROP TABLE IF EXISTS `summitExp`;
     529SET @saved_cs_client     = @@character_set_client;
     530SET character_set_client = utf8;
     531CREATE TABLE `summitExp` (
     532  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     533  `label` varchar(128) default 'NONE',
     534  `dbname` varchar(128) default 'NONE',
     535  `pending` bigint(20) NOT NULL,
     536  `faults` bigint(20) NOT NULL,
     537  `processed` bigint(20) NOT NULL default '0',
     538  KEY `summitExpIndex` (`timestamp`,`label`, `dbname`)
     539) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     540SET character_set_client = @saved_cs_client;
     541
     542--
     543-- Table structure for table `downloadExp`
     544--
     545
     546DROP TABLE IF EXISTS `downloadExp`;
     547SET @saved_cs_client     = @@character_set_client;
     548SET character_set_client = utf8;
     549CREATE TABLE `downloadExp` (
     550  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     551  `label` varchar(128) default 'NONE',
     552  `dbname` varchar(128) default 'NONE',
     553  `pending` bigint(20) NOT NULL,
     554  `faults` bigint(20) NOT NULL,
     555  `processed` bigint(20) NOT NULL default '0',
     556  KEY `downloadExpIndex` (`timestamp`,`label`, `dbname`)
     557) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     558SET character_set_client = @saved_cs_client;
     559
     560--
     561-- Table structure for table `newExp`
     562--
     563
     564DROP TABLE IF EXISTS `newExp`;
     565SET @saved_cs_client     = @@character_set_client;
     566SET character_set_client = utf8;
     567CREATE TABLE `newExp` (
     568  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     569  `label` varchar(128) default 'NONE',
     570  `dbname` varchar(128) default 'NONE',
     571  `pending` bigint(20) NOT NULL,
     572  `faults` bigint(20) NOT NULL,
     573  `processed` bigint(20) NOT NULL default '0',
     574  KEY `newExpIndex` (`timestamp`,`label`, `dbname`)
     575) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     576SET character_set_client = @saved_cs_client;
     577
     578--
     579-- Table structure for table `rawExp`
     580--
     581
     582DROP TABLE IF EXISTS `rawExp`;
     583SET @saved_cs_client     = @@character_set_client;
     584SET character_set_client = utf8;
     585CREATE TABLE `rawExp` (
     586  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
     587  `label` varchar(128) default 'NONE',
     588  `dbname` varchar(128) default 'NONE',
     589  `pending` bigint(20) NOT NULL,
     590  `faults` bigint(20) NOT NULL,
     591  `processed` bigint(20) NOT NULL default '0',
     592  KEY `rawExpIndex` (`timestamp`,`label`, `dbname`)
     593) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     594SET character_set_client = @saved_cs_client;
     595
     596
     597
    522598/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
    523599
    … …  
    529605/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
    530606/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
     607

  • trunk/ippMonitor/czartool/czarpoll.pl

    r40008 r40055  
    6363        "fullforce",
    6464        "dist",
    65         "pub");
     65        "pub",
     66        "summitExp",
     67        "downloadExp",
     68        "newExp",
     69        "rawExp");
     70
     71
    6672
    6773my @ippToPspsStages = (
    … …  
    274280        }
    275281
     282        my @trashFiles = </tmp/czarplot_gnuplot_*>;
     283        if ($#trashFiles != -1) {
     284            print "* WARNING: Cleaning " . ($#trashFiles + 1) . " files that were not auto cleaned.\n";
     285            foreach my $tFile (@trashFiles) {
     286                unlink($tFile);
     287            }
     288        }
     289
    276290        my $iterationTime = time() - $iterationStartTime;
    277291
    … …  
    350364        # skip update labels for now. The plots are not relevant.
    351365        next if $label =~ 'ps_ud';
     366        # Skip individual label tables for raw-like stages.
     367#       if ($stage =~ /Exp/) { next; }
    352368
    353369        $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
    … …  
    390406
    391407    foreach $stage (@stages) {
    392 
    393408        $server = $pantasks->getServerForThisStage($stage);
    394409        $pantasks->getRevertStatus($stage, \$reverting);
    … …  
    396411
    397412        print "* Checking labels for $stage stage\n";
     413
     414        # These have no labels to check, so just get them out of the way fast
     415        if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) {
     416            ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage);
     417            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc1', $new, $full, $faults);
     418
     419            ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage);
     420            $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", 'gpc2', $new, $full, $faults);
     421
     422            next;
     423        }
     424
    398425
    399426        $totalNew1=$totalFaults1=$totalFull1=0;

  • trunk/ippMonitor/czartool/czartool/CzarDb.pm

    r39805 r40055  
    1717        "dist",
    1818        "pub",
    19         "chipbackground"); # TODO put elsewhere
     19        "chipbackground",
     20        "summitExp",
     21        "downloadExp",
     22        "newExp",
     23        "rawExp"); # TODO put elsewhere
    2024
    2125my @ippToPspsStages = (

  • trunk/ippMonitor/czartool/czartool/Gpc1Db.pm

    r38258 r40055  
    77use warnings;
    88use strict;
     9use POSIX qw/strftime/;
    910
    1011use czartool::MySQLDb;
    … …  
    3435    elsif ($stage eq "pub")       {${$joinTable}="publishDone";}
    3536    elsif ($stage eq "chipbackground") {${$id}="chip_bg_id"; ${$joinTable}="chipBackgroundImfile";}
     37    elsif ($stage eq 'summitExp')    { ${$id} = 'summit_id'; ${$joinTable} = 'summitImfile'; }
     38    elsif ($stage eq 'downloadExp')  { ${$id} = 'summit_id'; ${$joinTable} = 'pzDownloadImfile'; }
     39    elsif ($stage eq 'newExp')       { ${$id} = 'exp_id';    ${$joinTable} = 'newImfile'; }
     40    elsif ($stage eq 'rawExp')       { ${$id} = 'exp_id';    ${$joinTable} = 'rawImfile'; }
     41   
    3642    else {
    3743        print "* ERROR: could not find joinTable and ID for '$stage' stage\n";
    … …  
    6369    elsif ($stage eq "pub" ) {return "publishRun";}
    6470    elsif ($stage eq "chipbackground" ) {return "chipBackgroundRun";}
    65 
     71    elsif ($stage eq 'summitExp' )   { return 'summitExp'; }
     72    elsif ($stage eq 'downloadExp' ) { return 'pzDownloadExp'; }
     73    elsif ($stage eq 'newExp' )      { return 'newExp'; }
     74    elsif ($stage eq 'rawExp' )      { return 'rawExp'; }
    6675    return "ERROR";
    6776}
    … …  
    95104return scalar $query->fetchrow_array();
    96105}
     106
     107###
     108# General purpose way to get early stage results.  These all have different join requirements than the later stages.
     109#
     110sub countRawExposures {
     111    my ($self, $date, $stage) = @_;
     112
     113    my $query;
     114    # Ignore the date input, because it's localtime, and not UTC, but dateobs is entirely UTC.
     115    $date = strftime('%Y-%m-%d', gmtime);
     116   
     117    if ($stage eq 'summitExp') {
     118        # summitExp is populated directly, and contains the fault information
     119        $query = $self->{_db}->prepare(<<SQL);
     120        SELECT summit_id,"full",fault
     121            FROM summitExp
     122            WHERE dateobs > '${date}T00:00:00'
     123            AND dateobs < '${date}T23:59:59'
     124SQL
     125    }
     126    elsif ($stage eq 'downloadExp') {
     127        # pzDownloadExp has the state, but the faults are in pzDownloadImfile.  dateobs needs to be grabbed from summitExp still.
     128        $query = $self->{_db}->prepare(<<SQL);
     129        SELECT summit_id,pzDownloadExp.state,sum(pzDownloadImfile.fault)
     130            FROM summitExp JOIN pzDownloadExp USING(summit_id) JOIN pzDownloadImfile USING(summit_id)
     131            WHERE dateobs > '${date}T00:00:00'
     132            AND dateobs < '${date}T23:59:59'
     133            GROUP BY summit_id
     134SQL
     135    }
     136    elsif ($stage eq 'newExp') {
     137        # This could skip pzDownloadExp.  This could probably skip everything, as this is autopopulated when the exposure completes downloading.
     138        $query = $self->{_db}->prepare(<<SQL);
     139        SELECT exp_id,newExp.state,0
     140            FROM summitExp JOIN pzDownloadExp USING(summit_id)
     141            JOIN newExp USING(summit_id)
     142            WHERE dateobs > '${date}T00:00:00'
     143            AND dateobs < '${date}T23:59:59'
     144            GROUP BY exp_id
     145SQL
     146    }
     147    elsif ($stage eq 'rawExp') {
     148        # The rawExp entry doesn't exist until all the rawImfiles are populated,
     149        $query = $self->{_db}->prepare(<<SQL);
     150        SELECT rawExp.exp_id,rawExp.state,sum(rawImfile.fault)
     151            FROM summitExp
     152            JOIN newExp USING(summit_id) JOIN newImfile ON (newExp.exp_id = newImfile.exp_id)
     153            LEFT JOIN rawImfile ON (newImfile.exp_id = rawImfile.exp_id AND newImfile.tmp_class_id = rawImfile.tmp_class_id)
     154            LEFT JOIN rawExp ON (rawImfile.exp_id = rawExp.exp_id)
     155            WHERE summitExp.dateobs > '${date}T00:00:00'
     156            AND summitExp.dateobs < '${date}T23:59:59'
     157            GROUP BY exp_id
     158SQL
     159    }
     160    $query->execute;
     161    my $result = $query->fetchall_arrayref();
     162
     163    my ($new,$full,$faults) = (0,0,0);
     164
     165    foreach my $row (@{ $result }) {
     166        my ($id,$state,$fault) = @{ $row };
     167        unless (defined($fault)) { $fault = 0; }
     168        if ($fault eq '') { $fault = 0; }
     169        if ($fault != 0) { $faults ++ ; next; }
     170        unless (defined($state)) { $new++; next; }
     171        if (($state eq 'new' )||($state eq 'run' )||($state eq 'reg' )) { $new ++;  next; }
     172        if (($state eq 'stop')||($state eq 'full')||($state eq 'drop')) { $full ++; next; }
     173       
     174        # Shouldn't get here.
     175        print STDERR "countRawExposures recieved bad line: STAGE: $stage ID: $id STATE: $state FAULT: $fault\n";
     176    }
     177
     178    return($new,$full,$faults);
     179}
     180   
    97181
    98182###########################################################################

  • trunk/ippMonitor/czartool/czartool/Pantasks.pm

    r39805 r40055  
    2424        );
    2525
     26my $ipphome = "/data/ippc64.1/ippitc";
     27
    2628###########################################################################
    2729#
    … …  
    9294    my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day);
    9395
    94     my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c ~ipp/stdscience/ptolemy.rc 2>&1`;
     96    print "stdscience : $ipphome/stdscience/ptolemy.rc\n";
     97    my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c $ipphome/stdscience/ptolemy.rc 2>&1`;
    9598    my $line;
    9699    foreach $line (@cmdOut) {
    … …  
    117120    my @labels;
    118121
    119     my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     122    print "stdscience : $ipphome/stdscience/ptolemy.rc\n";
     123    my @cmdOut = `echo "show.labels;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
    120124    my $line;
    121125    my $passedHeader=0;
    122126    foreach $line (@cmdOut) {
    123 
     127        print "labels: $line";
    124128        chomp($line);
    125129        if (!$self->outputOk($line)) {return \@labels;}
    … …  
    145149    my ($self, $server, $alive, $running) = @_;
    146150
    147     my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     151    my @cmdOut = `echo "status ; quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
    148152    my $line;
    149153    ${$alive} = 0;
    … …  
    166170    my ($self, $server) = @_;
    167171
    168     my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     172    my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
    169173    my $line;
    170174    my $passedHeader=0;
    … …  
    189193    if ($stage eq "cam") {$stage = "camera";}
    190194
    191     my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     195    my @cmdOut = `echo "status;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
    192196    my $line;
    193197    foreach $line (@cmdOut) {
    … …  
    217221    elsif ($server eq "registration") {$prefix = "register";}
    218222
    219     my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     223    my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c $ipphome/$server/ptolemy.rc 2>&1`;
    220224    my $line;
    221225    my $passedHeader=0;

  • trunk/ippMonitor/czartool/czartool/Plotter.pm

    r38258 r40055  
    2020     "fullforce",
    2121     "dist",
    22      "pub"
     22     "pub",
     23    'summitExp',
     24    'downloadExp',
     25    'newExp',
     26    'rawExp'
    2327    );
    2428
    … …  
    245249
    246250    # now delete temp dat files
    247     foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}
     251    foreach my $stage (keys %gnuplotFiles) {
     252        unlink($gnuplotFiles{$stage});
     253    }
    248254
    249255    return 1;
    … …  
    442448    use FileHandle;
    443449    GP->autoflush(1);
    444 
    445     if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
     450   
     451
     452    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";"; }
    446453    print GP
    447454        "set term $self->{_outputFormat};" .
    … …  
    573580    use FileHandle;
    574581    GP->autoflush(1);
    575 
    576582    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    577583    print GP
    … …  
    616622
    617623    # now delete temp dat files
    618     foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
     624
     625    foreach my $stage (keys %$gnuplotFiles) {
     626        my $ff = $gnuplotFiles->{$stage};
     627        unlink($gnuplotFiles->{$stage});
     628    }
    619629}
    620630
Note: See TracChangeset for help on using the changeset viewer.