IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38206 for trunk/ippMonitor


Ignore:
Timestamp:
Apr 28, 2015, 11:23:28 AM (11 years ago)
Author:
eugene
Message:

add fullforce stage, cleanup stage handling

Location:
trunk/ippMonitor
Files:
8 edited

Legend:

Unmodified
Added
Removed
  • trunk/ippMonitor/czartool/czarpoll.pl

    r38200 r38206  
    4545        "staticsky",
    4646        "diff",
     47        "fullforce",
    4748        "dist",
    48         "pub",
    49         "chipbackground");
     49        "pub");
     50
     51# "chipbackground"
    5052
    5153my @ippToPspsStages = (
  • trunk/ippMonitor/czartool/czartool.pl

    r37983 r38206  
    1616my $interval = undef;
    1717
    18 GetOptions ( 
     18GetOptions (
    1919        "dbname|d=s" => \$czarDbName,
    2020        "interval|i=s" => \$interval,
     
    3232my @publishingLabels = undef;
    3333
    34 my @stages = ("chip", "cam", "fake", "warp", "stack", "staticsky", "diff", "dist", "pub", "chipbackground");
     34my @stages = ("chip", "cam", "fake", "warp", "stack", "staticsky", "diff", "fullforce", "dist", "pub");
     35# "chipbackground"
     36
    3537promptPoll();
    3638
     
    4850###########################################################################
    4951#
    50 # Polls waiting for input from user 
     52# Polls waiting for input from user
    5153#
    5254###########################################################################
     
    124126        printf("| %12s ", $server);
    125127        chomp($results[0]);
    126         chomp($results[1]); 
     128        chomp($results[1]);
    127129        printf("|   %3s   |         %3s          |\n", $results[0], $results[1]);
    128130    }
  • trunk/ippMonitor/czartool/czartool/CzarDb.pm

    r38200 r38206  
    280280#        push @{$labels}, 'LAP.PV3.20140730.restack';
    281281#        push @{$labels}, 'LAP.PV3.20140730.bigmem';
    282         push @{$labels}, 'LAP.PV3.20140730.skycal01';
    283         push @{$labels}, 'LAP.PV3.20140730.sky01.remote';
    284         push @{$labels}, 'LAP.PV3.20140730.diff01';
    285         push @{$labels}, 'LAP.PV3.20140730.diff01.left';
     282#        push @{$labels}, 'LAP.PV3.20140730.skycal01';
     283#        push @{$labels}, 'LAP.PV3.20140730.sky01.remote';
     284#        push @{$labels}, 'LAP.PV3.20140730.diff01';
     285#        push @{$labels}, 'LAP.PV3.20140730.diff01.left';
     286        push @{$labels}, 'LAP.PV3.20140730.ff01';
     287        push @{$labels}, 'LAP.PV3.20140730.ff01.left';
    286288        push @{$labels}, 'LAP.PV3W.20150312';
    287289#       push @{$labels}, 'SAS.20141118';
  • trunk/ippMonitor/czartool/czartool/DayMetrics.pm

    r33814 r38206  
    2828    # Call the constructor of the parent class
    2929    my $self = $class->SUPER::new(
    30             $_[1],  # config object 
     30            $_[1],  # config object
    3131            $_[2],  # verbose
    3232            $_[3]   # save_temps
    33             ); 
     33            );
    3434
    3535    $self->{day} = $_[4];
     
    7878    print $htmlFile "<a href=\"../$previousDay/index.html\"> \< previous day</a> | <a href=\"../index.html\">all</a> | <a href=\"../$nextDay/index.html\">next day \></a><br>\n";
    7979    print $htmlFile "Measured from $self->{begin} to $self->{end} HST<br>\n";
    80     printf ( $htmlFile "%d exposures taken on summit last night, %d through chip today</h5>\n", 
     80    printf ( $htmlFile "%d exposures taken on summit last night, %d through chip today</h5>\n",
    8181            $summitExposures, $chipMetrics->getProcessed() ? $chipMetrics->getProcessed() : 0 );
    8282    print $htmlFile "</head>\n";
     
    9999
    100100    my @stages = (
    101             "chip",
    102             "cam",
    103             "fake",
    104             "warp",
    105             "stack",
    106             "staticsky",
    107             "diff",
    108             "dist",
    109             "chipbackground");
    110 
     101            "chip",
     102            "cam",
     103            "fake",
     104            "warp",
     105            "stack",
     106            "staticsky",
     107            "diff",
     108            "fullforce",
     109            "dist");
     110
     111#           "chipbackground"
    111112
    112113# loop through all labels and create tables for each that have content
     
    175176        $table .= "<br>\n";
    176177
    177         if ($haveLabelData) { 
    178             $labelTables{$label} = $table; 
     178        if ($haveLabelData) {
     179            $labelTables{$label} = $table;
    179180            if ($self->{verbose}) {print " - $label\n";}
    180181        }
     
    274275    # MDF surveys
    275276    my @mdfs = ("M31.nightlyscience",
    276             "MD01.nightlyscience", 
    277             "MD02.nightlyscience", 
    278             "MD03.nightlyscience", 
    279             "MD04.nightlyscience", 
    280             "MD05.nightlyscience", 
    281             "MD06.nightlyscience", 
    282             "MD07.nightlyscience", 
    283             "MD08.nightlyscience", 
    284             "MD09.nightlyscience", 
     277            "MD01.nightlyscience",
     278            "MD02.nightlyscience",
     279            "MD03.nightlyscience",
     280            "MD04.nightlyscience",
     281            "MD05.nightlyscience",
     282            "MD06.nightlyscience",
     283            "MD07.nightlyscience",
     284            "MD08.nightlyscience",
     285            "MD09.nightlyscience",
    285286            "MD10.nightlyscience",
    286287            "MD11.nightlyscience");
  • trunk/ippMonitor/czartool/czartool/Gpc1Db.pm

    r36478 r38206  
    1919    ${$id} = $stage."_id";
    2020
    21     if ($stage eq "chip") {${$joinTable}="chipProcessedImfile";}
    22     elsif ($stage eq "cam") {${$joinTable}="camProcessedExp";}
    23     elsif ($stage eq "fake") {${$joinTable}="fakeProcessedImfile";}
    24     elsif ($stage eq "warp") {${$joinTable}="warpSkyfile";}
    25     elsif ($stage eq "stack") {${$joinTable}="stackSumSkyfile";}
     21    if ($stage eq "chip")         {${$joinTable}="chipProcessedImfile";}
     22    elsif ($stage eq "cam")       {${$joinTable}="camProcessedExp";}
     23    elsif ($stage eq "fake")      {${$joinTable}="fakeProcessedImfile";}
     24    elsif ($stage eq "warp")      {${$joinTable}="warpSkyfile";}
     25    elsif ($stage eq "stack")     {${$joinTable}="stackSumSkyfile";}
    2626    elsif ($stage eq "staticsky") {${$id} = "sky_id"; ${$joinTable}="staticskyResult";}
    27     elsif ($stage eq "diff") {${$joinTable}="diffSkyfile";}
    28     elsif ($stage eq "magic") {${$joinTable}="magicNodeResult";}
    29     elsif ($stage eq "magicDS") {${$id} = "magic_ds_id"; ${$joinTable}="magicDSFile";}
    30     elsif ($stage eq "dist") {${$joinTable}="distComponent";}
    31     elsif ($stage eq "pub") {${$joinTable}="publishDone";}
     27    elsif ($stage eq "diff")      {${$joinTable}="diffSkyfile";}
     28    elsif ($stage eq "fullforce") {${$id} = "ff_id"; ${$joinTable}="fullForceResult";}
     29    elsif ($stage eq "magic")     {${$joinTable}="magicNodeResult";}
     30    elsif ($stage eq "magicDS")   {${$id} = "magic_ds_id"; ${$joinTable}="magicDSFile";}
     31    elsif ($stage eq "dist")      {${$joinTable}="distComponent";}
     32    elsif ($stage eq "pub")       {${$joinTable}="publishDone";}
    3233    elsif ($stage eq "chipbackground") {${$id}="chip_bg_id"; ${$joinTable}="chipBackgroundImfile";}
    3334    else {
    34 
    3535        print "* ERROR: could not find joinTable and ID for '$stage' stage\n";
    3636        return 0;
     
    5555    elsif ($stage eq "staticsky" ) {return "staticskyRun";}
    5656    elsif ($stage eq "diff" ) {return "diffRun";}
     57    elsif ($stage eq "fullforce" ) {return "fullForceRun";}
    5758    elsif ($stage eq "magic" ) {return "magicRun";}
    5859    elsif ($stage eq "magicDS" ) {return "magicDSRun";}
     
    6768#
    6869# Returns the number of science exposures taken 'last night' for a given day
    69 # 
     70#
    7071# Notes:
    7172#
     
    7879    my ($self, $day) = @_;
    7980
    80     my $begin = "$day 03"; 
    81     my $end = "$day 17"; 
     81    my $begin = "$day 03";
     82    my $end = "$day 17";
    8283
    8384    my $query = $self->{_db}->prepare(<<SQL);
    8485    SELECT COUNT(*)
    85         FROM summitExp 
    86         WHERE dateobs > '$begin' 
     86        FROM summitExp
     87        WHERE dateobs > '$begin'
    8788        AND dateobs < '$end' AND
    8889        exp_type = 'OBJECT';
     
    9697#
    9798# Returns the number of exposures between two dates that have been registered with chip
    98 # 
     99#
    99100# Notes:
    100101#
     
    109110
    110111    my $query = $self->{_db}->prepare(<<SQL);
    111         SELECT rawExp.exp_id 
    112         FROM chipRun, rawExp 
    113         WHERE chipRun.exp_id = rawExp.exp_id 
    114         AND rawExp.dateobs >= '$beginDate' 
     112        SELECT rawExp.exp_id
     113        FROM chipRun, rawExp
     114        WHERE chipRun.exp_id = rawExp.exp_id
     115        AND rawExp.dateobs >= '$beginDate'
    115116        AND rawExp.dateobs <= '$endDate';
    116117SQL
     
    145146
    146147    my $query =  $self->{_db}->prepare(<<SQL);
    147     SELECT COUNT(DISTINCT $id) 
     148    SELECT COUNT(DISTINCT $id)
    148149        FROM $table
    149150        JOIN $joinTable USING ($id)
     
    164165
    165166    my $query = $self->{_db}->prepare(<<SQL);
    166     SELECT priority 
    167         FROM Label 
    168         WHERE label LIKE '$label' 
     167    SELECT priority
     168        FROM Label
     169        WHERE label LIKE '$label'
    169170SQL
    170171
     
    188189
    189190    my $query = $self->{_db}->prepare(<<SQL);
    190     SELECT COUNT(*) 
    191         FROM summitExp 
    192         JOIN newExp USING (summit_id) 
    193         JOIN rawExp USING(exp_id) 
    194         JOIN chipRun USING(exp_id) 
    195         WHERE summitExp.dateobs > '$date' 
    196         AND summitExp.exp_type = 'OBJECT' 
     191    SELECT COUNT(*)
     192        FROM summitExp
     193        JOIN newExp USING (summit_id)
     194        JOIN rawExp USING(exp_id)
     195        JOIN chipRun USING(exp_id)
     196        WHERE summitExp.dateobs > '$date'
     197        AND summitExp.exp_type = 'OBJECT'
    197198        AND chipRun.label = "$label";
    198199SQL
     
    225226            $query = $self->{_db}->prepare(<<SQL);
    226227            SELECT COUNT(state)
    227                 FROM $table JOIN $joinTable 
    228                 USING($id) 
     228                FROM $table JOIN $joinTable
     229                USING($id)
    229230                WHERE label LIKE '$label'
    230                 AND state = 'update' 
     231                AND state = 'update'
    231232                AND data_state = 'update';
    232233SQL
     
    237238                FROM $table
    238239                WHERE label LIKE '$label'
    239                 AND state = 'update'; 
    240 SQL
    241        
     240                AND state = 'update';
     241SQL
     242
    242243        }
    243244
    244245    }
    245     # 'new' stuff really includes 'update' stuff, at least for LAP. This is a hack, but there you go   
     246    # 'new' stuff really includes 'update' stuff, at least for LAP. This is a hack, but there you go
    246247    elsif ($state eq "new") {
    247248
     
    249250            # get count of runs in new state and update state that have components left to update
    250251            $query = $self->{_db}->prepare(<<SQL);
    251             SELECT COUNT(DISTINCT $id) 
     252            SELECT COUNT(DISTINCT $id)
    252253                FROM $table LEFT JOIN $joinTable
    253254                USING($id)
    254                 WHERE label LIKE '$label' 
     255                WHERE label LIKE '$label'
    255256                AND (state = 'new' OR (state = 'update' AND data_state = 'update'))
    256257SQL
     
    260261                FROM $table
    261262                WHERE label LIKE '$label'
    262                 AND (state = 'new' OR state = 'update'); 
     263                AND (state = 'new' OR state = 'update');
    263264SQL
    264265        }
     
    267268
    268269        $query = $self->{_db}->prepare(<<SQL);
    269         SELECT COUNT(state) 
    270             FROM $table 
    271             WHERE label LIKE '$label' 
     270        SELECT COUNT(state)
     271            FROM $table
     272            WHERE label LIKE '$label'
    272273            AND state = '$state'
    273274SQL
     
    289290
    290291            my $query = $self->{_db}->prepare(<<SQL);
    291             SELECT component, streak_frac 
    292                 FROM magicDSFile 
    293                 JOIN magicDSRun USING(magic_ds_id) 
    294                 JOIN camRun USING(cam_id) 
    295                 JOIN chipRun USING(chip_id) 
    296                 WHERE exp_id = $exp_id 
     292            SELECT component, streak_frac
     293                FROM magicDSFile
     294                JOIN magicDSRun USING(magic_ds_id)
     295                JOIN camRun USING(cam_id)
     296                JOIN chipRun USING(chip_id)
     297                WHERE exp_id = $exp_id
    297298                AND camRun.label LIKE "%nightlyscience"
    298299                AND component LIKE "XY%";
     
    306307###########################################################################
    307308#
    308 # Returns average magic mask fraction, sum of mask fractions and chip count 
     309# Returns average magic mask fraction, sum of mask fractions and chip count
    309310# for a particular exposure
    310311#
     
    314315
    315316            my $query = $self->{_db}->prepare(<<SQL);
    316             SELECT AVG(streak_frac), SUM(streak_frac), COUNT(*) 
    317                 FROM magicDSFile 
    318                 JOIN magicDSRun USING(magic_ds_id) 
    319                 JOIN camRun USING(cam_id) 
    320                 JOIN chipRun USING(chip_id) 
    321                 WHERE exp_id = $exp_id 
     317            SELECT AVG(streak_frac), SUM(streak_frac), COUNT(*)
     318                FROM magicDSFile
     319                JOIN magicDSRun USING(magic_ds_id)
     320                JOIN camRun USING(cam_id)
     321                JOIN chipRun USING(chip_id)
     322                WHERE exp_id = $exp_id
    322323                AND camRun.label LIKE "%nightlyscience"
    323324                AND component LIKE "XY%"
  • trunk/ippMonitor/czartool/czartool/Plotter.pm

    r38062 r38206  
    1111
    1212my @allStages = (
    13         "chip",
    14         "cam",
    15         "fake",
    16         "warp",
    17         "stack",
    18         "staticsky",
    19         "diff",
    20         "dist",
    21         "pub",
    22         "chipbackground");
     13    "chip",
     14    "cam",
     15    "fake",
     16    "warp",
     17    "stack",
     18    "staticsky",
     19    "diff",
     20    "fullforce",
     21    "dist",
     22    "pub");
     23
     24#        "chipbackground"
    2325
    2426my @allIppToPspsStages = (
     
    671673        "set ylabel \"Space (TB)\";" .
    672674        "set xtic rotate by -90 scale 0;" .
    673         "set yrange [:100];" .
     675        "set yrange [:100];" .
    674676        "plot '".$tmpFile->filename."' " .
    675677        "using 2:xtic(1) t \"Used\" lt 7," .
     
    716718        "set style data histograms;" .
    717719        "set ylabel \"Space (TB)\";" .
    718         "set yrange [:100];" .
     720        "set yrange [:100];" .
    719721        "set xtic rotate by -90 scale 0;" .
    720722        "plot '".$tmpFile->filename."' " .
  • trunk/ippMonitor/czartool/roboczar.pl

    r33814 r38206  
    2222
    2323my @stages = (
    24         "chip",
    25         "cam",
    26         "fake",
    27         "warp",
    28         "stack",
    29         "staticsky",
    30         "diff",
    31         "dist",
    32         "chipbackground"); # TODO get from Pantasks
     24        "chip",
     25        "cam",
     26        "fake",
     27        "warp",
     28        "stack",
     29        "staticsky",
     30        "diff",
     31        "fullforce",
     32        "dist"); # TODO get from Pantasks
     33
     34#       "chipbackground"
    3335
    3436my @interestedServers = $config->getRoboczarInterestedServers();
     
    5759
    5860    if ($anythingToReport && $message ne $lastMessage) {
    59    
     61
    6062print "\n\n$message\n\n";
    6163
    6264        sendEmail($config->getRoboczarEmail(),
    63                 "Roboczar warnings", 
     65                "Roboczar warnings",
    6466                "$message\n");
    6567    }
     
    101103
    102104        if ($stageMetrics->getStuck()) {
    103        
     105
    104106            ${$message} = ${$message} ."\n - '$stage' is stuck with ".$stageMetrics->getFinalPending()." pending exposures (and ".$stageMetrics->getFaults()." faults)";
    105107            $anyStuckStages = 1;
     
    112114###########################################################################
    113115#
    114 # Checks that the important servers are running 
     116# Checks that the important servers are running
    115117#
    116118###########################################################################
     
    141143
    142144    return $anythingToReport;
    143 } 
     145}
    144146
    145147###########################################################################
    146148#
    147 # Sends an email 
     149# Sends an email
    148150#
    149151###########################################################################
  • trunk/ippMonitor/raw/czartool_labels.php

    r38200 r38206  
    3434        "stack",
    3535        "staticsky",
     36        "fullforce",
    3637        "diff",
    3738        "dist",
     
    3940        );
    4041        # "chipbackground");
     42
     43$getFailures = array(
     44        "chip"      => "failedChipProcessedImfile.php",
     45        "cam"       => "failedCamProcessedExp.php",
     46        "fake"      => "failedFakeProcessedImfile.php",
     47        "warp"      => "failedWarpSkyfiles.php",
     48        "stack"     => "failedStackSkyfile.php",
     49        "staticsky" => "failedStaticsky.php",
     50        "fullforce" => "failedChipProcessedImfile.php",
     51        "diff"      => "failedDiffSkyfile.php",
     52        "dist"      => "NONE",
     53        "pub"       => "publishDone.php",
     54);           
     55
     56$mainTable = array(
     57        "chip"      => "chipRun",
     58        "cam"       => "camRun",
     59        "fake"      => "fakeRun",
     60        "warp"      => "warpRun",
     61        "stack"     => "stackRun",
     62        "staticsky" => "staticskyRun",
     63        "fullforce" => "fullForceRun",
     64        "diff"      => "diffRun",
     65        "dist"      => "distRun",
     66        "pub"       => "publishRun",
     67);           
     68
     69$addSearchState = array(
     70        "chip"      => 0,
     71        "cam"       => 0,
     72        "fake"      => 1,
     73        "warp"      => 0,
     74        "stack"     => 0,
     75        "staticsky" => 0,
     76        "fullforce" => 0,
     77        "diff"      => 0,
     78        "dist"      => 0,
     79        "pub"       => 0,
     80);           
    4181
    4282$servers=array(
     
    448488        $anyFaults = false;
    449489
    450         // Note: Do not include the state for chip, cam, and warp so that both new and update faults are listed
    451         $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel ; # . "&chipRun.state=".$searchState;
    452         getStateAndFaults($db, $thisLabel, $selectedState, "chip", $str, $anyFaults);
    453         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    454 
    455         $link = "failedCamProcessedExp.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $thisLabel ; # . "&camRun.state=".$searchState;
    456         getStateAndFaults($db, $thisLabel, $selectedState, "cam", $str, $anyFaults);
    457         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    458 
    459         $link =  "failedFakeProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&fakeRun.label=" . $thisLabel . "&fakeRun.state=".$searchState;
    460         getStateAndFaults($db, $thisLabel, $selectedState, "fake", $str, $anyFaults);
    461         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    462 
    463         $link = "failedWarpSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $thisLabel ; # . "&warpRun.state=".$searchState;
    464         getStateAndFaults($db, $thisLabel, $selectedState, "warp", $str, $anyFaults);
    465         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    466 
    467         $link = "failedStackSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $thisLabel . "&stackRun.state=".$searchState;
    468         getStateAndFaults($db, $thisLabel, $selectedState, "stack", $str, $anyFaults);
    469         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    470 
    471         $link = "failedStaticsky.php?pass=" . $pass . "&proj=" . $proj . "&staticskyRun.label=" . $thisLabel . "&staticskyRun.state=".$searchState;
    472         getStateAndFaults($db, $thisLabel, $selectedState, "staticsky", $str, $anyFaults);
    473         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    474 
    475         $link = "failedDiffSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $thisLabel . "&diffRun.state=".$searchState;
    476         getStateAndFaults($db, $thisLabel, $selectedState, "diff", $str, $anyFaults);
    477         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    478 
    479         $link = $defaultlink;
    480         getStateAndFaults($db, $thisLabel, $selectedState, "dist", $str, $anyFaults);
    481         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$distributing) ? "gray" : null);
    482 
    483         $link = "publishDone.php?pass=" . $pass . "&proj=" . $proj . "&publishRun.label=" . $thisLabel . "&publishRun.state=".$searchState;
    484         getStateAndFaults($db, $thisLabel, $selectedState, "pub", $str, $anyFaults);
    485         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$publishing) ? "gray" : null);
    486 
    487         $link = $defaultlink;
    488         getStateAndFaults($db, $thisLabel, $selectedState, "chipbackground", $str, $anyFaults);
    489         createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0);
    490 
     490        foreach ($stages as &$stage) {
     491          // Note: Do not include the state for chip, cam, and warp so that both new and update faults are listed
     492          $myTable    = $mainTable[$stage];
     493          $myFailures = $getFailures[$stage];
     494
     495          if ("$myFailures" == "NONE") {
     496            $link = $defaultlink;
     497          } else {
     498            $link = $getFailures[$stage] . "?pass=" . $pass . "&proj=" . $proj . "&" . $myTable . ".label=" . $thisLabel;
     499            if ($addSearchState[$stage]) {
     500              // Note: Do not include the state for chip, cam, and warp so that both new and update faults are listed
     501              $link .= "&" . $myTable . ".state=" . $searchState;
     502            }
     503          }
     504          getStateAndFaults($db, $thisLabel, $selectedState, $stage, $str, $anyFaults);
     505
     506          $cellcolor = 0;
     507          if ("$stage" == "dist") {
     508            $cellcolor = (!$isUpdate && !$distributing) ? "gray" : 0;
     509          }
     510          if ("$stage" == "pub") {
     511            $cellcolor = (!$isUpdate && !$publishing) ? "gray" : 0;
     512          }
     513          createFormattedTableCell($anyFaults, $link, $str, $anyFaults, $cellcolor);
     514        }
    491515        echo "</tr>\n";
    492516    }
Note: See TracChangeset for help on using the changeset viewer.