IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jan 5, 2015, 2:10:23 PM (12 years ago)
Author:
bills
Message:

Changes to the postage stamp dependency checking script.

  1. By default reduce the amount of output

that goes to the log files by a significant amount. This is accomplished by not including the
results of the ippTools commands used to check the state of the inputs.
Also added several print statements to make it easy to figure out what the state of things is.
Added a flag that may be used to include the ippTool outputs.

  1. Fixed a bug where faults in warps that are needed for diffs are not reported to check_states_diff.

This causes the dependent to never be faulted. This was the cause of several jobs that were
hung in limbo for weeks due to nfs problems during warp updates that are reported as fault 4
instead of 2. Only fault 2s are currently reverted.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/pstamp_checkdependent.pl

    r37772 r37784  
    2424my $ra_max_3PI_hours = 22.5;  # disable updates for skycells with ra greater than this (hours)
    2525my $ra_max_3PI = $ra_max_3PI_hours * 15 * 3.14149 / 180.; # rawExp.ra is in radians
     26
     27# verbose flag for parse commands
     28my $parse_verbose = 0;
    2629
    2730# XXX: put this in a module somewhere
    … …  
    4952    'dbserver=s'    =>  \$ps_dbserver,  # postage stamp server dbserver
    5053    'verbose'       =>  \$verbose,
     54    'parse-verbose' =>  \$parse_verbose,
    5155    'save-temps'    =>  \$save_temps,
    5256    'no-update'     =>  \$no_update,
    … …  
    7882}
    7983
     84print "\n==== Starting dependency checking for $dep_id $stage $stage_id $component at " . (scalar localtime) . ". ====\n";
     85
    8086if ($label) {
    8187    # rlabel is deprecated. Use one based on the supplied label parameter which is the current label
    … …  
    8490    # (and formerly in pstampparse.pl) is not particularly clean but it's simple.
    8591    my $new_rlabel = 'ps_ud_' . $label;
     92    $rlabel = "undefined" if !defined $rlabel;
    8693    if ($new_rlabel ne $rlabel) {
    8794        print "Notice: using $new_rlabel instead of $rlabel for update label.\n";
    … …  
    120127}
    121128
    122 my $it = runToolAndParseExpectOne($cmd, $verbose);
     129my $it = runToolAndParseExpectOne($cmd, $parse_verbose);
    123130
    124131my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it);
    … …  
    136143}
    137144
     145
    138146# magic is no longer rquired
    139147$need_magic = 0;
    … …  
    142150if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')) {
    143151
     152    print "\nDependency Satisfied for $stage $stage_id $component\n";
    144153    # This Dependency is satisfied. All done. Release the pstampJobs
    145154    #
    … …  
    178187        }
    179188
    180         # assume the fault is transient.
     189        # hope the fault is transient.
    181190        # fault the dependent to give the fault a chance to correct itself
    182191        my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR);
    … …  
    279288            ($fault eq $PSTAMP_GONE)) {
    280289
    281             print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n";
     290            print "chipRun state is $chip->{chip_id} has state: $state class_id $chip->{class_id} data_state: $data_state fault: $fault cannot update.\n";
    282291            my $error_code;
    283292            if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) {
    … …  
    291300        } elsif ($chip->{state} eq 'goto_cleaned') {
    292301
    293             # we don't need to check for full data_state
    294             # if ($chip->{data_state} eq 'full') {
    295302            {
     303                print "dependent chip is in state goto_cleaned, changing state to update.\n";
     304
    296305                # cleanup must not be running. Set state to update. If this chip is not 'full' it will be
    297306                # set to be updated the next time this script is invoked
    298                 # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks
     307                # XXX: In the very unlikely case that this chip run is actually in the running cleanup pantasks
    299308                # queue things may get confused but we can live with that
    300309                my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
    … …  
    313322            }
    314323        } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
     324
     325            print "Setting chip imfile $chip_id $chip->{class_id} to be updated.\n";
    315326
    316327            # chiptool does more state checking to insure this isn't done prematurely.
    … …  
    343354            # we've had a number of runs in this limbo state
    344355
     356            print "Changing chip run $chip_id to state update for $chip->{class_id} which is in data_state update.\n";
    345357            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
    346358            $command .= " -set_label $rlabel" if $rlabel;
    … …  
    364376                return $PSTAMP_GONE;
    365377            } elsif ($fault_count > $max_fault_count) {
    366                 print "$stage $stage_id has faulted $fault_count times. Giving up\n";
     378                print "Dependency for $stage $stage_id has been found in fault state $fault_count times. Giving up. Fault: $fault\n";
    367379                # XXX: stop faulting chips for now
    368380#                $fault = $PSTAMP_GONE;
    … …  
    421433    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
    422434         ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
    423         print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n";
     435        print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs.\n";
    424436        my $error_code;
    425437        if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) {
    … …  
    430442        return $error_code
    431443    }
    432     if (($data_state eq 'full') and ($metadata->{fault})) {
     444    if ((($data_state eq 'full') or ($data_state eq 'update')) and ($metadata->{fault})) {
    433445        # fault dependent.
    434446        my $fault = $metadata->{fault};
    435         print STDERR "warp $warp_id $skycell_id faulted: $fault";
     447        print STDERR "warp $warp_id $skycell_id faulted with code: $fault.";
     448        # XXX: TODO maybe: If ($fault != 2 and $fault != $PSTAMP_GONE) revert the skycell. The warp task only reverts fault 2
     449        # and sometimes warps will fault due to nfs problems that end up with fault = 4;.
    436450        return $fault;
    437451    }
    438452    if ($metadata->{quality} ne 0) {
    439         print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}\n";
     453        print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}.\n";
    440454        return $PSTAMP_GONE;
    441455    }
    … …  
    448462        # It's "safe" to set the state to update. If the skycell is not full it will be set to update
    449463        # the next time this task runs
     464        print "Change state of warp $warp_id to update.\n";
    450465        my $command = "$warptool -updaterun -set_state update -warp_id $warp_id";
    451466        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    465480    # get the list of input chips for this skycell
    466481    my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
    467     my $data = runToolAndParse($command, $verbose);
     482    my $data = runToolAndParse($command, $parse_verbose);
    468483    if (!$data or scalar @$data == 0) {
    469484        # This happens if the chipProcessedImfile disappears which happened when earlier
    470485        # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before
    471486        # deleting the row.
    472         print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id";
     487        print STDERR "failed to find warpSkyCellMap for warpRun $warp_id skycell_id $skycell_id";
    473488        return $PSTAMP_GONE;
    474489    }
    … …  
    478493    my $chip_id;
    479494    foreach my $chip (@$data) {
     495        # XXX: change tools to include cam_state (camRun.state). If it is not full the warp updates will never run.
    480496        my $cam_fault = $chip->{cam_fault};
    481497        if (defined $cam_fault and $cam_fault > 0) {
    … …  
    488504        $chip->{decl} = $metadata->{decl};
    489505
     506if (0) {
    490507        # if chip has been magicked before require it to be magicked again
    491508        # because the warp pending query requires it.
    492 if (0) {
    493509        if ($chip->{magicked} < 0) {
    494510            print "Input has been destreaked so we must destreak before warping\n";
    … …  
    514530        # the reason we defer setting the warp to update is so that we can handle error conditions at previous
    515531        # stages more easily.
     532        print "Chips are ready for warp $warp_id $skycell->{skycell_id}. Setting skycell to be updated.\n";
    516533        my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
    517534        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    527544        }
    528545    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
     546        print "Chips are ready for warp $warp_id $skycell->{skycell_id} but run state is $skycell->{state}. Changing state to update.\n";
    529547        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
    530548        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    566584    my $skycell_id = $skycell->{skycell_id};
    567585
    568     # if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') {
    569586    if ($metadata->{state} eq 'goto_cleaned') {
    570587        # cleanup must not be running. Set state to update. If the skycell is not 'full' it will be
    571         # set to be updated the next time this script is invoked
    572         print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n";
     588        # set to be updated the next time this script is invoked.
     589        print "Changing state of diffRun $diff_id from goto_cleaned to update.\n";
    573590        my $command = "$difftool -updaterun -set_state update -diff_id $diff_id";
    574591        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    589606        # check the state of the template stack
    590607        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    591         my $stack = runToolAndParseExpectOne($command, $verbose);
     608        my $stack = runToolAndParseExpectOne($command, $parse_verbose);
    592609        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    593610
    … …  
    597614            return $PSTAMP_GONE;
    598615        }
     616        print " Stack $skycell->{stack2} is ready.\n";
    599617
    600618        # now check the warp
    601619        $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    602         my $warp = runToolAndParseExpectOne($command, $verbose);
     620        my $warp = runToolAndParseExpectOne($command, $parse_verbose);
    603621        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    604622
    … …  
    615633        }
    616634        # warps are ready fall through and queue the diff update
     635        print " Warp $skycell->{warp1} $skycell_id is ready.\n";
    617636    } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
    618637        my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    619         my $warp1 = runToolAndParseExpectOne($command, $verbose);
     638        my $warp1 = runToolAndParseExpectOne($command, $parse_verbose);
    620639        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
    621640
    … …  
    637656        }
    638657        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    639         my $warp2 = runToolAndParseExpectOne($command, $verbose);
     658        my $warp2 = runToolAndParseExpectOne($command, $parse_verbose);
    640659        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
    641660
    … …  
    661680        # check the state of the input stack
    662681        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    663         my $stack1 = runToolAndParseExpectOne($command, $verbose);
     682        my $stack1 = runToolAndParseExpectOne($command, $parse_verbose);
    664683        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
    665684
    … …  
    671690        # check the state of the template stack
    672691        $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    673         my $stack2 = runToolAndParseExpectOne($command, $verbose);
     692        my $stack2 = runToolAndParseExpectOne($command, $parse_verbose);
    674693        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2;
    675694
    … …  
    684703        # check the state of the input stack
    685704        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    686         my $stack = runToolAndParseExpectOne($command, $verbose);
     705        my $stack = runToolAndParseExpectOne($command, $parse_verbose);
    687706        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    688707
    … …  
    695714        # now check the template warp
    696715        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    697         my $warp = runToolAndParseExpectOne($command, $verbose);
     716        my $warp = runToolAndParseExpectOne($command, $parse_verbose);
    698717        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    699718
    … …  
    736755            print "skipping $command\n";
    737756        }
     757    } else {
     758        print " diff is in update state\n";
    738759    }
    739760
    … …  
    763784    my $now = DateTime->now->mjd;
    764785    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
    765     print "Time to run $program: $dtime_tool\n";
     786    if ($dtime_tool > 0.1) {
     787        print "Time to run $program: $dtime_tool\n";
     788    }
    766789
    767790    my $buf = join "", @$stdout_buf;
    … …  
    777800
    778801    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
    779     print "Time to parse results from $program: $dtime_parse\n";
     802    if ($dtime_parse > 0.1) {
     803        print "Time to parse results from $program: $dtime_parse\n";
     804    }
    780805
    781806    return $results;
    … …  
    848873    if (!$dsRun_state) {
    849874        my $command = "$chiptool -listrun -chip_id $stage_id";
    850         my $data = runToolAndParse($command, $verbose);
     875        my $data = runToolAndParse($command, $parse_verbose);
    851876        my $chipRun = $data->[0];
    852877        $dsRun_state = $chipRun->{dsRun_state};
    … …  
    862887            foreach my $c (@$components) {
    863888                my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c";
    864                 my $dsfile = runToolAndParseExpectOne($command, $verbose);
     889                my $dsfile = runToolAndParseExpectOne($command, $parse_verbose);
    865890                if (!$dsfile) {
    866891                    my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR);
Note: See TracChangeset for help on using the changeset viewer.