Index: trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 30565) +++ trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 30567) @@ -4,6 +4,6 @@ # # Check the status of a pending pstampDependent insuring that -# the dependent processing has been queued and whether it is -# finished or not +# the any update processing that is needed has been queued and determine check +# whether the processing has finished or not use warnings; @@ -27,6 +27,6 @@ my $IPP_DIFF_MODE_STACK_STACK = 4; -my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic); -my ($dbname, $dbserver, $verbose, $save_temps, $no_update); +my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count); +my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update); GetOptions( @@ -35,9 +35,11 @@ 'stage_id=i' => \$stage_id, 'component=s' => \$component, - 'imagedb=s' => \$imagedb, # dbname for images. + 'imagedb=s' => \$imagedb, # dbname for images lookups. 'rlabel=s' => \$rlabel, 'need_magic' => \$need_magic, + 'fault_count=i' => \$fault_count, + 'max_fault_count=i' => \$max_fault_count, 'dbname=s' => \$dbname, # postage stamp server dbname - 'dbserver=s' => \$dbserver, # postage stamp server dbserver + 'dbserver=s' => \$ps_dbserver, # postage stamp server dbserver 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -48,4 +50,7 @@ if !(defined $dep_id and defined $stage and defined $stage_id and defined $component and defined $imagedb); + +$max_fault_count = 5 if !$max_fault_count; +$fault_count = 0 if !defined $fault_count; my $missing_tools; @@ -61,5 +66,14 @@ } +my $ipprc = PS::IPP::Config->new(); + +if (!$ps_dbserver) { + $ps_dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $ps_dbserver"; + # Append imagedb to the ippTools +# Note: configured DBSERVER is used for this server $chiptool .= " -dbname $imagedb"; $warptool .= " -dbname $imagedb"; @@ -68,42 +82,17 @@ $magicdstool .= " -dbname $imagedb"; -my $ipprc = PS::IPP::Config->new(); - -if (!$dbserver) { - $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); -} - my $tool; my $cmd; my $dsRun_state = ""; -my $whole_run = ($component eq 'all'); - -# XXX: whole_run was a concept that isn't practical. Having one dependent for -# a whole run makes finding errors too hard. We always have a dependent for each -# component. -# XXX: remove the unneeded code -my_die("component = 'all' not supported", $PS_EXIT_PROG_ERROR) if $whole_run;; - -if ($whole_run) { - if ($stage eq "chip") { - $cmd = "$chiptool -listrun -chip_id $stage_id"; - } elsif ($stage eq "warp") { - $cmd = "$warptool -listrun -warp_id $stage_id"; - } elsif ($stage eq "diff") { - $cmd = "$difftool -listrun -diff_id $stage_id"; - } else { - my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); - } + +if ($stage eq "chip") { + $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; +} elsif ($stage eq "warp") { + $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; +} elsif ($stage eq "diff") { + $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; } else { - if ($stage eq "chip") { - $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; - } elsif ($stage eq "warp") { - $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; - } elsif ($stage eq "diff") { - $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; - } else { - my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); - } + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); } @@ -124,11 +113,10 @@ } my $status = 0; -if (($it->{state} eq 'full') or ($it->{state} eq 'update') and ($whole_run or ($it->{data_state} eq 'full')) +if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full') and (!$need_magic or $magic_ok or $it->{magicked} > 0)) { - # This Dependency is satisfied. All done! + # This Dependency is satisfied. All done. Release the pstampJobs + # my $command = "$pstamptool -updatedependent -set_state full -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -143,23 +131,42 @@ } elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) { # For warp and diff stages we need to call the 'queue_update' subroutines even if the - # data_state is update in order to check the earlier stages in the pipeline - # For example if warpSkyfile is in update state but the chip run that it depends on hasn't - # been updated we need to go and queue it. - - if (($it->{state} ne 'cleaned') and $it->{fault}) { - my_die("Component faulted on update dep_id: $dep_id", - $PS_EXIT_SYS_ERROR); + # data_state is update in order to check the state of inputs in earlier stages in the pipeline + # For example if warpSkyfile is in update state but the chipRun that it depends on hasn't + # been updated we need to go and queue the chips for processing. + + my $fault = $it->{fault}; + if (($fault eq $PSTAMP_GONE) or (($it->{state} eq 'update') and $fault)) { + $fault_count++; + print "$stage $stage_id $component has fault $fault\n"; + if ($it->{fault} eq $PSTAMP_GONE) { + faultJobs($PSTAMP_GONE); + exit 0; + } elsif ($fault_count >= $max_fault_count) { + print "$stage $stage_id $component has faulted $fault_count times. Giving up\n"; + + faultComponent($stage, $stage_id, $component, $PSTAMP_GONE); + + # fault the jobs + faultJobs($PSTAMP_GONE); + exit 0; + } + + # assume the fault is transient. + my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR); } if ($stage eq 'chip') { - # check_states_chip takes an array so that check_states_warp can pass it set of chips + # check_states_chip takes an array so that check_states_warp can pass it a set of chips my $chips = [$it]; - $status = check_states_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); + $status = check_states_chip($it->{chip_id}, $chips, $rlabel, $need_magic); } elsif ($stage eq 'warp') { - $status = check_states_warp($it, $whole_run, $rlabel, $need_magic); + $status = check_states_warp($it, $rlabel, $need_magic); } elsif ($stage eq 'diff') { - $status = check_states_diff($it, $whole_run, $rlabel, $need_magic); + $status = check_states_diff($it, $rlabel, $need_magic); } else { my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR); + } + if ($status >= $PSTAMP_FIRST_ERROR_CODE) { + faultJobs($status); } } else { @@ -173,20 +180,8 @@ my $job_fault = 0; -if (0) { - if ($stage eq 'chip') { - # XXX: There is no need to check this anymore. All magicked chipRuns have abs(burntool_state) >= 13 - # If something changes that causes an error, we will figure that out from the chip failure - my $burntool_state = $it->{burntool_state}; - # XXX: get the value of 13 from the ppImage recipe - if ($burntool_state and (abs($burntool_state) < 13)) { - print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n"; - $job_fault = $PSTAMP_NOT_AVAILABLE; - } - } -} if ($state eq 'error_cleaned') { $job_fault = $PSTAMP_NOT_AVAILABLE; - } elsif (($state =~ /scrub/) or ($state =~ /purge/)) { - # jobs must have changed state since depenency was made + } elsif (($state =~ /scrub/) or ($state =~ /purge/) or ($state eq 'drop')) { + # Component state must have been changed state since dependency was inserted. print STDERR "Dependency cannot be satisfied\n"; $job_fault = $PSTAMP_GONE; @@ -195,5 +190,10 @@ my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR); } + if (!$job_fault and ($stage eq 'chip')) { + # should only get here with data_state 'full' and perhaps destreaking not done + my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR) + if $it->{data_state} ne 'full'; + # chip processing is done, start destreaking. my @chips; @@ -201,6 +201,7 @@ $job_fault = check_states_magicDSRun($stage, $stage_id, \@chips, $rlabel, $need_magic, $it->{raw_magicked}, $it->{magic_ds_id}, $it->{dsRun_state}); } - if ($job_fault) { - faultJobs($state, $stage, $stage_id, $job_fault); + + if ($job_fault >= $PSTAMP_FIRST_ERROR_CODE) { + faultJobs($job_fault); } } @@ -211,9 +212,7 @@ sub check_states_chip { my $chip_id = shift; - my $whole_run = shift; # if true queue entire run for update - my $metadatas = shift; # an array of hashes, either from -processedimfile or -listrun + my $metadatas = shift; # an array of hashes, either from chiptool -processedimfile or warptool -scmap my $rlabel = shift; # if defined a new label for the chipRun my $need_magic = shift; - my $dsRun_state; @@ -221,138 +220,32 @@ my @chips; my $magic_ds_id; - if (!$whole_run) { - foreach my $chip (@$metadatas) { - $dsRun_state = $chip->{dsRun_state}; - $raw_all_magicked &= ($chip->{raw_magicked} > 0); - $magic_ds_id = $chip->{magic_ds_id}; - - push @chips, $chip->{class_id}; - - my $state = $chip->{state}; - my $data_state = $chip->{data_state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or - ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) { - - print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - - return 0; - } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { - - my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chip->{fault}) { - # fault the dependent - my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault}); - } elsif ($chip->{dsFile_fault}) { - # fault the dependent - my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault}); - } - } - } else { - my $run = $metadatas->[0]; - $dsRun_state = $run->{dsRun_state}; - $raw_all_magicked = ($run->{raw_magicked} > 0); - my $state = $run->{state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/)) { - print "chipRun state is $run->{chip_id} is in state $state cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - return 0; - } - - # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update' - my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } - - my $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state); - - return $status; -} - -sub check_states_warp { - # check status of input chips. If state is not updatable set error code for job - - # if chipProcessedImfile.state is cleaned call check_states_chip - - my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update - my $rlabel = shift; # if defined a new label for the chipRun - my $need_magic = shift; - - my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked - - my $warp_id = $metadata->{warp_id}; - my $skycell_id = $metadata->{skycell_id}; - my $state = $metadata->{state}; - my $data_state = $metadata->{data_state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or - ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) { - print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - exit 0; - } - if (($state eq 'update') and ($metadata->{fault})) { - # fault dependent. - my $fault = $metadata->{fault}; - my_die("warp $warp_id $skycell_id faulted: $fault", $fault); - } - - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; - my $data = runToolAndParse($command, $verbose); - if (!$data or scalar @$data == 0) { - # This happens if the chipProcessedImfile disappears which happened when earlier - # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before - # deleing the row. - # Fault the jobs so that the Request can finish ... - faultJobs('stop', undef, undef, $PSTAMP_GONE); - # ... and fault the dependent so that we have a record of the error - my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", - $PSTAMP_GONE); - } - - - my $chips_ready = 1; - my @chipsToUpdate; - my $chip_id; - foreach my $chip (@$data) { - $chip_id = $chip->{chip_id}; - if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { - $chips_ready = 0; - $chip->{fault} = $chip->{chip_fault}; - push @chipsToUpdate, $chip; + + my $queued_update = 0; + foreach my $chip (@$metadatas) { + $dsRun_state = $chip->{dsRun_state}; + $raw_all_magicked &= ($chip->{raw_magicked} > 0); + $magic_ds_id = $chip->{magic_ds_id}; + + push @chips, $chip->{class_id}; + + my $state = $chip->{state}; + my $data_state = $chip->{data_state}; + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or + ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { + + print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n"; + my $error_code; + if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) { + $error_code = $PSTAMP_NOT_AVAILABLE; } else { - # this chip is good to go - } - } - - if ($chips_ready and $skycell->{data_state} ne 'update') { - # the reason we defer setting the warp to update is so that we can handle error conditions at previous - # stages more easily. - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $error_code = $PSTAMP_GONE; + } + + # caller will fault the jobs + return $error_code; + } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { + + # chiptool does more state checking to insure this isn't done prematurely. + my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; $command .= " -set_label $rlabel" if $rlabel; @@ -366,52 +259,129 @@ print "skipping $command\n"; } - } elsif (scalar @chipsToUpdate > 0) { - return check_states_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); - } - } else { - my $warpRun = $metadata; - my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data != 1; - - my $chipRun = $data->[0]; - - my $chipRunState = $chipRun->{state}; - if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { - print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; - faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); - return 0; - } - my $warpRunState = $warpRun->{state}; - if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { - # The inputs and outputs are ready. Queue the warpRun for update. - - # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chipRunState eq 'cleaned' or - (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { - my $data = [$chipRun]; - return check_states_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); - } - } - - # return value may be the return status of script so zero is good - return 0; -} + $queued_update = 1; + } elsif ($chip->{fault}) { + $fault_count++; + my $fault = $chip->{fault}; + + if ($fault eq $PSTAMP_GONE) { + # caller will fault jobs + return $PSTAMP_GONE; + } elsif ($fault_count > $max_fault_count) { + print "$stage $stage_id has faulted $fault_count times. Giving up\n"; + $fault = $PSTAMP_GONE; + faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # fault the dependent + my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault}); + } elsif ($chip->{dsFile_fault} eq $PSTAMP_GONE) { + print STDERR "magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} is GONE"; + return $PSTAMP_GONE; + } elsif ($chip->{dsFile_fault} and ($chip->{dsFile_data_state} eq 'update')) { + # fault the dependent + my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault}); + } + } + + my $status = 0; + if (!$queued_update) { + $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state); + } + + return $status; +} + +sub check_states_warp { + my $metadata = shift; + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $exit_status = 0; + + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + + my $warp_id = $metadata->{warp_id}; + my $skycell_id = $metadata->{skycell_id}; + my $state = $metadata->{state}; + my $data_state = $metadata->{data_state}; + + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or + ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { + print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n"; + my $error_code; + if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) { + $error_code = $PSTAMP_NOT_AVAILABLE; + } else { + $error_code = $PSTAMP_GONE; + } + return $error_code + } + if (($state eq 'update') and ($metadata->{fault})) { + # fault dependent. + my $fault = $metadata->{fault}; + print STDERR "warp $warp_id $skycell_id faulted: $fault"; + return $fault; + } + + my $skycell = $metadata; + + # get the list of input chips for this skycell + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; + my $data = runToolAndParse($command, $verbose); + if (!$data or scalar @$data == 0) { + # This happens if the chipProcessedImfile disappears which happened when earlier + # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before + # deleting the row. + print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id"; + return $PSTAMP_GONE; + } + + my $chips_ready = 1; + my @chipsToUpdate; + my $chip_id; + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $chips_ready = 0; + $chip->{fault} = $chip->{chip_fault}; + push @chipsToUpdate, $chip; + } else { + # this chip is done + } + } + + if ($chips_ready and $skycell->{data_state} ne 'update') { + # the reason we defer setting the warp to update is so that we can handle error conditions at previous + # stages more easily. + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic); + if ($fault) { + if ($fault eq $PSTAMP_GONE) { + # chip or dsfile that this skycell depends on has faulted in a way that is not recoverable + # fault the skycell + faultComponent('warp', $warp_id, $skycell->{skycell_id}, $PSTAMP_GONE); + } + $exit_status = $fault; + } + } + + # return value may be used as the return status of script so we use zero as success + return $exit_status; +} + sub check_states_diff { my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update my $rlabel = shift; # if defined a new label for the chipRun my $need_magic = shift; @@ -419,126 +389,132 @@ my $diff_id = $metadata->{diff_id}; my $diff_mode = $metadata->{diff_mode}; - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { - # check the state of the template stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; - - if ($stack->{state} ne 'full') { - print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # now check the warp - $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; - - if ($warp->{data_state} ne 'full') { - return check_states_warp($warp, 0, $rlabel, $need_magic); - } - # warps are ready fall through and queue the diff update - } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) { - my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp1 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1; - - my $warps_ready = 1; - my $status = 0; - if ($warp1->{data_state} ne 'full') { - $warps_ready = 0; - $status = check_states_warp($warp1, 0, $rlabel, $need_magic); - if ($status) { - return $status; + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { + # check the state of the template stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the warp + $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; + + if ($warp->{data_state} ne 'full') { + my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + return $warp_status; + } + # warps are ready fall through and queue the diff update + } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) { + my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp1 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1; + + my $warps_ready = 1; + my $warp_status = 0; + if ($warp1->{data_state} ne 'full') { + $warps_ready = 0; + $warp_status = check_states_warp($warp1, 0, $rlabel, $need_magic); + if ($warp_status) { + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); } - } - $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp2 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2; - - if ($warp2->{data_state} ne 'full') { - $warps_ready = 0; - $status = check_states_warp($warp2, 0, $rlabel, $need_magic); - } - - if (!$warps_ready) { - # don't queue the diff update yet - return $status; - } - - } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) { - # check the state of the input stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack1 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1; - - if ($stack1->{state} ne 'full') { - print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - # check the state of the template stack - $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack2 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2; - - if ($stack2->{state} ne 'full') { - print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # inputs are ready fall through and queue the diff update - } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) { - # check the state of the input stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; - my $stack = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; - - if ($stack->{state} ne 'full') { - print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # now check the template warp - $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; - - if ($warp->{data_state} ne 'full') { - return check_states_warp($warp, 0, $rlabel, $need_magic); - } - # warps are ready fall through and queue the diff update + return $warp_status; + } + } + $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; + my $warp2 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2; + + if ($warp2->{data_state} ne 'full') { + $warps_ready = 0; + $warp_status = check_states_warp($warp2, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + } + + if (!$warps_ready) { + # don't queue the diff update yet + return $warp_status; + } + # inputs are ready fall through and queue the diff update + + } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) { + # check the state of the input stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack1 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1; + + if ($stack1->{state} ne 'full') { + print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # check the state of the template stack + $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack2 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2; + + if ($stack2->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # inputs are ready fall through and queue the diff update + } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) { + # check the state of the input stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the template warp + $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; + my $warp = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; + + if ($warp->{data_state} ne 'full') { + my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + return $warp_status; + } + # warps are ready fall through and queue the diff update + } else { + my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR); + } + + if ($skycell->{data_state} ne 'update') { + my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } } else { - my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR); - } - - if ($skycell->{data_state} ne 'update') { - my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } - } else { - # XXX todo whole_run - return $PS_EXIT_PROG_ERROR; + print "skipping $command\n"; + } } @@ -633,22 +609,8 @@ } -sub faultJobs { - my ($state, $stage, $stage_id, $job_fault) = @_; - - my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to set pstampJob.fault for dep_id: $dep_id", - $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } -} - +# Check the data_state of the magicDSFile associated with this component (currently only chip stage is supported) +# Returns zero on success. +# returns the PSTAMP fault code to use for the jobs if an unrecoverable error is detected +# And faults the dependent if transient errors occur sub check_states_magicDSRun { my $stage = shift; @@ -661,5 +623,5 @@ my $dsRun_state = shift; - # XXX: this code assumes that destreaking is handled at the chip stage + # XXX: this code assumes that for update destreaking is only performed for chip stage my_die ("check_states_magicDSRun only implemented for chip stage", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; @@ -675,6 +637,7 @@ if ($need_magic and !$input_magicked) { if (!defined($dsRun_state) or ($dsRun_state eq 'NULL')) { + # it is arguably a programming error if we get here print "No magicDSRun for chipRun $stage_id and magic is required\n"; - faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED); + return $PSTAMP_NOT_DESTREAKED; } elsif (($dsRun_state eq 'cleaned') or ($dsRun_state eq 'update')) { foreach my $c (@$components) { @@ -684,5 +647,17 @@ my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR); } - if ($dsfile->{fault} > 0) { + if ($dsfile->{fault} eq $PSTAMP_GONE) { + print "magicDSFile has fault $PSTAMP_GONE\n"; + return $PSTAMP_GONE; + } + # destreak faults get cleared when the component is set to be updated + if (($dsfile->{data_state} eq 'update') and ($dsfile->{fault} > 0)) { + $fault_count++; + if ($fault_count > $max_fault_count) { + print "Destreak file $magic_ds_id $component for $stage $stage_id has faulted $fault_count times. Giving up\n"; + faultComponent('destreak', $magic_ds_id, $component, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # Assume fault is transient my_die("faulted magicDSFile for ${stage}Run $stage_id $c fault: $dsfile->{fault}", $PS_EXIT_UNKNOWN_ERROR); @@ -690,5 +665,6 @@ if ($dsfile->{data_state} eq 'cleaned') { $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c"; - # XXX: get the recoveryroot from a config file (it isn't actually used except to check whether it is in nebulous) + # XXX: get the recoveryroot from a config file + # (It isn't actually used except to check whether it is a nebulous path) $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover"; $command .= " -set_label $rlabel" if $rlabel; @@ -708,6 +684,7 @@ } } elsif ($dsRun_state eq 'failed_revert') { + # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE); + return $PSTAMP_NOT_AVAILABLE; } else { print "magicDSRun.state = $dsRun_state for chipRun $stage_id"; @@ -716,5 +693,51 @@ } } + return 0; +} + +sub faultJobs { + my ($job_fault) = @_; + + my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id"; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstampJob.fault for dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} + +sub faultComponent { + my ($stage, $stage_id, $component, $fault) = @_; + + my $command; + if ($stage eq 'chip') { + $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $component"; + } elsif ($stage eq 'warp') { + $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'diff') { + $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'destreak') { + $command = "$magicdstool -updatedestreakedfile -magic_ds_id $stage_id -component $component"; + } else { + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); + } + + $command .= " -fault $fault"; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp "$cmd failed"; + } + } else { + print "skipping $command\n"; + } } @@ -726,6 +749,4 @@ my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =