IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 28, 2010, 3:09:59 PM (16 years ago)
Author:
watersc1
Message:

JPEG work should be done. Transitioning to try to pick up stackAssociation concept in this branch as well.

Location:
branches/czw_branch/20100519
Files:
6 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20100519

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_checkdependent.pl

    r27921 r28164  
    8080my $whole_run = ($component eq 'all');
    8181
     82# XXX: whole_run was a concept that isn't practical. Having one dependent for
     83# a whole run makes finding errors too hard. We always have a dependent for each
     84# component.
     85# XXX: remove the unneeded code
     86my_die("component = 'all' not supported", $PS_EXIT_PROG_ERROR) if $whole_run;;
     87
    8288if ($whole_run) {
    8389    if ($stage eq "chip") {
    … …  
    135141        print "skipping $command\n";
    136142    }
    137 } elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update') and
    138         ($whole_run or ($it->{data_state} eq 'cleaned'))) {
    139 
    140     # Update has not been queued for this component yet. 
    141 
    142     # XXX: if $whole_run this is going to repeatedly call -setimfiletoupdate. This will be a no-op
    143     # but is wasteful.
    144     # Maybe we should deal with having three states for pstampDependent.state. 'new', 'run', 'full'
    145     # need to queue the updates
     143} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) {
     144    #       For warp and diff stages we need to call the 'queue_update' subroutines even if the
     145    #       data_state is update in order to check the earlier stages in the pipeline
     146    #       For example if warpSkyfile is in update state but the chip run that it depends on hasn't
     147    #       been updated we need to go and queue it.
     148
     149    if ($it->{fault}) {
     150        my_die("Component faulted on update dep_id: $dep_id",
     151                $PS_EXIT_SYS_ERROR);
     152    }
     153
    146154    if ($stage eq 'chip') {
    147         # queue_update_chip takes an array so that queue_update_warp can pass it set of chips
     155        # check_states_chip takes an array so that check_states_warp can pass it set of chips
    148156        my $chips = [$it];
    149         $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic);
     157        $status = check_states_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic);
    150158    } elsif ($stage eq 'warp') {
    151         $status = queue_update_warp($it, $whole_run, $rlabel, $need_magic);
     159        $status = check_states_warp($it, $whole_run, $rlabel, $need_magic);
    152160    } elsif ($stage eq 'diff') {
    153         $status = queue_update_diff($it, $whole_run, $rlabel, $need_magic);
     161        $status = check_states_diff($it, $whole_run, $rlabel, $need_magic);
    154162    } else {
    155163        my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR);
    156164    }
    157165} else {
    158     # update for this component has been queued by hasn't completed yet or the state is an error state
    159     # XXX: in this state we aren't checking the dependents anymore. If something goes wrong this
    160     # will never complete. Consider calling the 'queue_update' functions again just to check. They are relatively cheap
    161 
    162 
    163166    print "${stage}Run $stage_id state is $it->{state} $component data_state is $it->{data_state}\n";
    164167
    … …  
    170173    my $job_fault = 0;
    171174
    172     # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl
    173     # so runs in this state should not get queued
     175if (0) {
    174176    if ($stage eq 'chip') {
     177        # XXX: There is no need to check this anymore. All magicked chipRuns have abs(burntool_state) >= 13
     178        # If something changes that causes an error, we will figure that out from the chip failure
    175179        my $burntool_state = $it->{burntool_state};
    176         if ($burntool_state and (abs($burntool_state) < 14)) {
     180        # XXX: get the value of 13 from the ppImage recipe
     181        if ($burntool_state and (abs($burntool_state) < 13)) {
    177182            print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n";
    178183            $job_fault = $PSTAMP_NOT_AVAILABLE;
    179184        }
    180185    }
     186}
    181187    if ($state eq 'error_cleaned') {
    182188        $job_fault = $PSTAMP_NOT_AVAILABLE;
    … …  
    191197    if (!$job_fault and ($stage eq 'chip')) {
    192198        # chip processing is done, start destreaking.
    193         $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state});
     199        $job_fault = check_states_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state});
    194200    }
    195201    if ($job_fault) {
    … …  
    201207
    202208
    203 sub queue_update_chip {
     209sub check_states_chip {
    204210    my $chip_id = shift;
    205211    my $whole_run = shift;  # if true queue entire run for update
    … …  
    216222            $raw_all_magicked &= ($chip->{raw_magicked} > 0);
    217223
    218             if ($chip->{state} =~ /error/) {
     224            if (($chip->{state} =~ /error/) or ($chip->{state} =~ /purged/) or ($chip->{state} =~ /scrubbed/)) {
     225                print "chipRun state is $chip->{chip_id} is in state $chip->{state} cannot update\n";
    219226                faultJobs('stop', undef, undef, $PSTAMP_GONE);
     227                return;
    220228            } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
    221229                my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}";
    … …  
    237245        $dsRun_state = $run->{dsRun_state};
    238246        $raw_all_magicked = ($run->{raw_magicked} > 0);
     247        my $state = $run->{state};
     248        if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/)) {
     249            print "chipRun state is $run->{chip_id} is in state $state cannot update\n";
     250            faultJobs('stop', undef, undef, $PSTAMP_GONE);
     251            return;
     252        }
    239253
    240254        # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update'
    … …  
    253267    }
    254268
    255     my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state);
     269    my $status = check_states_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state);
    256270
    257271    return $status;
    258272}
    259273
    260 sub queue_update_warp {
     274sub check_states_warp {
    261275    # check status of input chips. If state is not updatable set error code for job
    262276
    263     # if chipProcessedImfile.state is cleaned call queue_update_chip
     277    # if chipProcessedImfile.state is cleaned call check_states_chip
    264278
    265279    # need to code warptool -setskyfiletoupdate
    … …  
    301315        }
    302316
    303         if ($chips_ready) {
     317        if ($chips_ready and $skycell->{data_state} ne 'update') {
    304318            # the reason we defer setting the warp to update is so that we can handle error conditions at previous
    305319            # stages more easily.
    … …  
    317331            }
    318332        } elsif (scalar @chipsToUpdate > 0) {
    319             return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
     333            return check_states_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
    320334        }
    321335    } else {
    … …  
    354368            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
    355369            my $data = [$chipRun];
    356             return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
     370            return check_states_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
    357371        }
    358372    }
    … …  
    361375    return 0;
    362376}
    363 sub queue_update_diff {
     377sub check_states_diff {
    364378    my $metadata = shift;
    365379    my $whole_run = shift;  # if true queue entire run for update
    … …  
    392406
    393407            if ($warp->{data_state} ne 'full') {
    394                 return queue_update_warp($warp, 0, $rlabel, $need_magic);
     408                return check_states_warp($warp, 0, $rlabel, $need_magic);
    395409            }
    396410            # warps are ready fall through and queue the diff update
    … …  
    404418            if ($warp1->{data_state} ne 'full') {
    405419                $warps_ready = 0;
    406                 $status = queue_update_warp($warp1, 0, $rlabel, $need_magic);
     420                $status = check_states_warp($warp1, 0, $rlabel, $need_magic);
    407421                if ($status) {
    408422                    return $status;
    … …  
    415429            if ($warp2->{data_state} ne 'full') {
    416430                $warps_ready = 0;
    417                 $status = queue_update_warp($warp2, 0, $rlabel, $need_magic);
     431                $status = check_states_warp($warp2, 0, $rlabel, $need_magic);
    418432            }
    419433
    … …  
    467481
    468482            if ($warp->{data_state} ne 'full') {
    469                 return queue_update_warp($warp, 0, $rlabel, $need_magic);
     483                return check_states_warp($warp, 0, $rlabel, $need_magic);
    470484            }
    471485            # warps are ready fall through and queue the diff update
    … …  
    474488        }
    475489
    476         my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
    477         $command .= " -set_label $rlabel" if $rlabel;
    478 
    479         if (!$no_update) {
    480             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    481                         run(command => $command, verbose => $verbose);
    482             unless ($success) {
    483                 my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
    484             }
    485         } else {
    486             print "skipping $command\n";
     490        if ($skycell->{data_state} ne 'update') {
     491            my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
     492            $command .= " -set_label $rlabel" if $rlabel;
     493
     494            if (!$no_update) {
     495                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     496                            run(command => $command, verbose => $verbose);
     497                unless ($success) {
     498                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
     499                }
     500            } else {
     501                print "skipping $command\n";
     502            }
    487503        }
    488504    } else {
    … …  
    599615}
    600616
    601 sub queue_update_magicDSRun {
     617sub check_states_magicDSRun {
    602618    my $stage = shift;
    603619    my $stage_id = shift;
    … …  
    608624
    609625    # XXX: this code assumes that destreaking is handled at the chip stage
    610     my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip';
    611 
    612     # if called from queue_update_warp dsRun_state is unknown. Go find it.
     626    my_die ("check_states_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip';
     627
     628    # if called from check_states_warp dsRun_state is unknown. Go find it.
    613629    if (!$dsRun_state) {
    614630        my $command = "$chiptool -listrun -chip_id $stage_id";

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_finish.pl

    r27896 r28164  
    176176        my $exp_id = $job->{exp_id};
    177177
     178        my $error_string = get_error_string($fault);
     179
    178180        if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
    179181            # this request had a duplicate request name yet the parser didn't give
    … …  
    220222
    221223                    # add line to the table definition file
    222                     print $tdf "$rownum|$fault|$img_name|$job_id|";
     224                    print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
    223225
    224226                    # ra_deg and dec_deg are the coordinates of center of the stamp
    … …  
    241243            } else {
    242244                print STDERR "no reglist file for job $job_id\n" if $verbose;
    243                 print $tdf "$rownum|$fault|0|$job_id|";
     245                print $tdf "$rownum|$fault|$error_string|0|$job_id|";
    244246                print $tdf "0|0|";       # center of (non-existent) stamp
    245247                print $tdf "$exp_info|";

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_request_file

    r27751 r28164  
    2929
    3030printhelp($0) if $help;
     31
    3132pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    32 pod2usage( -msg => "Required options: --input",
    33            -exitval => 3) unless defined $input;
     33
     34unless (defined $input) {
     35    print STDERR "Required options: --input\n";
     36    printhelp($0);
     37}
    3438
    3539# The header kewords

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_results_file.pl

    r26289 r28164  
    5454        # error code from processing this image
    5555        { name => 'ERROR_CODE',type => 'V',  writetype => TULONG },
     56        # error string correspoding to ERROR_CODE
     57        { name => 'ERROR_STR',type => '24A', writetype => TSTRING },
    5658        # name of the postage stamp image
    5759        { name => 'IMG_NAME', type => '64A', writetype => TSTRING },

  • branches/czw_branch/20100519/pstamp/scripts/pstampparse.pl

    r28021 r28164  
    811811    # chipRun's can be in full state if destreaking is necessary
    812812    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned') and
    813         ($stage eq 'chip' and $state eq 'full')) {
     813        ($stage ne 'chip' and $state eq 'full')) {
    814814        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
    815815    }
Note: See TracChangeset for help on using the changeset viewer.