IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 18, 2010, 5:06:01 PM (16 years ago)
Author:
watersc1
Message:

Almost working version of the mask-stats/software revision code.

Almost.

Location:
branches/czw_branch/20100427
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20100427

  • branches/czw_branch/20100427/pstamp/scripts/pstampparse.pl

    r27783 r28017  
    1717use Carp;
    1818use POSIX;
    19 use DateTime;
     19use Time::HiRes qw(gettimeofday);
    2020
    2121my $verbose;
    … …  
    2525my $request_file_name;
    2626my $mode = "list_uri";
    27 my $out_dir;
     27my $outdir;
    2828my $product;
    2929my $label;
    … …  
    3434    'file=s'    =>  \$request_file_name,
    3535    'req_id=s'  =>  \$req_id,
    36     'out_dir=s' =>  \$out_dir,
     36    'outdir=s'  =>  \$outdir,
    3737    'product=s' =>  \$product,
    3838    'label=s'   =>  \$label,
    … …  
    5050if ($mode ne "list_uri") {
    5151    die "req_id is required"   if !$req_id;
    52     die "out_dir is required"  if !$out_dir;
     52    die "outdir is required"  if !$outdir;
    5353    die "product is required"  if !$product;
    5454}
    … …  
    129129    # update the database with the request name. This will be used as the
    130130    # the output data store's product name
    131     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
    132     $command .= " -outProduct $product";
     131    my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     132    $command .= " -set_outProduct $product";
    133133    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    134134        run(command => $command, verbose => $verbose);
    … …  
    147147my $rows;
    148148{
     149    my $start_request_file = gettimeofday();
    149150    my $command = "$pstampdump $request_file_name";
    150151    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    154155    }
    155156    if (@$stdout_buf) {
    156         my $table =  $mdcParser->parse(join "", @$stdout_buf) or
    157             my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
    158         $rows = parse_md_list($table);
    159     }
     157        $rows = parse_md_fast($mdcParser, join "", @$stdout_buf);
     158    }
     159    my $dtime_request_file = gettimeofday() - $start_request_file;
     160    print "Time to read and parse request file: $dtime_request_file\n";
    160161
    161162}
    … …  
    223224    my $rownum   = $row->{ROWNUM};
    224225    if (!validID($rownum)) {
    225         print STDERR "$rownum is not a valid ROWNUM\n"  if $verbose;
     226        print STDERR "$rownum is not a valid ROWNUM\n";
    226227        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    227228        return 0;
    … …  
    229230    my $job_type = $row->{JOB_TYPE};
    230231    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
    231         print STDERR "$job_type is not a valid JOB_TYPE\n"  if $verbose;
     232        print STDERR "$job_type is not a valid JOB_TYPE\n";
    232233        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    233234        return 0;
    … …  
    237238    if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
    238239        ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
    239         print STDERR "$req_type is not a valid REQ_TYPE\n"  if $verbose;
     240        print STDERR "$req_type is not a valid REQ_TYPE\n";
    240241        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    241242        return 0;
    242243    }
     244    if ($job_type eq 'get_image') {
     245        unless ($req_type eq 'byid' or $req_type eq 'byexp') {
     246            print STDERR "REQ_TYPE must be 'byid' or 'byexp' for JOB_TYPE 'get_image'\n";
     247            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
     248            return 0;
     249        }
     250    }
    243251
    244252
    245253    my $component = $row->{COMPONENT};
    246254    if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
    247         $row->{COMPONENT} = $component = "";
     255        if ($job_type eq 'get_image') {
     256            $row->{COMPONENT} = 'all';
     257        } else {
     258            $row->{COMPONENT} = $component = "";
     259        }
    248260    }
    249261    $row->{TESS_ID} = "" if !defined $row->{TESS_ID};
    … …  
    258270    my $mjd_min = $row->{MJD_MIN};
    259271    if (defined($mjd_min) and !validNumber($mjd_min)) {
    260         print STDERR "$mjd_min is not a valid MJD_MIN\n"  if $verbose;
     272        print STDERR "$mjd_min is not a valid MJD_MIN\n";
    261273        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    262274        return 0;
    … …  
    264276    my $mjd_max = $row->{MJD_MAX};
    265277    if (defined($mjd_max) and !validNumber($mjd_max)) {
    266         print STDERR "$mjd_max is not a valid MJD_MAX\n"  if $verbose;
     278        print STDERR "$mjd_max is not a valid MJD_MAX\n";
    267279        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    268280        return 0;
    … …  
    288300
    289301    if (!$skycenter and !$component) {
    290         print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose;
     302        print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n";
    291303        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    292304        return 0;
    … …  
    295307    my $stage = $row->{IMG_TYPE};
    296308    if (!check_image_type($stage)) {
    297         print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose;
     309        print STDERR "invalid IMG_TYPE for row $rownum\n";
    298310        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    299311        return 0;
    … …  
    301313
    302314    if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) {
    303         print STDERR "invalid ROI for row $rownum\n" if $verbose;
     315        print STDERR "invalid ROI for row $rownum\n";
    304316        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    305317        return 0;
    … …  
    307319
    308320    if (($req_type eq "byexp") and ($stage eq "stack")) {
    309         print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose;
     321        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
    310322        insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
    311323        return 0;
    … …  
    319331    if ($req_type eq "bycoord") {
    320332        if (!$skycenter) {
    321             print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
     333            print STDERR "center must be specified in sky coordintes for bycoord";
    322334            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    323335            return 0;
    … …  
    327339    if (($req_type eq "byid") or ($req_type eq "bydiff")) {
    328340        if (!validID($row->{ID})) {
    329             print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose;
     341            print STDERR "ID must be a positive integer for req_type $req_type\n";
    330342            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    331343            return 0
    … …  
    349361    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    350362    if (!$proj_hash) {
    351         print STDERR "project $project not found\n"  if $verbose;
     363        print STDERR "project $project not found\n" ;
    352364        foreach $row (@$rowList) {
    353365            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
    … …  
    383395        }
    384396    }
     397
     398    $need_magic = 0 if $stage eq 'stack';
    385399   
    386400    my $numRows = scalar @$rowList;
    387 
    388 #    $tess_id = "" if !defined $tess_id;
    389 #    $component = "" if !defined $component;
    390401
    391402    print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
    … …  
    393404    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
    394405    # request specification. An array reference is returned.
    395     my $start_locate = DateTime->now->mjd;
     406    my $start_locate = gettimeofday();
    396407
    397408    # XXX: perhaps we should get rid of most of this argument list.
    … …  
    402413                $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
    403414
    404     # XXX: why use mjd? It doesn't have great precision.
    405     my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.;
     415    my $dtime_locate = gettimeofday() - $start_locate;
    406416    print "Time to locate_images for row $rownum $dtime_locate\n";
    407417
    408     if (!$imageList or !@$imageList) {
    409         print STDERR "no matching images found for row $rownum\n" if $verbose;
    410         # note in this case queueJobs inserts the fake job for these rows
    411     }
    412     # handle this
     418    # handle this XXX: what did I mean by this comment
    413419    $row->{need_magic} = $need_magic;
    414420
    … …  
    502508    $image->{job_args} = $args;
    503509
    504     # XXX: we can get rid of the following everything that we need is
    505     # in the params file
    506 
    507     $args .= " -file $imagefile";
    508 
    509     if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
    510         $args .= " -mask $image->{mask}";
    511     }
    512     if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
    513         $args .= " -variance $image->{weight}";
    514     }
    515 
    516510    my $base = basename($image->{image});
    517511    if (! $base =~ /.fits$/ ) {
    … …  
    520514    $base =~ s/.fits$//;
    521515           
    522     my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
    523     my $argslist = "${output_base}.args";
    524 
    525     # copy the argument list to a file
    526     open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
    527     print ARGSLIST "$args\n";
    528     close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
    529 
     516    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
    530517    write_params($output_base, $image);
    531518
    … …  
    536523    # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
    537524    if ($stage ne 'raw') {
    538         # not ready to handle diff update yet. May never support stack
    539         my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff'));
     525        # updates for stack stage not supported yet
     526        my $allow_wait_for_update = ($stage ne 'stack');
    540527        my $run_state = $image->{state};
    541528        my $data_state = $image->{data_state};
    … …  
    548535            $newState = 'stop';
    549536            $fault = $PSTAMP_GONE;
    550         } elsif ($allow_wait_for_update and
    551                 (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) {
     537        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
    552538            if ($stage eq 'chip') {
    553539                my $burntool_state = $image->{burntool_state};
    … …  
    556542                    $fault = $PSTAMP_NOT_AVAILABLE;
    557543                }
     544            }
     545            if (!$allow_wait_for_update) {
     546                print STDERR "wait for update not supported for stage $stage yet\n";
     547                $newState = 'stop';
     548                $fault = $PSTAMP_NOT_AVAILABLE;
    558549            }
    559550            if (!$fault) {
    … …  
    658649}
    659650
    660 #        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
    661651sub queueGetImageJobs
    662652{
    … …  
    706696        my $exp_id = $image->{exp_id};
    707697           
    708         my $output_base = "$out_dir/${rownum}_${job_num}";
     698        my $output_base = "$outdir/${rownum}_${job_num}";
    709699
    710700        write_params($output_base, $image);
    … …  
    731721                    # cause the image to be re-made
    732722                    # set up to queue an update run
    733                     queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db},
     723                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb},
    734724                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
    735725                }
    … …  
    878868    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
    879869
     870    # XXX: The update process for warp and subsequent stages requires
     871    # destreaking to run because the -pending queries require it when magicked > 0
     872    # queries.
     873    # The case of stack-stack diffs is taken care of in pstamp_checkdependent
     874    $need_magic = 1 if $imagedb eq 'gpc1';
     875
    880876    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
    881877        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
    … …  
    883879
    884880    my $dep_id;
    885     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component";
     881    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
    886882    $command .= " -need_magic" if $need_magic;
    887883
Note: See TracChangeset for help on using the changeset viewer.