IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 18, 2010, 12:49:05 PM (16 years ago)
Author:
eugene
Message:

merging changes from trunk into branches/pap

Location:
branches/pap
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/pap

  • branches/pap/pstamp/scripts/pstamp_finish.pl

    r27670 r28003  
    2525use PS::IPP::PStamp::Job qw( :standard );
    2626
    27 my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
     27my %imagedb_cache;
     28
     29my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
    2830
    2931# the char to the right of the bar may be used as a single - alias for the longer name
    … …  
    3436           'req_file=s'     => \$req_file,
    3537           'product=s'      => \$product,
    36            'out_dir=s'      => \$out_dir,
     38           'outdir=s'       => \$outdir,
    3739           'dbname=s'       => \$dbname,
    3840           'dbserver=s'     => \$dbserver,
    … …  
    4446pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4547
    46 die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n"
    47     if !$req_id or !$req_name or !$req_file or !$product or !$out_dir;
     48die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n"
     49    if !$req_id or !$req_name or !$req_file or !$product or !$outdir;
     50
     51die "outdir is NULL\n" if $outdir eq "NULL";
    4852
    4953my $ipprc = PS::IPP::Config->new(); # IPP Configuration
    5054if ($redirect_output) {
    51     my $logDest = "$out_dir/psfinish.$req_id.log";
     55    # XXX: what happens here if the directory does not exist? We check below
     56    my $logDest = "$outdir/psfinish.$req_id.log";
    5257    $ipprc->redirect_output($logDest);
    5358}
    … …  
    7075
    7176if ($product eq "NULL") {
     77    # nothing more to do
    7278    stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
    7379}
    74 
    75 my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
    76 exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message
    7780
    7881my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    … …  
    8285    my $fileset = $req_name;
    8386
    84     print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
    85 
    86     if (!-e $out_dir) {
     87    print STDERR "product: $product  REQ_NAME: $req_name $outdir\n" if $verbose;
     88
     89    if (!-e $outdir) {
    8790        # something must have gone wrong parsing the request
    88         print STDERR  "output directory $out_dir does not exist\n";
    89 
    90         if (!mkdir $out_dir) {
    91             print STDERR "cannot create output directory $out_dir";
     91        print STDERR  "output directory $outdir does not exist\n";
     92
     93        if (!mkdir $outdir) {
     94            print STDERR "cannot create output directory $outdir";
    9295            stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
    9396        }
    9497
    9598
    96     } elsif (! -d $out_dir ) {
     99    } elsif (! -d $outdir ) {
    97100        # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
    98101        # request
    99         print STDERR "output directory $out_dir exists but is not a directory";
     102        print STDERR "output directory $outdir exists but is not a directory";
    100103        stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
    101104    }
    … …  
    116119    }
    117120
    118     # at this point we need to find out what kind of request type it is and
    119     # split the processing depending on the request type
    120     # the only processing required for a detectabilty query is to build the output fileset.
    121 
    122     # the following is for a postage stamp request
    123 
    124     my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps);
     121    # start building the list of files to be placed in the output fileset
     122    my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps);
     123
     124    # results file
    125125    print $rlf "results.fits|||table|\n";
    126     print $rlf "results.mdc|||txt|\n";
     126    # human readable representation of the results file
     127    print $rlf "results.mdc|||text|\n";
    127128
    128129    my $err_file = "parse_error.txt";
    129     if (-e "$out_dir/$err_file" ) {
     130    if (-e "$outdir/$err_file" ) {
    130131        print $rlf "$err_file|||text|\n";
    131132    }
    132133
    133     my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps);
    134 
     134    # The results table definition file
     135    my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
    135136
    136137    # data for the header
    137138    print $tdf "$req_name|$req_id|\n";
     139
     140    # get the list of jobs generated for this request
    138141    my @jobs;
    139142    {
    … …  
    151154                print STDERR "Request $req_id produced no jobs.\n"
    152155            }
    153             # This should not happen. A fake job should be queued
     156            # This should not happen. A fake job should have been entered
    154157            stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
    155158        } else {
     159if (0) {
    156160            my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
    157161
    158162            my $jobs = parse_md_list($metadata);
     163}
     164            my $jobs = parse_md_fast($mdcParser, $output);
    159165
    160166            @jobs = @$jobs;
    … …  
    163169
    164170    my $exp_info;
    165     my $last_exp_id = -1;
    166171    foreach my $job (@jobs) {
    167172        my $job_id = $job->{job_id};
    … …  
    172177
    173178        if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
    174             # this request had a duplicate request name. We can't put the results
    175             # on the data store since the product name is already used
    176             # maybe we could be more clever about this and choose a fileset name
    177             stop_request_and_exit($req_id, $fault);
     179            # this request had a duplicate request name yet the parser didn't give
     180            # it an "ERROR style name.
     181            stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
    178182        }
    179183        my ($row, $req_info, $project) = get_request_info($rows, $rownum);
    … …  
    188192        }
    189193
    190         if ($exp_id ne $last_exp_id) {
     194        my $job_params = get_job_parameters($job);
     195        my $stage = "";
     196        if ($job_params) {
     197            $stage = $job_params->{stage};
     198        }
     199
     200        if ($stage ne 'stack') {
    191201            # get the metadata for the exposure (if any i.e. stack)
    192202            # returns an appropriate string if !$exp_id
    193             $exp_info = get_exposure_info($image_db, $exp_id);
    194             $last_exp_id = $exp_id;
     203            $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
     204        } else {
     205            my $filter = $job_params->{filter};
     206            $filter = "0" if !$filter;
     207            $exp_info = "0|0|0|$filter|0|0";
    195208        }
    196209
    197210        if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
    198             my $jreglist = "$out_dir/reglist$job_id";
     211            my $jreglist = "$outdir/reglist$job_id";
    199212            if (open JRL, "<$jreglist") {;
    200213                # process the reglist file to get the list of files produced by this job
    … …  
    213226                    # If not found check the PHU. If that doesn't work just set them to zero.
    214227                    # XXX do this more cleanly
    215                     my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
     228                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
    216229                    if (!defined $ra_deg) {
    217                         (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`;
     230                        (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
    218231                    }
    219232                    $ra_deg = 0.0 if (!$ra_deg);
    … …  
    246259    # make the results file
    247260    {
    248         my $command = "$pstamp_results --input $table_def_name --output $out_dir/results.fits";
     261        my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
    249262        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    250263            run(command => $command, verbose => $verbose);
    … …  
    254267        } else {
    255268            # dump a textual representation
    256             my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc";
     269            my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
    257270            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    258271                run(command => $command, verbose => $verbose);
    … …  
    266279        # register the fileset
    267280        my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
    268         $command .= " --link --datapath $out_dir --ps0 $req_id";
    269         $command .= " --dbname $dbname" if $dbname;
     281        $command .= " --link --datapath $outdir --ps0 $req_id";
     282# XXX: let dsreg and config handle resolving dbname and dbserver
     283#        $command .= " --dbname $dbname" if $dbname;
    270284
    271285        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    280294    # set the request's state to stop
    281295    {
    282         my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault";
     296        my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault";
    283297        $command   .= " -dbname $dbname" if $dbname;
    284298        $command   .= " -dbserver $dbserver" if $dbserver;
    … …  
    296310    my $fault  = shift;
    297311
    298     my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault";
     312    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault";
    299313    $command   .= " -dbname $dbname" if $dbname;
    300314    $command   .= " -dbserver $dbserver" if $dbserver;
    … …  
    333347    $data_group = "null" if !$data_group;
    334348
    335     # This is ugly, error prone and hard to change.
     349    # XXX: This is ugly, error prone and hard to change.
    336350    # Create a results file module and provide a list of the names (we have the data in the columns)
    337351    my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
    … …  
    344358}
    345359
     360sub get_job_parameters {
     361    my $job = shift;
     362    if (!$job->{outputBase}) {
     363        print "get_job_parameters: $job->{job_id} has no outputBase\n";
     364        return undef;
     365    }
     366    my $params_file = $job->{outputBase} . '.mdc';
     367    if (! -e $params_file ) {
     368        print "get_job_parameters: $job->{job_id} has no parameters file\n";
     369        return undef;
     370    }
     371    open IN, "<$params_file" or die "unable to open $params_file";
     372    my $data = join "", (<IN>);
     373    close IN;
     374    if (! $data ) {
     375        print "get_job_parameters: parameters file is empty\n";
     376        return undef;
     377    }
     378    my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc");
     379
     380    # no need to use parse_md_fast here
     381    my $results = parse_md_list($metadata);
     382    if (scalar @$results != 1) {
     383        print STDERR "get_job_params: failed to parse_md_list\n";
     384        return undef;
     385    }
     386    return $results->[0];
     387}
     388
    346389sub get_exposure_info {
     390    my $job_params = shift;
    347391    my $image_db= shift;
    348392    my $exp_id = shift;
    349393
    350     if (!$exp_id or !$image_db) {
    351         # no exposure id just return zeros
    352         # XXX: we could put a value in for filter, but we don't have a good place to find it
    353 
    354         #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
    355         return "0|0|0|0|0|0";
    356     }
    357 
    358     my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
    359     if ($missing_tools) {
    360         warn("Can't find required tools.");
    361         exit ($PS_EXIT_CONFIG_ERROR);
    362     }
    363 
    364     my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id";
    365 
    366     # run the tool and parse the output
    367     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    368                 run(command => $command, verbose => $verbose);
    369     unless ($success) {
    370         # not sure if we should die here
    371         die "cannot get exposure information for $exp_id from image database $image_db";
    372     }
    373     my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    374 
    375     my $output = join "", @$stdout_buf;
    376     if (!$output) {
    377         print STDERR "no output returned from $command\n" if $verbose;
    378         return undef;
    379     }
    380     my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
    381 
    382     my $exposures = parse_md_list($metadata);
    383     my $numExp = @$exposures;
    384 
    385     die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1;
     394    my ($dateobs, $ra, $decl, $filter, $exp_time, $exp_name);
     395
     396    $dateobs  = $job_params->{dateobs};
     397    $ra       = $job_params->{ra};
     398    $decl     = $job_params->{decl};
     399    $filter   = $job_params->{filter};
     400    $exp_time = $job_params->{exp_time};
     401    $exp_name = $job_params->{exp_name};
     402
     403    unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name) {
     404        # job params don't have all of the values that we need (most likely this is a diff stage job)
     405        # go look up the exposure if we have one
     406        if (!$exp_id or !$image_db) {
     407            # no exposure id just return zeros
     408            # XXX: we could put a value in for filter, but we don't have a good place to find it
     409
     410            #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
     411            return "0|0|0|0|0|0";
     412        }
     413
     414        my $exp;
     415        # get cache of exposures for this image_db
     416        my $exp_cache = $imagedb_cache{$image_db};
     417        if (defined $exp_cache) {
     418            $exp = $exp_cache->{$exp_id};
     419        }
     420
     421        unless (defined $exp) {
     422
     423            my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
     424            if ($missing_tools) {
     425                warn("Can't find required tools.");
     426                exit ($PS_EXIT_CONFIG_ERROR);
     427            }
     428
     429            my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id";
     430
     431            # run the tool and parse the output
     432            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     433                        run(command => $command, verbose => $verbose);
     434            unless ($success) {
     435                # not sure if we should die here
     436                die "cannot get exposure information for $exp_id from image database $image_db";
     437            }
     438            my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
     439
     440            my $output = join "", @$stdout_buf;
     441            if (!$output) {
     442                print STDERR "no output returned from $command\n" if $verbose;
     443                return undef;
     444            }
     445if (0) {
     446            my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
     447
     448            my $exposures = parse_md_list($metadata);
     449}
     450            my $exposures = parse_md_fast($mdcParser, $output);
     451            my $numExp = @$exposures;
     452
     453            die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1;
     454
     455            $exp = $exposures->[0];
     456
     457            unless (defined $exp_cache) {
     458                my %new_exp_cache;
     459                $exp_cache = \%new_exp_cache;
     460                $imagedb_cache{$image_db} = $exp_cache;
     461            }
     462            $exp_cache->{$exp_id} = $exp;
     463        } else {
     464            print "found $exp_id in cache\n";
     465        }
     466       
     467        #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
     468        $dateobs  = $exp->{dateobs};
     469        $ra       = $exp->{ra};
     470        $decl     = $exp->{decl};
     471        $filter   = $exp->{filter};
     472        $exp_time = $exp->{exp_time};
     473        $exp_name = $exp->{exp_name};
     474    }
     475
     476    die "failed to find exp_info for $exp_id" unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name);
    386477   
    387     my $exp = $exposures->[0];
    388 
    389     #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
    390 
    391478    use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0);
    392     my $ra_deg   = $exp->{ra} * RADIANS_TO_DEGREES;
    393     my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES;
    394     my $mjd_obs = dateobs_to_mjd($exp->{dateobs});
    395 
    396     my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}";
     479    my $ra_deg   = $ra * RADIANS_TO_DEGREES;
     480    my $decl_deg = $decl * RADIANS_TO_DEGREES;
     481    my $mjd_obs = dateobs_to_mjd($dateobs);
     482
     483    my $info = "$mjd_obs|$ra_deg|$decl_deg|$filter|$exp_time|$exp_name";
    397484           
    398485    return $info;
Note: See TracChangeset for help on using the changeset viewer.