Index: branches/pap/pstamp/scripts/pstamp_finish.pl =================================================================== --- branches/pap/pstamp/scripts/pstamp_finish.pl (revision 27708) +++ branches/pap/pstamp/scripts/pstamp_finish.pl (revision 28003) @@ -25,5 +25,7 @@ use PS::IPP::PStamp::Job qw( :standard ); -my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); +my %imagedb_cache; + +my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); # the char to the right of the bar may be used as a single - alias for the longer name @@ -34,5 +36,5 @@ 'req_file=s' => \$req_file, 'product=s' => \$product, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'dbname=s' => \$dbname, 'dbserver=s' => \$dbserver, @@ -44,10 +46,13 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n" - if !$req_id or !$req_name or !$req_file or !$product or !$out_dir; +die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n" + if !$req_id or !$req_name or !$req_file or !$product or !$outdir; + +die "outdir is NULL\n" if $outdir eq "NULL"; my $ipprc = PS::IPP::Config->new(); # IPP Configuration if ($redirect_output) { - my $logDest = "$out_dir/psfinish.$req_id.log"; + # XXX: what happens here if the directory does not exist? We check below + my $logDest = "$outdir/psfinish.$req_id.log"; $ipprc->redirect_output($logDest); } @@ -70,9 +75,7 @@ if ($product eq "NULL") { + # nothing more to do stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } - -my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT'); -exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -82,20 +85,20 @@ my $fileset = $req_name; - print STDERR "product: $product REQ_NAME: $req_name $out_dir\n" if $verbose; - - if (!-e $out_dir) { + print STDERR "product: $product REQ_NAME: $req_name $outdir\n" if $verbose; + + if (!-e $outdir) { # something must have gone wrong parsing the request - print STDERR "output directory $out_dir does not exist\n"; - - if (!mkdir $out_dir) { - print STDERR "cannot create output directory $out_dir"; + print STDERR "output directory $outdir does not exist\n"; + + if (!mkdir $outdir) { + print STDERR "cannot create output directory $outdir"; stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } - } elsif (! -d $out_dir ) { + } elsif (! -d $outdir ) { # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the # request - print STDERR "output directory $out_dir exists but is not a directory"; + print STDERR "output directory $outdir exists but is not a directory"; stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -116,24 +119,24 @@ } - # at this point we need to find out what kind of request type it is and - # split the processing depending on the request type - # the only processing required for a detectabilty query is to build the output fileset. - - # the following is for a postage stamp request - - my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); + # start building the list of files to be placed in the output fileset + my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps); + + # results file print $rlf "results.fits|||table|\n"; - print $rlf "results.mdc|||txt|\n"; + # human readable representation of the results file + print $rlf "results.mdc|||text|\n"; my $err_file = "parse_error.txt"; - if (-e "$out_dir/$err_file" ) { + if (-e "$outdir/$err_file" ) { print $rlf "$err_file|||text|\n"; } - my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps); - + # The results table definition file + my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps); # data for the header print $tdf "$req_name|$req_id|\n"; + + # get the list of jobs generated for this request my @jobs; { @@ -151,10 +154,13 @@ print STDERR "Request $req_id produced no jobs.\n" } - # This should not happen. A fake job should be queued + # This should not happen. A fake job should have been entered stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } else { +if (0) { my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); my $jobs = parse_md_list($metadata); +} + my $jobs = parse_md_fast($mdcParser, $output); @jobs = @$jobs; @@ -163,5 +169,4 @@ my $exp_info; - my $last_exp_id = -1; foreach my $job (@jobs) { my $job_id = $job->{job_id}; @@ -172,8 +177,7 @@ if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { - # this request had a duplicate request name. We can't put the results - # on the data store since the product name is already used - # maybe we could be more clever about this and choose a fileset name - stop_request_and_exit($req_id, $fault); + # this request had a duplicate request name yet the parser didn't give + # it an "ERROR style name. + stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } my ($row, $req_info, $project) = get_request_info($rows, $rownum); @@ -188,13 +192,22 @@ } - if ($exp_id ne $last_exp_id) { + my $job_params = get_job_parameters($job); + my $stage = ""; + if ($job_params) { + $stage = $job_params->{stage}; + } + + if ($stage ne 'stack') { # get the metadata for the exposure (if any i.e. stack) # returns an appropriate string if !$exp_id - $exp_info = get_exposure_info($image_db, $exp_id); - $last_exp_id = $exp_id; + $exp_info = get_exposure_info($job_params, $image_db, $exp_id); + } else { + my $filter = $job_params->{filter}; + $filter = "0" if !$filter; + $exp_info = "0|0|0|$filter|0|0"; } if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) { - my $jreglist = "$out_dir/reglist$job_id"; + my $jreglist = "$outdir/reglist$job_id"; if (open JRL, "<$jreglist") {; # process the reglist file to get the list of files produced by this job @@ -213,7 +226,7 @@ # If not found check the PHU. If that doesn't work just set them to zero. # XXX do this more cleanly - my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`; + my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`; if (!defined $ra_deg) { - (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`; + (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`; } $ra_deg = 0.0 if (!$ra_deg); @@ -246,5 +259,5 @@ # make the results file { - my $command = "$pstamp_results --input $table_def_name --output $out_dir/results.fits"; + my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -254,5 +267,5 @@ } else { # dump a textual representation - my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc"; + my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -266,6 +279,7 @@ # register the fileset my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS"; - $command .= " --link --datapath $out_dir --ps0 $req_id"; - $command .= " --dbname $dbname" if $dbname; + $command .= " --link --datapath $outdir --ps0 $req_id"; +# XXX: let dsreg and config handle resolving dbname and dbserver +# $command .= " --dbname $dbname" if $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -280,5 +294,5 @@ # set the request's state to stop { - my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -296,5 +310,5 @@ my $fault = shift; - my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -333,5 +347,5 @@ $data_group = "null" if !$data_group; - # This is ugly, error prone and hard to change. + # XXX: This is ugly, error prone and hard to change. # Create a results file module and provide a list of the names (we have the data in the columns) my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|"; @@ -344,55 +358,128 @@ } +sub get_job_parameters { + my $job = shift; + if (!$job->{outputBase}) { + print "get_job_parameters: $job->{job_id} has no outputBase\n"; + return undef; + } + my $params_file = $job->{outputBase} . '.mdc'; + if (! -e $params_file ) { + print "get_job_parameters: $job->{job_id} has no parameters file\n"; + return undef; + } + open IN, "<$params_file" or die "unable to open $params_file"; + my $data = join "", (); + close IN; + if (! $data ) { + print "get_job_parameters: parameters file is empty\n"; + return undef; + } + my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); + + # no need to use parse_md_fast here + my $results = parse_md_list($metadata); + if (scalar @$results != 1) { + print STDERR "get_job_params: failed to parse_md_list\n"; + return undef; + } + return $results->[0]; +} + sub get_exposure_info { + my $job_params = shift; my $image_db= shift; my $exp_id = shift; - if (!$exp_id or !$image_db) { - # no exposure id just return zeros - # XXX: we could put a value in for filter, but we don't have a good place to find it - - #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; - return "0|0|0|0|0|0"; - } - - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - - my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; - - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - die "cannot get exposure information for $exp_id from image database $image_db"; - } - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); - - my $exposures = parse_md_list($metadata); - my $numExp = @$exposures; - - die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; + my ($dateobs, $ra, $decl, $filter, $exp_time, $exp_name); + + $dateobs = $job_params->{dateobs}; + $ra = $job_params->{ra}; + $decl = $job_params->{decl}; + $filter = $job_params->{filter}; + $exp_time = $job_params->{exp_time}; + $exp_name = $job_params->{exp_name}; + + unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name) { + # job params don't have all of the values that we need (most likely this is a diff stage job) + # go look up the exposure if we have one + if (!$exp_id or !$image_db) { + # no exposure id just return zeros + # XXX: we could put a value in for filter, but we don't have a good place to find it + + #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + return "0|0|0|0|0|0"; + } + + my $exp; + # get cache of exposures for this image_db + my $exp_cache = $imagedb_cache{$image_db}; + if (defined $exp_cache) { + $exp = $exp_cache->{$exp_id}; + } + + unless (defined $exp) { + + my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); + } + + my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; + + # run the tool and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # not sure if we should die here + die "cannot get exposure information for $exp_id from image database $image_db"; + } + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + my $output = join "", @$stdout_buf; + if (!$output) { + print STDERR "no output returned from $command\n" if $verbose; + return undef; + } +if (0) { + my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); + + my $exposures = parse_md_list($metadata); +} + my $exposures = parse_md_fast($mdcParser, $output); + my $numExp = @$exposures; + + die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; + + $exp = $exposures->[0]; + + unless (defined $exp_cache) { + my %new_exp_cache; + $exp_cache = \%new_exp_cache; + $imagedb_cache{$image_db} = $exp_cache; + } + $exp_cache->{$exp_id} = $exp; + } else { + print "found $exp_id in cache\n"; + } + + #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + $dateobs = $exp->{dateobs}; + $ra = $exp->{ra}; + $decl = $exp->{decl}; + $filter = $exp->{filter}; + $exp_time = $exp->{exp_time}; + $exp_name = $exp->{exp_name}; + } + + die "failed to find exp_info for $exp_id" unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name); - my $exp = $exposures->[0]; - - #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; - use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0); - my $ra_deg = $exp->{ra} * RADIANS_TO_DEGREES; - my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES; - my $mjd_obs = dateobs_to_mjd($exp->{dateobs}); - - my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}"; + my $ra_deg = $ra * RADIANS_TO_DEGREES; + my $decl_deg = $decl * RADIANS_TO_DEGREES; + my $mjd_obs = dateobs_to_mjd($dateobs); + + my $info = "$mjd_obs|$ra_deg|$decl_deg|$filter|$exp_time|$exp_name"; return $info;