Index: trunk/pstamp/scripts/pstamp_finish.pl =================================================================== --- trunk/pstamp/scripts/pstamp_finish.pl (revision 27895) +++ trunk/pstamp/scripts/pstamp_finish.pl (revision 27896) @@ -24,4 +24,6 @@ use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); + +my %imagedb_cache; my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); @@ -34,5 +36,5 @@ 'req_file=s' => \$req_file, 'product=s' => \$product, - 'outdir=s' => \$outdir, + 'outdir=s' => \$outdir, 'dbname=s' => \$dbname, 'dbserver=s' => \$dbserver, @@ -47,6 +49,9 @@ if !$req_id or !$req_name or !$req_file or !$product or !$outdir; +die "outdir is NULL\n" if $outdir eq "NULL"; + my $ipprc = PS::IPP::Config->new(); # IPP Configuration if ($redirect_output) { + # XXX: what happens here if the directory does not exist? We check below my $logDest = "$outdir/psfinish.$req_id.log"; $ipprc->redirect_output($logDest); @@ -70,9 +75,7 @@ if ($product eq "NULL") { + # nothing more to do stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } - -my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT'); -exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -116,12 +119,10 @@ } - # at this point we need to find out what kind of request type it is and - # split the processing depending on the request type - # the only processing required for a detectabilty query is to build the output fileset. - - # the following is for a postage stamp request - + # start building the list of files to be placed in the output fileset my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps); + + # results file print $rlf "results.fits|||table|\n"; + # human readable representation of the results file print $rlf "results.mdc|||text|\n"; @@ -131,9 +132,11 @@ } + # The results table definition file my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps); - # data for the header print $tdf "$req_name|$req_id|\n"; + + # get the list of jobs generated for this request my @jobs; { @@ -151,10 +154,13 @@ print STDERR "Request $req_id produced no jobs.\n" } - # This should not happen. A fake job should be queued + # This should not happen. A fake job should have been entered stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } else { +if (0) { my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); my $jobs = parse_md_list($metadata); +} + my $jobs = parse_md_fast($mdcParser, $output); @jobs = @$jobs; @@ -163,5 +169,4 @@ my $exp_info; - my $last_exp_id = -1; foreach my $job (@jobs) { my $job_id = $job->{job_id}; @@ -171,10 +176,8 @@ my $exp_id = $job->{exp_id}; - if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { - # this request had a duplicate request name. We can't put the results - # on the data store since the product name is already used - # maybe we could be more clever about this and choose a fileset name - stop_request_and_exit($req_id, $fault); + # this request had a duplicate request name yet the parser didn't give + # it an "ERROR style name. + stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } my ($row, $req_info, $project) = get_request_info($rows, $rownum); @@ -198,11 +201,9 @@ # get the metadata for the exposure (if any i.e. stack) # returns an appropriate string if !$exp_id - $exp_info = get_exposure_info($image_db, $exp_id); - $last_exp_id = $exp_id; + $exp_info = get_exposure_info($job_params, $image_db, $exp_id); } else { my $filter = $job_params->{filter}; $filter = "0" if !$filter; $exp_info = "0|0|0|$filter|0|0"; - $last_exp_id = -1; } @@ -346,5 +347,5 @@ $data_group = "null" if !$data_group; - # This is ugly, error prone and hard to change. + # XXX: This is ugly, error prone and hard to change. # Create a results file module and provide a list of the names (we have the data in the columns) my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|"; @@ -377,4 +378,5 @@ my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); + # no need to use parse_md_fast here my $results = parse_md_list($metadata); if (scalar @$results != 1) { @@ -386,54 +388,98 @@ sub get_exposure_info { + my $job_params = shift; my $image_db= shift; my $exp_id = shift; - if (!$exp_id or !$image_db) { - # no exposure id just return zeros - # XXX: we could put a value in for filter, but we don't have a good place to find it - - #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; - return "0|0|0|0|0|0"; - } - - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - - my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; - - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - die "cannot get exposure information for $exp_id from image database $image_db"; - } - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); - - my $exposures = parse_md_list($metadata); - my $numExp = @$exposures; - - die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; + my ($dateobs, $ra, $decl, $filter, $exp_time, $exp_name); + + $dateobs = $job_params->{dateobs}; + $ra = $job_params->{ra}; + $decl = $job_params->{decl}; + $filter = $job_params->{filter}; + $exp_time = $job_params->{exp_time}; + $exp_name = $job_params->{exp_name}; + + unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name) { + # job params don't have all of the values that we need (most likely this is a diff stage job) + # go look up the exposure if we have one + if (!$exp_id or !$image_db) { + # no exposure id just return zeros + # XXX: we could put a value in for filter, but we don't have a good place to find it + + #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + return "0|0|0|0|0|0"; + } + + my $exp; + # get cache of exposures for this image_db + my $exp_cache = $imagedb_cache{$image_db}; + if (defined $exp_cache) { + $exp = $exp_cache->{$exp_id}; + } + + unless (defined $exp) { + + my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); + } + + my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; + + # run the tool and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # not sure if we should die here + die "cannot get exposure information for $exp_id from image database $image_db"; + } + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + my $output = join "", @$stdout_buf; + if (!$output) { + print STDERR "no output returned from $command\n" if $verbose; + return undef; + } +if (0) { + my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); + + my $exposures = parse_md_list($metadata); +} + my $exposures = parse_md_fast($mdcParser, $output); + my $numExp = @$exposures; + + die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; + + $exp = $exposures->[0]; + + unless (defined $exp_cache) { + my %new_exp_cache; + $exp_cache = \%new_exp_cache; + $imagedb_cache{$image_db} = $exp_cache; + } + $exp_cache->{$exp_id} = $exp; + } else { + print "found $exp_id in cache\n"; + } + + #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + $dateobs = $exp->{dateobs}; + $ra = $exp->{ra}; + $decl = $exp->{decl}; + $filter = $exp->{filter}; + $exp_time = $exp->{exp_time}; + $exp_name = $exp->{exp_name}; + } + + die "failed to find exp_info for $exp_id" unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name); - my $exp = $exposures->[0]; - - #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; - use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0); - my $ra_deg = $exp->{ra} * RADIANS_TO_DEGREES; - my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES; - my $mjd_obs = dateobs_to_mjd($exp->{dateobs}); - - my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}"; + my $ra_deg = $ra * RADIANS_TO_DEGREES; + my $decl_deg = $decl * RADIANS_TO_DEGREES; + my $mjd_obs = dateobs_to_mjd($dateobs); + + my $info = "$mjd_obs|$ra_deg|$decl_deg|$filter|$exp_time|$exp_name"; return $info;