- Timestamp:
- May 18, 2010, 5:06:01 PM (16 years ago)
- Location:
- branches/czw_branch/20100427
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
pstamp/scripts/pstamp_finish.pl (modified) (19 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/czw_branch/20100427
- Property svn:mergeinfo changed
-
branches/czw_branch/20100427/pstamp/scripts/pstamp_finish.pl
r27751 r28017 25 25 use PS::IPP::PStamp::Job qw( :standard ); 26 26 27 my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); 27 my %imagedb_cache; 28 29 my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); 28 30 29 31 # the char to the right of the bar may be used as a single - alias for the longer name … … 34 36 'req_file=s' => \$req_file, 35 37 'product=s' => \$product, 36 'out _dir=s' => \$out_dir,38 'outdir=s' => \$outdir, 37 39 'dbname=s' => \$dbname, 38 40 'dbserver=s' => \$dbserver, … … 44 46 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; 45 47 46 die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n" 47 if !$req_id or !$req_name or !$req_file or !$product or !$out_dir; 48 die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n" 49 if !$req_id or !$req_name or !$req_file or !$product or !$outdir; 50 51 die "outdir is NULL\n" if $outdir eq "NULL"; 48 52 49 53 my $ipprc = PS::IPP::Config->new(); # IPP Configuration 50 54 if ($redirect_output) { 51 my $logDest = "$out_dir/psfinish.$req_id.log"; 55 # XXX: what happens here if the directory does not exist? We check below 56 my $logDest = "$outdir/psfinish.$req_id.log"; 52 57 $ipprc->redirect_output($logDest); 53 58 } … … 70 75 71 76 if ($product eq "NULL") { 77 # nothing more to do 72 78 stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); 73 79 } 74 75 my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');76 exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message77 80 78 81 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files … … 82 85 my $fileset = $req_name; 83 86 84 print STDERR "product: $product REQ_NAME: $req_name $out _dir\n" if $verbose;85 86 if (!-e $out _dir) {87 print STDERR "product: $product REQ_NAME: $req_name $outdir\n" if $verbose; 88 89 if (!-e $outdir) { 87 90 # something must have gone wrong parsing the request 88 print STDERR "output directory $out _dir does not exist\n";89 90 if (!mkdir $out _dir) {91 print STDERR "cannot create output directory $out _dir";91 print STDERR "output directory $outdir does not exist\n"; 92 93 if (!mkdir $outdir) { 94 print STDERR "cannot create output directory $outdir"; 92 95 stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); 93 96 } 94 97 95 98 96 } elsif (! -d $out _dir ) {99 } elsif (! -d $outdir ) { 97 100 # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the 98 101 # request 99 print STDERR "output directory $out _dir exists but is not a directory";102 print STDERR "output directory $outdir exists but is not a directory"; 100 103 stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); 101 104 } … … 116 119 } 117 120 118 # at this point we need to find out what kind of request type it is and 119 # split the processing depending on the request type 120 # the only processing required for a detectabilty query is to build the output fileset. 121 122 # the following is for a postage stamp request 123 124 my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); 121 # start building the list of files to be placed in the output fileset 122 my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps); 123 124 # results file 125 125 print $rlf "results.fits|||table|\n"; 126 # human readable representation of the results file 126 127 print $rlf "results.mdc|||text|\n"; 127 128 128 129 my $err_file = "parse_error.txt"; 129 if (-e "$out _dir/$err_file" ) {130 if (-e "$outdir/$err_file" ) { 130 131 print $rlf "$err_file|||text|\n"; 131 132 } 132 133 133 my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps);134 134 # The results table definition file 135 my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps); 135 136 136 137 # data for the header 137 138 print $tdf "$req_name|$req_id|\n"; 139 140 # get the list of jobs generated for this request 138 141 my @jobs; 139 142 { … … 151 154 print STDERR "Request $req_id produced no jobs.\n" 152 155 } 153 # This should not happen. A fake job should be queued156 # This should not happen. A fake job should have been entered 154 157 stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); 155 158 } else { 159 if (0) { 156 160 my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); 157 161 158 162 my $jobs = parse_md_list($metadata); 163 } 164 my $jobs = parse_md_fast($mdcParser, $output); 159 165 160 166 @jobs = @$jobs; … … 163 169 164 170 my $exp_info; 165 my $last_exp_id = -1;166 171 foreach my $job (@jobs) { 167 172 my $job_id = $job->{job_id}; … … 171 176 my $exp_id = $job->{exp_id}; 172 177 173 174 178 if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { 175 # this request had a duplicate request name. We can't put the results 176 # on the data store since the product name is already used 177 # maybe we could be more clever about this and choose a fileset name 178 stop_request_and_exit($req_id, $fault); 179 # this request had a duplicate request name yet the parser didn't give 180 # it an "ERROR style name. 181 stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); 179 182 } 180 183 my ($row, $req_info, $project) = get_request_info($rows, $rownum); … … 198 201 # get the metadata for the exposure (if any i.e. stack) 199 202 # returns an appropriate string if !$exp_id 200 $exp_info = get_exposure_info($image_db, $exp_id); 201 $last_exp_id = $exp_id; 203 $exp_info = get_exposure_info($job_params, $image_db, $exp_id); 202 204 } else { 203 205 my $filter = $job_params->{filter}; 204 206 $filter = "0" if !$filter; 205 207 $exp_info = "0|0|0|$filter|0|0"; 206 $last_exp_id = -1;207 208 } 208 209 209 210 if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) { 210 my $jreglist = "$out _dir/reglist$job_id";211 my $jreglist = "$outdir/reglist$job_id"; 211 212 if (open JRL, "<$jreglist") {; 212 213 # process the reglist file to get the list of files produced by this job … … 225 226 # If not found check the PHU. If that doesn't work just set them to zero. 226 227 # XXX do this more cleanly 227 my (undef, $ra_deg, $dec_deg) = split " ", `echo $out _dir/$img_name | fields -x 0 RA_DEG DEC_DEG`;228 my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`; 228 229 if (!defined $ra_deg) { 229 (undef, $ra_deg, $dec_deg) = split " ", `echo $out _dir/$img_name | fields RA_DEG DEC_DEG`;230 (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`; 230 231 } 231 232 $ra_deg = 0.0 if (!$ra_deg); … … 258 259 # make the results file 259 260 { 260 my $command = "$pstamp_results --input $table_def_name --output $out _dir/results.fits";261 my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits"; 261 262 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 262 263 run(command => $command, verbose => $verbose); … … 266 267 } else { 267 268 # dump a textual representation 268 my $command = "$pstampdump $out _dir/results.fits > $out_dir/results.mdc";269 my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc"; 269 270 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 270 271 run(command => $command, verbose => $verbose); … … 278 279 # register the fileset 279 280 my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS"; 280 $command .= " --link --datapath $out_dir --ps0 $req_id"; 281 $command .= " --dbname $dbname" if $dbname; 281 $command .= " --link --datapath $outdir --ps0 $req_id"; 282 # XXX: let dsreg and config handle resolving dbname and dbserver 283 # $command .= " --dbname $dbname" if $dbname; 282 284 283 285 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 292 294 # set the request's state to stop 293 295 { 294 my $command = "$pstamptool -updatereq -req_id $req_id -s tate stop -fault $request_fault";296 my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault"; 295 297 $command .= " -dbname $dbname" if $dbname; 296 298 $command .= " -dbserver $dbserver" if $dbserver; … … 308 310 my $fault = shift; 309 311 310 my $command = "$pstamptool -updatereq -req_id $req_id -s tate stop -fault $fault";312 my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault"; 311 313 $command .= " -dbname $dbname" if $dbname; 312 314 $command .= " -dbserver $dbserver" if $dbserver; … … 345 347 $data_group = "null" if !$data_group; 346 348 347 # This is ugly, error prone and hard to change.349 # XXX: This is ugly, error prone and hard to change. 348 350 # Create a results file module and provide a list of the names (we have the data in the columns) 349 351 my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|"; … … 376 378 my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); 377 379 380 # no need to use parse_md_fast here 378 381 my $results = parse_md_list($metadata); 379 382 if (scalar @$results != 1) { … … 385 388 386 389 sub get_exposure_info { 390 my $job_params = shift; 387 391 my $image_db= shift; 388 392 my $exp_id = shift; 389 393 390 if (!$exp_id or !$image_db) { 391 # no exposure id just return zeros 392 # XXX: we could put a value in for filter, but we don't have a good place to find it 393 394 #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; 395 return "0|0|0|0|0|0"; 396 } 397 398 my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); 399 if ($missing_tools) { 400 warn("Can't find required tools."); 401 exit ($PS_EXIT_CONFIG_ERROR); 402 } 403 404 my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; 405 406 # run the tool and parse the output 407 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 408 run(command => $command, verbose => $verbose); 409 unless ($success) { 410 # not sure if we should die here 411 die "cannot get exposure information for $exp_id from image database $image_db"; 412 } 413 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 414 415 my $output = join "", @$stdout_buf; 416 if (!$output) { 417 print STDERR "no output returned from $command\n" if $verbose; 418 return undef; 419 } 420 my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); 421 422 my $exposures = parse_md_list($metadata); 423 my $numExp = @$exposures; 424 425 die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; 394 my ($dateobs, $ra, $decl, $filter, $exp_time, $exp_name); 395 396 $dateobs = $job_params->{dateobs}; 397 $ra = $job_params->{ra}; 398 $decl = $job_params->{decl}; 399 $filter = $job_params->{filter}; 400 $exp_time = $job_params->{exp_time}; 401 $exp_name = $job_params->{exp_name}; 402 403 unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name) { 404 # job params don't have all of the values that we need (most likely this is a diff stage job) 405 # go look up the exposure if we have one 406 if (!$exp_id or !$image_db) { 407 # no exposure id just return zeros 408 # XXX: we could put a value in for filter, but we don't have a good place to find it 409 410 #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; 411 return "0|0|0|0|0|0"; 412 } 413 414 my $exp; 415 # get cache of exposures for this image_db 416 my $exp_cache = $imagedb_cache{$image_db}; 417 if (defined $exp_cache) { 418 $exp = $exp_cache->{$exp_id}; 419 } 420 421 unless (defined $exp) { 422 423 my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); 424 if ($missing_tools) { 425 warn("Can't find required tools."); 426 exit ($PS_EXIT_CONFIG_ERROR); 427 } 428 429 my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; 430 431 # run the tool and parse the output 432 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 433 run(command => $command, verbose => $verbose); 434 unless ($success) { 435 # not sure if we should die here 436 die "cannot get exposure information for $exp_id from image database $image_db"; 437 } 438 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 439 440 my $output = join "", @$stdout_buf; 441 if (!$output) { 442 print STDERR "no output returned from $command\n" if $verbose; 443 return undef; 444 } 445 if (0) { 446 my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); 447 448 my $exposures = parse_md_list($metadata); 449 } 450 my $exposures = parse_md_fast($mdcParser, $output); 451 my $numExp = @$exposures; 452 453 die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; 454 455 $exp = $exposures->[0]; 456 457 unless (defined $exp_cache) { 458 my %new_exp_cache; 459 $exp_cache = \%new_exp_cache; 460 $imagedb_cache{$image_db} = $exp_cache; 461 } 462 $exp_cache->{$exp_id} = $exp; 463 } else { 464 print "found $exp_id in cache\n"; 465 } 466 467 #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; 468 $dateobs = $exp->{dateobs}; 469 $ra = $exp->{ra}; 470 $decl = $exp->{decl}; 471 $filter = $exp->{filter}; 472 $exp_time = $exp->{exp_time}; 473 $exp_name = $exp->{exp_name}; 474 } 475 476 die "failed to find exp_info for $exp_id" unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name); 426 477 427 my $exp = $exposures->[0];428 429 #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";430 431 478 use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0); 432 my $ra_deg = $ exp->{ra}* RADIANS_TO_DEGREES;433 my $decl_deg = $ exp->{decl}* RADIANS_TO_DEGREES;434 my $mjd_obs = dateobs_to_mjd($ exp->{dateobs});435 436 my $info = "$mjd_obs|$ra_deg|$decl_deg|$ exp->{filter}|$exp->{exp_time}|$exp->{exp_name}";479 my $ra_deg = $ra * RADIANS_TO_DEGREES; 480 my $decl_deg = $decl * RADIANS_TO_DEGREES; 481 my $mjd_obs = dateobs_to_mjd($dateobs); 482 483 my $info = "$mjd_obs|$ra_deg|$decl_deg|$filter|$exp_time|$exp_name"; 437 484 438 485 return $info;
Note:
See TracChangeset
for help on using the changeset viewer.
