Index: /trunk/pstamp/scripts/pstamp_finish.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_finish.pl (revision 19264) +++ /trunk/pstamp/scripts/pstamp_finish.pl (revision 19265) @@ -9,4 +9,5 @@ use Pod::Usage qw( pod2usage ); +use Time::Local; use Sys::Hostname; use IPC::Cmd 0.36 qw( can_run run ); @@ -82,9 +83,20 @@ } - - # this function is PStamp::RequestFile::read_request_file + if (! -e $req_file ) { + print STDERR "request file $req_file is missing\n"; + stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); + exit $PS_EXIT_CONFIG_ERROR; + } + + # this function is PStamp::RequestFile::read_request_file my ($header, $rows) = read_request_file($req_file); - die "failed to read request_file $req_file" if !$header or !$rows; + if (!$header or !$rows) { + # Since a request got queued, the request file must have been readable at some + # point + print STDERR "failed to read request_file $req_file" + stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); + exit $PS_EXIT_CONFIG_ERROR; + } # at this point we need to find out what kind of request type it is and @@ -144,7 +156,13 @@ my $rownum = $job->{rownum}; my $fault = $job->{fault}; - - my $row_info = get_request_info($rows, $rownum); - # rownum fault img_name job_id + my $exp_id = $job->{exp_id}; + + # XXX: get the image_db from a table in the database, or maybe save it in the job + my $image_db = "ps_simtest"; + my $exp_info = get_exposure_info($image_db, $exp_id); + + # unless $exp_id is null (e.g. stack images) get the metadata for this exposure + my ($row, $req_info) = get_request_info($rows, $rownum) if $exp_id; + if (($job_type eq "stamp") || ($job_type eq "get_image")) { my $jreglist = "$out_dir/reglist$job_id"; @@ -158,18 +176,15 @@ my ($img_name, undef) = split '\|', $line; - # XXX TODO: need to get these from somewhere. Maybe the stamp. - # - # ra_deg and dec_deg are the coordinates of center of the stamp - # the rest of the metadata come from the original FPA. - # we could get these out of the rawExp table in the database.... - # - # Much of those data do not apply to stamps from Stack images. - my ($ra_deg, $dec_deg, $mjd_obs, $ra_obs, $dec_obs, $filter, $exp_time, $fpa_id) = - (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ); - # add line to the table definition file print $tdf "$rownum|$fault|$img_name|$job_id|"; - print $tdf "$ra_deg|$dec_deg|$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id|"; - print $tdf "$row_info"; + + # ra_deg and dec_deg are the coordinates of center of the stamp + # + # XXX we need to get the center coordinate of the stamp. + # if request used -skycenter then we have it in the request info + + print $tdf "0.0|0.0|"; + print $tdf "$exp_info|"; + print $tdf "$req_info|"; print $tdf "\n"; } @@ -178,8 +193,11 @@ print STDERR "no reglist file for job $job_id\n" if $verbose; print $tdf "$rownum|$fault|0|$job_id|"; - print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; + print $tdf "0|0|"; # center of (non-existent) stamp + print $tdf "$exp_info|"; + print $tdf "$req_info|"; } } else { # XXX do list jobs + # we can probably arange things to use the code as above and skip the fileset registration print STDERR "Unknown jobType: $job_type"; next; @@ -226,4 +244,5 @@ } } + sub stop_request { my $req_id = shift; @@ -252,6 +271,78 @@ $rowinfo .= "$row->{ID}|$row->{CLASS_ID}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|"; $rowinfo .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|"; - $rowinfo .= "$row->{WIDTH}|$row->{HEIGHT}|"; - - return $rowinfo; -} + $rowinfo .= "$row->{WIDTH}|$row->{HEIGHT}"; + + return ($row, $rowinfo); +} + +sub get_exposure_info { + my $image_db= shift; + my $exp_id = shift; + + if (!$exp_id) { + # no exposure id just return zeros + # XXX: we could put a value in for filter, but we don't have a good place to find it + + #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + return "0|0|0|0|0|0"; + } + + my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); + } + + my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id"; + + # run the tool and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # not sure if we should die here + die "cannot get exposure information for $exp_id from image database $image_db"; + } + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + my $output = join "", @$stdout_buf; + if (!$output) { + print STDERR "no output returned from $command\n" if $verbose; + return undef; + } + my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); + + my $exposures = parse_md_list($metadata); + my $numExp = @$exposures; + + die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1; + + my $exp = $exposures->[0]; + + #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id"; + + use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0); + my $ra_deg = $exp->{ra} * RADIANS_TO_DEGREES; + my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES; + my $mjd_obs = dateobs_to_mjd($exp->{dateobs}, $exp->{exp_time}); + + my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}"; + + return $info; +} + +sub dateobs_to_mjd { + my $dateobs = shift; + my $exp_time = shift; + + # dateobs is in format: 1970-01-01T00:00:00 + + my ($date, $time) = split "T", $dateobs; + my ($year, $mon, $day) = split "-", $date; + my ($hr, $min, $sec) = split ":", $time; + + my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900); + + $ticks += $exp_time / 2.0; + + return 40587.0 + ($ticks/86400.); +} Index: /trunk/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 19264) +++ /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 19265) @@ -3,7 +3,7 @@ ### pstampparser_run.pl ### Run the request parser for a given request id +### This script should be called request_parser.pl since it handles more than postage +### stamp requests ### - -#XXX see notes about error handling use warnings; @@ -172,24 +172,39 @@ if (! -d $outProductDir); - -# run the appropriate parse command to parse the queue the jobs for this request -# first check the extension header to find the EXTNAME -my $request_type = find_request_type($uri); - -print STDERR "request_type for $req_id is $request_type\n" if $verbose; - my $parse_cmd; +my $request_type; my $reqType; # for the database -if ($request_type eq "PS1_PS_REQUEST") { - $reqType = 'pstamp'; - $parse_cmd = $pstampparse; -} elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { - $reqType = 'dquery'; - $parse_cmd = $dqueryparse; + +if (-r $uri) { + # run the appropriate parse command to parse the queue the jobs for this request + # first check the extension header to find the EXTNAME + $request_type = find_request_type($uri); + + if ($request_type) { + print STDERR "request_type for $req_id is $request_type\n" if $verbose; + if ($request_type eq "PS1_PS_REQUEST") { + $reqType = 'pstamp'; + $parse_cmd = $pstampparse; + } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { + $reqType = 'dquery'; + $parse_cmd = $dqueryparse; + } else { + print STDERR "Unknown request type $request_type found in $uri"; + } + } else { + print STDERR "No EXTNAME found keyword in $uri"; + } +} else { + if (-e $uri) { + print STDERR "Request file $uri is not readable"; + } else { + print STDERR "Request file $uri does not exist"; + } } if (!$parse_cmd) { - print STDERR "No EXTNAME found in $uri" if !$request_type; - print STDERR "Unknown request type $request_type found in $uri"; + # can't go any farther, set fault and set request state to run + # request_finish.pl will clean up, perhaps notifiying the operator of the input data store + # that they sent us a request file that we don't understand my $command = "$pstamptool -updatereq -req_id $req_id -state run"; @@ -211,5 +226,5 @@ { my $error_file_name = "$workdir/parse_error.txt"; - # get rid of any error file from previous run + # get rid of any error file from previous attempt to parse this request unlink $error_file_name if (-e $error_file_name); @@ -219,5 +234,5 @@ # save the contents of stderr (if any) to a file. This is relevant if - # the file parsed properl but one of the rows in the request file generated an error + # the file was parseable but one or more of the rows in the request file generated an error my $errbuf = join "", @$stderr_buf; if ($errbuf) { @@ -258,13 +273,17 @@ exit 0; + sub find_request_type { # find the EXTNAME in the input fits table - # TODO: do this right handling errors etc. my $file_name = shift; my $out = `echo $file_name | fields -x 0 EXTNAME`; - # output from fields is filename value - my ($dummy, $extname) = split " ", $out; - - return $extname; -} + if ($out) { + # output from fields is filename value + my ($dummy, $extname) = split " ", $out; + + return $extname; + } else { + return undef; + } +} Index: /trunk/pstamp/scripts/pstamp_queue_requests.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_queue_requests.pl (revision 19264) +++ /trunk/pstamp/scripts/pstamp_queue_requests.pl (revision 19265) @@ -22,5 +22,5 @@ 'verbose' => \$verbose, 'dbname=s' => \$dbname, - 'limit=s' => \$limit, + 'limit=i' => \$limit, ); @@ -119,6 +119,4 @@ my $out_buf = join("", @$stdout_buf); # split raw output into lines - # XXX: why do I think that i need the map? - #my @lines = map {split "\n"} $raw_output; @lines = split /^/, $out_buf; } @@ -127,11 +125,15 @@ # each line contains a fileset # - # XXX: we probably should set a limit on the number of these we process - # otherwise this could take a long time and the task could time out + + # number that we've processed + my $numFilesets = 0; foreach my $line (@lines) { # parse the line into fields split by whitespace my ($uri, $fs_name, $date, $type) = split " ", $line; + # skip comment lines next if ( $uri =~ /^#.*/); + + $numFilesets++; my @files; @@ -163,18 +165,21 @@ my $command = "$pstamptool -addreq -uri $req_uri -ds_id $ds_id"; $command .= " -dbname $dbname" if $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - # XXX: what do we do now? - die("Unable to perform $command: $error_code"); - } + + unless ($success) { + die("Unable to perform $command: $error_code"); + } } - $lastFileset = $fs_name; } + $lastFileset = $fs_name; + last if ($numFilesets >= $limit); } ## now update the last_fileset column in pstampDataStore - ## XXX: we should probably do this before we process the fileset. Otherwise - ## if we get an error from pstamptool we'll get stuck processing it over and over. + ## XXX: we should perhaps do this while processing each fileset so that if a later + ## one has an error we don't get repeats. + if ($lastFileset) { # print "last fileset: $lastFileset\n"; Index: /trunk/pstamp/scripts/pstampparse.pl =================================================================== --- /trunk/pstamp/scripts/pstampparse.pl (revision 19264) +++ /trunk/pstamp/scripts/pstampparse.pl (revision 19265) @@ -53,5 +53,4 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); -my $locateimages = can_run('locateimages') or (warn "Can't find locateimages" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);