Index: branches/bills_081204/pstamp/scripts/pstamp_finish.pl =================================================================== --- trunk/pstamp/scripts/pstamp_finish.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstamp_finish.pl (revision 20890) @@ -21,4 +21,5 @@ use PS::IPP::Config qw( :standard ); use PStamp::RequestFile qw( :standard ); +use PStamp::Job qw( :standard ); my ( $req_id, $req_name, $req_file, $product, $dbname, $verbose, $save_temps ); @@ -73,12 +74,15 @@ print STDERR "output fileset directory $out_dir does not exist\n"; - # XXX TODO: if this fails fault the request so we pstamp_finish - - mkdir $out_dir or die "cannot create output directory $out_dir"; + if (!mkdir $out_dir) { + print STDERR "cannot create output directory $out_dir"; + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); + } + } elsif (! -d $out_dir ) { # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the # request - die "output fileset directory $out_dir exists but is not a directory"; + print STDERR "output fileset directory $out_dir exists but is not a directory"; + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); } @@ -86,5 +90,4 @@ print STDERR "request file $req_file is missing\n"; stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); - exit $PS_EXIT_CONFIG_ERROR; } @@ -97,5 +100,4 @@ print STDERR "failed to read request_file $req_file" ; stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); - exit $PS_EXIT_CONFIG_ERROR; } @@ -108,4 +110,6 @@ my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); print $rlf "results.fits|||table|\n"; + + # XXX: this file is in the request's workdir not the out_dir my $err_file = "$out_dir/parse_error.txt"; if (-e $err_file ) { @@ -136,10 +140,10 @@ # including those that produced no jobs. # for now add an entry for rownum 1 and a phony error code. - # we've included parse_results.txt to the fileset if it exists + # we've included parse_error.txt to the fileset if it exists # - my $rownum = 1; + my $rownum = 0; my $fault = 42; # get a real error code print $tdf "$rownum|$fault|0|0|"; - print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; + print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; } else { my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); @@ -158,10 +162,11 @@ my $exp_id = $job->{exp_id}; - # XXX: get the image_db from a table in the database, or maybe save it in the job - my $image_db = "ps_simtest"; + my ($row, $req_info, $project) = get_request_info($rows, $rownum); + my $proj_hash = resolve_project($ipprc, $project, $dbname); + my $image_db = $proj_hash->{dbname}; + + # get the metadata for the exposure (if any i.e. stack) + # returns an appropriate string if !$exp_id my $exp_info = get_exposure_info($image_db, $exp_id); - - # unless $exp_id is null (e.g. stack images) get the metadata for this exposure - my ($row, $req_info) = get_request_info($rows, $rownum) if $exp_id; if (($job_type eq "stamp") || ($job_type eq "get_image")) { @@ -180,9 +185,7 @@ # ra_deg and dec_deg are the coordinates of center of the stamp - # - # XXX we need to get the center coordinate of the stamp. - # if request used -skycenter then we have it in the request info - + # XXX: parse the stamp header to find it print $tdf "0.0|0.0|"; + print $tdf "$exp_info|"; print $tdf "$req_info|"; @@ -231,4 +234,5 @@ print STDERR "Unable to perform $command error code: $error_code\n"; $request_fault = $error_code >> 8; + # fall through to stop the request } } @@ -258,4 +262,5 @@ die("Unable to perform $command error code: $error_code"); } + exit $fault; } @@ -273,5 +278,5 @@ $rowinfo .= "$row->{WIDTH}|$row->{HEIGHT}"; - return ($row, $rowinfo); + return ($row, $rowinfo, $row->{PROJECT}); } Index: branches/bills_081204/pstamp/scripts/pstamp_get_image_job.pl =================================================================== --- trunk/pstamp/scripts/pstamp_get_image_job.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstamp_get_image_job.pl (revision 20890) @@ -106,4 +106,5 @@ my $pathname = shift @fields; my $filetype = shift @fields; + my $class_id = shift @fields; ($pathname , my $filename) = resolvepath($pathname); @@ -121,5 +122,5 @@ # XXX is pstamp always the right file type, if not where can we get the right one? - print REGLIST file_registration_line($filename, $dest_path, $filetype); + print REGLIST file_registration_line($filename, $dest_path, $filetype, $class_id); foreach my $f (@fields) { @@ -136,4 +137,8 @@ my $pathname = $_[0]; + # use the basename of the unresolved file as the name of the file + # in order to avoid nebulous name mangled paths in the datastore + my $file = basename($pathname); + my $slash = index($pathname, "/"); if ($slash != 0) { @@ -141,8 +146,7 @@ $ipprc = PS::IPP::Config->new(); } - $pathname = $ipprc->convert_filename_absolute($pathname); + $pathname = $ipprc->file_resolve($pathname); } - my $file = basename($pathname); return ($pathname, $file); @@ -154,4 +158,7 @@ my $path = shift; my $filetype = shift; + my $chipname = shift; + $chipname = $chipname ? "$chipname|" : ""; + if (-e $path) { my @finfo = stat($path); @@ -160,5 +167,5 @@ my $md5sum = file_md5_hex($path); - return "$filename|$bytes|$md5sum|$filetype|"; + return "$filename|$bytes|$md5sum|$filetype|$chipname"; } else { die "$filename not found at $path"; Index: branches/bills_081204/pstamp/scripts/pstamp_job_run.pl =================================================================== --- trunk/pstamp/scripts/pstamp_job_run.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstamp_job_run.pl (revision 20890) @@ -13,11 +13,5 @@ use File::Basename; use Digest::MD5::File qw( file_md5_hex ); - -# values for OPTION_MASK -# XXX put these in a module -use constant PSTAMP_SELECT_IMAGE => 1; -use constant PSTAMP_SELECT_MASK => 2; -use constant PSTAMP_SELECT_WEIGHT => 4; - +use PStamp::RequestFile qw( :standard ); my $verbose; @@ -124,11 +118,24 @@ open F, ">$reglist" or die "can't open $reglist for output"; - # we are assuming an interface with ppstamp here. + # figure out what output images we should expect + + # Note: we are assuming the contents of the filerules here. my @extensions = ( "fits", "mk.fits", "wt.fits"); - # XXX TODO: add mask of expected file types to pstampJob so we know what to expect - - my $output_mask = PSTAMP_SELECT_IMAGE; - my $m = 1; # XXX we're getting a bit intimate with the bit field definitions here. do better use a hash + # we always require an image + my $output_mask = $PSTAMP_SELECT_IMAGE; + + # we search the argString for -mask and -weight. + # searching the arg string allows us to avoid adding a column in pstampJob + # XXX yeah but does this make us susceptible to user input fouling us up? + # For example, what if somebody sets the output base to file-mask-weight + # TODO: clean this up + $output_mask |= $PSTAMP_SELECT_MASK if ($argString =~ /-mask/); + $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/); + + # XXX we're getting a bit intimate with the bit field definitions here. + # do better use a hash + + my $m = 1; foreach my $extension (@extensions) { my $do_this_one = $m & $output_mask; @@ -141,5 +148,5 @@ my $path = "${outputBase}.${extension}"; - # XXX is pstamp always the right file type, if not where can we get the right one? + # XXX is pstamp always the right file type, if not where do we get the right one? print F file_registration_line($filename, $path, "pstamp") . "\n"; } @@ -182,5 +189,7 @@ exit $jobStatus; +# create a string to be passed as input to dsreg when registering this file in a fileset # XXX move this to a module so it can be shared + sub file_registration_line { my $filename = shift; Index: branches/bills_081204/pstamp/scripts/pstamp_results_file.pl =================================================================== --- trunk/pstamp/scripts/pstamp_results_file.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstamp_results_file.pl (revision 20890) @@ -91,5 +91,4 @@ # output parameters -# { name => 'STAMP_NAME', type => '16A', writetype => TSTRING }, { name => 'OPTION_MASK',type => 'J', writetype => TULONG }, Index: branches/bills_081204/pstamp/scripts/pstamp_revert_request.pl =================================================================== --- trunk/pstamp/scripts/pstamp_revert_request.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstamp_revert_request.pl (revision 20890) @@ -41,4 +41,6 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage ( -msg => "--req_id is required", -exitval => 2 ) if !$req_id; my $missing_tools; Index: branches/bills_081204/pstamp/scripts/pstampparse.pl =================================================================== --- trunk/pstamp/scripts/pstampparse.pl (revision 19274) +++ branches/bills_081204/pstamp/scripts/pstampparse.pl (revision 20890) @@ -123,4 +123,5 @@ } +my $num_jobs = 0; foreach my $row (@$rows) { my $rownum = $row->{ROWNUM}; @@ -131,5 +132,4 @@ my $id = $row->{ID}; my $class_id = $row->{CLASS_ID}; - my $stamp_name = $row->{STAMP_NAME}; my $filter = $row->{REQFILT}; my $mjd_min = $row->{MJD_MIN}; @@ -142,5 +142,5 @@ my $option_mask= $row->{OPTION_MASK}; - $stamp_name = "" if $stamp_name eq "null"; + $class_id = "" if ($class_id eq "null" or $class_id eq "all"); # XXX: TODO: sanity check all parameters @@ -151,5 +151,11 @@ # trash the request. die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri"); - my ($image_db, $lookup_class_id, $camera, $magic_required ) = resolve_project($project, $img_type, $class_id); + + my $proj_hash = resolve_project($ipprc, $project, $dbname); + die "project $project not found\n" unless $proj_hash; + + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + my $need_magic = $proj_hash->{need_magic}; my $roi_string; @@ -182,7 +188,11 @@ # Call PStamp::Job's locate_images routinte to get the parameters for this request specification - my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $lookup_class_id, + my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id, $x, $y, $mjd_min, $mjd_max, $filter); + if (!$images) { + print STDERR "no matching images found for row $rownum\n"; + next; + } if ($mode eq "list_uri") { foreach my $image (@$images) { @@ -194,11 +204,23 @@ my $listfile = "$out_dir/filelist"; + + # map our img_type to the Data Store file types. + my %filelist_img_types = ( "raw" => "chip", + "chip" => "chipproc", + "warp" => "warp", + "stack"=>"stack", + "diff" => "diff"); + + my $filelist_img_type = $filelist_img_types{$img_type}; + open LISTFILE, ">$listfile" or die "failed to open file list: $listfile while parsing get_image request $req_id"; foreach my $image (@$images) { - print LISTFILE "$image->{image}|$img_type\n"; + my $class_id = $image->{class_id} ? $image->{class_id} : ""; + print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n"; } close LISTFILE; + $num_jobs++; my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image" . " -uri $listfile -outputBase $out_dir -rownum $rownum"; @@ -217,21 +239,19 @@ } else { my $args = $roi_string ? $roi_string : ""; - $args .= " -class_id $class_id" if ($class_id and ($class_id ne "null")); + $args .= " -class_id $class_id" if $class_id; # sequence number for the the job for a request spec. Not to be confused with job_id my $job_num = 0; - # XXX for raw and chip level images and class_id "null" if there are multiple images + # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images # use -list instead of -file foreach my $image (@$images) { my $uri = $image->{image}; my $exp_id = $image->{exp_id}; + $job_num++; my $output_base = "$out_dir/${rownum}_${job_num}"; - - # By commenting out the next line, I hereby drop the use of STAMP_NAME - # $output_base .= "_${stamp_name}" if $stamp_name; # add astrometry file for raw and chip images if one is available @@ -240,13 +260,13 @@ } - if ($option_mask & $PSTAMP_SELECT_MASK) { + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { $args .= " -mask $image->{mask}"; } - if ($option_mask & $PSTAMP_SELECT_WEIGHT) { + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { $args .= " -weight $image->{weight}"; } # XXX sounds like magic will be handled outside of the postage stamp server - #if ($magic_required) { + #if ($need_magic) { # die "no magic mask available" if ! $image->{magic_mask}; # $args .= " -magic_mask $image->{magic_mask}" @@ -258,4 +278,5 @@ my $state = "run"; + $num_jobs++; my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" @@ -284,55 +305,13 @@ } -exit 0; - - -# resolve_project() -# get project specific information -# This is a placeholder final implementation will use configuration file and database to -# map from "project" to image database, dvo database, telescope, camera, etc -# -# For now just get the database and the class_id that will be used for image lookup - -sub resolve_project { - my $project_name = shift; - my $img_type = shift; - my $class_id = shift; - die "project is not defined" if !$project_name; - - # for megacam and simtest since images are at the fpa level, leave out the - # class_id at the lookup stage - # - # TODO: get this information from a database or a configuration file - - my $ret_dbname; - my $ret_class_id; - my $ret_inst; - my $ret_magic; - my $ret_camera; - - if ($project_name eq "megacam-mops") { - $ret_dbname = "mops"; - $ret_class_id = "null"; - $ret_camera = "MEGACAM"; - $ret_magic = 0; - } elsif ($project_name eq "simtest") { - $ret_dbname = "ps_simtest"; - $ret_class_id = "null"; - $ret_camera = "SIMTEST"; - $ret_magic = 0; - } elsif ($project_name eq "gpc1") { - $ret_dbname = "gpc1"; - $ret_class_id = $class_id; - $ret_camera = "GPC1"; - $ret_magic = 0; # switch this to 1 when we're ready +if ($mode eq "queue_jobs") { + if ($num_jobs) { + exit 0; } else { - die "unknown project $project_name"; - } - - if (($img_type ne "raw") and ($img_type ne "chip")) { - # don't need class_id - $ret_class_id = ""; - } - - return ($ret_dbname, $ret_class_id, $ret_camera, $ret_magic); -} + # no jobs queued return error to get the request stopped + # TODO: need to define meaningful error codes + exit $PS_EXIT_UNKNOWN_ERROR; + } +} else { + exit 0; +}