Index: /trunk/pstamp/scripts/pstamp_finish.pl
===================================================================
--- /trunk/pstamp/scripts/pstamp_finish.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstamp_finish.pl	(revision 21410)
@@ -14,4 +14,5 @@
 use File::Temp qw( tempfile );
 use File::Copy;
+use File::Basename qw(dirname);
 
 use PS::IPP::Metadata::Config;
@@ -23,22 +24,31 @@
 use PStamp::Job qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $product, $dbname, $verbose, $save_temps );
+my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output);
 
 # the char to the right of the bar may be used as a single - alias for the longer name
 # for example --input and -i are equivalent
 GetOptions(
-           'req_id=s'   => \$req_id,
-           'req_name=s' => \$req_name,
-           'req_file=s' => \$req_file,
-           'product=s'  => \$product,
-	   'dbname=s'   => \$dbname,
-	   'verbose'    => \$verbose,
-	   'save-temps' => \$save_temps,
+           'req_id=s'       => \$req_id,
+           'req_name=s'     => \$req_name,
+           'req_file=s'     => \$req_file,
+           'product=s'      => \$product,
+           'out_dir=s'      => \$out_dir,
+	   'dbname=s'       => \$dbname,
+	   'verbose'        => \$verbose,
+	   'save-temps'     => \$save_temps,
+	   'redirect-output' => \$redirect_output,
 ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 
-die "usage: --req_id id --req_name name --req_file file --product product [--dbname dbname --verbose]\n"
-    if !$req_id or !$req_name or !$req_file or !$product;
+die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n"
+    if !$req_id or !$req_name or !$req_file or !$product or !$out_dir;
+
+my $ipprc = PS::IPP::Config->new(); # IPP Configuration
+
+if ($redirect_output) {
+    my $logDest = "$out_dir/psfinish.$req_id.log";
+    $ipprc->redirect_output($logDest);
+}
 
 my $missing_tools;
@@ -53,5 +63,4 @@
 }
 
-my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 
 my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
@@ -64,7 +73,11 @@
     my $fileset = $req_name;
 
+
     # Here we invoke the assumption that the output for the request is placed in the
     # fileset directory directly
-    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
+#    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
+
+    # now we are assuming that the output directory is the dirname of the request file
+    # XXX: put this in the database
 
     print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
@@ -72,5 +85,5 @@
     if (!-e $out_dir) {
         # something must have gone wrong parsing the request
-        print STDERR  "output fileset directory $out_dir does not exist\n";
+        print STDERR  "output directory $out_dir does not exist\n";
 
         if (!mkdir $out_dir) {
@@ -83,5 +96,5 @@
         # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
         # request
-        print STDERR "output fileset directory $out_dir exists but is not a directory";
+        print STDERR "output directory $out_dir exists but is not a directory";
         stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname);
     }
@@ -111,7 +124,6 @@
     print $rlf "results.fits|||table|\n";
 
-    # XXX: this file is in the request's workdir not the out_dir
-    my $err_file = "$out_dir/parse_error.txt";
-    if (-e $err_file ) {
+    my $err_file = "parse_error.txt";
+    if (-e "$out_dir/$err_file" ) {
         print $rlf "$err_file|||text|\n";
     }
@@ -222,8 +234,6 @@
     if (!$request_fault) {
         # register the fileset
-        # Note that we are assuming that the fileset's files are already in the fileset directory.
-        # We could put them somewhere else and copy them in but for now we set the output directory
-        # to the fileset directory at parse time
         my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
+        $command .= " --link --datapath $out_dir";
         $command .= " --dbname $dbname" if $dbname;
 
Index: /trunk/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /trunk/pstamp/scripts/pstamp_job_run.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstamp_job_run.pl	(revision 21410)
@@ -18,4 +18,6 @@
 my $dbname;
 my $job_id;
+my $redirect_output;
+my $output_base;
 
 my $host = hostname();
@@ -23,7 +25,9 @@
 
 GetOptions(
-    'job_id=s'  =>  \$job_id,
-    'dbname=s'  =>  \$dbname,
-    'verbose'   =>  \$verbose,
+    'job_id=s'      =>  \$job_id,
+    'output_base=s' =>  \$output_base,
+    'redirect-output' => \$redirect_output,
+    'dbname=s'      =>  \$dbname,
+    'verbose'       =>  \$verbose,
 );
 
@@ -35,4 +39,5 @@
 
 die "job_id is required" if !$job_id;
+die "output_base is required" if !$output_base;
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -53,4 +58,10 @@
 		       caturi
 		       );
+my $ipprc = PS::IPP::Config->new(); # IPP Configuration
+
+if ($redirect_output) {
+    my $logDest = "$output_base.log";
+    $ipprc->redirect_output($logDest);
+}
 
 my $missing_tools;
Index: /trunk/pstamp/scripts/pstamp_listjobs.pl
===================================================================
--- /trunk/pstamp/scripts/pstamp_listjobs.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstamp_listjobs.pl	(revision 21410)
@@ -101,5 +101,5 @@
     my $add_fits = 0;
     if ($job->{jobType} ne "get_image") {
-        $add_fits = 1;
+#        $add_fits = 1;
     }
 
@@ -115,5 +115,5 @@
     }
 
-    print "$job->{job_id} $job->{state} $uri\n";
+    print "$job->{job_id} $job->{state} $job->{fault} $job->{name} $job->{outProduct} $uri\n";
 }
 
Index: /trunk/pstamp/scripts/pstamp_parser_run.pl
===================================================================
--- /trunk/pstamp/scripts/pstamp_parser_run.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstamp_parser_run.pl	(revision 21410)
@@ -12,13 +12,20 @@
 use Sys::Hostname;
 use Getopt::Long qw( GetOptions );
-
+use File::Basename qw( basename dirname);
+
+my $req_id;
+my $uri;
+my $redirect_output;
+my $product;
 my $verbose;
 my $dbname;
-my $req_id;
 
 GetOptions(
-    'verbose'   =>  \$verbose,
-    'dbname=s'  =>  \$dbname,
-    'req_id=s'  =>  \$req_id,
+    'req_id=s'          =>  \$req_id,
+    'uri=s'             =>  \$uri,
+    'product=s'         =>  \$product,
+    'redirect-output'   =>  \$redirect_output,
+    'verbose'           =>  \$verbose,
+    'dbname=s'          =>  \$dbname,
 );
 
@@ -29,5 +36,8 @@
 }
 
-die "req_id is required" if !defined($req_id);
+die "--req_id --uri --product are required" 
+    if !defined($req_id) or
+       !defined($uri) or
+       !defined($product);
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -66,4 +76,9 @@
 }
 
+if ($redirect_output) {
+    my $logDest = "$workdir/psparse.$req_id.log";
+    $ipprc->redirect_output($logDest);
+}
+
 my $defaultDSProduct = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_PRODUCT');
 exit ($PS_EXIT_CONFIG_ERROR) unless defined $defaultDSProduct;
@@ -83,9 +98,10 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $psrequest;
-#Look up the uri for the given request
-{
-    my $command = "$pstamptool -pendingreq -req_id $req_id";
-    $command .= " -dbname $dbname" if $dbname;
+
+my $fn = basename($uri);
+my $new_uri = "$workdir/$fn";
+if ($uri =~ /^http:/) {
+    # if the uri is an http uri download the file 
+    my $command = "$dsget --uri $uri --filename $new_uri";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -93,82 +109,21 @@
         die("Unable to perform $command error code: $error_code");
     }
-
-    if (@$stdout_buf == 0) {
-        print STDERR "pending pstamp request id $req_id not found\n";
-        exit 1;
-    }
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
-
-    my $requests = parse_md_list($metadata);
-
-    my $num = @$requests;
-    die "unexpected number of requests $num found" if $num > 1;
-
-    $psrequest = $requests->[0];
-}
-
-if (!$psrequest) {
-    print STDERR "pending postage stamp request $req_id not found\n";
-    exit 1;
-}
-
-my $uri = $psrequest->{uri};
-
-# if the uri is from a data store, download it 
-
-my $ds_id = $psrequest->{ds_id};
-
-my $new_uri;
-if ($ds_id && ($uri =~ /^http:/)) {
-    {
-        $new_uri = "$workdir/request.fits";
-        my $command = "$dsget --uri $uri --filename $new_uri";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            die("Unable to perform $command error code: $error_code");
-        }
-
-        $uri = $new_uri;
-    }
-}
-
-#  find the name of the output directory
-my $product;
-if ($ds_id) {
-    my $command = "$pstamptool -datastore -ds_id $ds_id";
-    $command .= " -dbname $dbname" if $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        die("Unable to perform $command error code: $error_code");
-    }
-
-    if (@$stdout_buf == 0) {
-        print STDERR "data store with data store id $ds_id not found\n";
-        exit 1;
-    }
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
-
-    my $dataStores = parse_md_list($metadata);
-
-    my $dataStore = $dataStores->[0];
-
-    $product = $dataStore->{outProduct};
-} else {
+} elsif ($uri ne $new_uri) {
+    # put a link to the file into the workdir
+    if (-e $new_uri) {
+        unlink $new_uri or die "failed to unlink $new_uri";
+    }
+    if (! symlink $uri, $new_uri) {
+        die ("failed to link request file $uri to workdir $workdir");
+    }
+}
+$uri = $new_uri;
+
+die "request file $uri not found" if ! -e $uri;
+
+#  if product was not defined (in database), use the default
+if ($product eq "NULL") {
     $product = $defaultDSProduct;
 }
-
-#
-# Here is one of the places that we assume that the output for a request goes directly into
-# the output fileset directory in the data store.
-#
-my $outProductDir = "$outputDataStoreRoot/$product";
-
-die "output product directory $outProductDir does not exist" if (! -e $outProductDir) ;
-die "file with name of output product directory $outProductDir exists but is not a directory" 
-    if (! -d $outProductDir);
 
 my $parse_cmd;
@@ -220,6 +175,7 @@
 }
 
-$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $outProductDir --file $uri";
+$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri";
 $parse_cmd .= " --dbname $dbname" if $dbname;
+$parse_cmd .= " --verbose" if $verbose;
 
 my $newState = "run";
Index: /trunk/pstamp/scripts/pstamp_queue_requests.pl
===================================================================
--- /trunk/pstamp/scripts/pstamp_queue_requests.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstamp_queue_requests.pl	(revision 21410)
@@ -107,7 +107,17 @@
         my $command = "$dsproductls --uri $ds->{uri}/index.txt";
         $command .= " --last_fileset $ds->{lastFileset}" if $ds->{lastFileset};
+
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         unless ($success) {
+            # dsproductls exit status is the http error code - 300
+            my $exit_status = $error_code >> 8;
+
+            # don't die on "common faults"
+            # perhaps after some number of these we should mark the data store as disabled
+            next if ($exit_status == 200);  # 500 cannot connect (data store down)
+            next if ($exit_status == 104);  # 404 not found
+#            next if ($exit_status == 100);  # 400 bad request
+
             die("Unable to perform $command: $error_code");
         }
@@ -143,4 +153,6 @@
                     run(command => $command, verbose => $verbose);
             unless ($success) {
+                # we don't want to die here. We need to set lastFileset
+                # so that we don't re queue requsts that we have proccessed
                 die("Unable to perform $command: $error_code");
             }
@@ -175,14 +187,7 @@
         }
         $lastFileset = $fs_name;
-        last if ($numFilesets >= $limit);
-    }
-
-    ## now update the last_fileset column in pstampDataStore
-    ## XXX: we should perhaps do this while processing each fileset so that if a later
-    ## one has an error we don't get repeats.
-
-    if ($lastFileset) {
-        # print "last fileset: $lastFileset\n";
+
         {
+        ## now update the last_fileset column in pstampDataStore
         my $command = "$pstamptool -ds_id $ds_id -moddatastore -last_fileset $lastFileset";
         $command .= " -dbname $dbname" if $dbname;
@@ -193,4 +198,5 @@
             }
         }
+        last if ($numFilesets >= $limit);
     }
 }
Index: /trunk/pstamp/scripts/pstampparse.pl
===================================================================
--- /trunk/pstamp/scripts/pstampparse.pl	(revision 21409)
+++ /trunk/pstamp/scripts/pstampparse.pl	(revision 21410)
@@ -84,13 +84,4 @@
 die "REQ_NAME not found in $request_file_name"  if (!$req_name);
 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
-
-if ($out_dir) {
-    $out_dir .= "/$req_name";
-    if (! -e $out_dir ) {
-        mkdir $out_dir or die "cannot create output directory $out_dir";
-    } elsif (! -d $out_dir ) {
-        die "output fileset directory $out_dir exists but is not a directory";
-    }
-}
 
 if ($req_id) {
@@ -187,7 +178,7 @@
     }
 
-    # Call PStamp::Job's locate_images routinte to get the parameters for this request specification
+    # Call PStamp::Job's locate_images routine to get the parameters for this request specification
     my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
-            $x, $y, $mjd_min, $mjd_max, $filter);
+            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
 
     if (!$images) {
Index: /trunk/pstamp/scripts/request_finish.pl
===================================================================
--- /trunk/pstamp/scripts/request_finish.pl	(revision 21409)
+++ /trunk/pstamp/scripts/request_finish.pl	(revision 21410)
@@ -13,4 +13,5 @@
 use File::Temp qw( tempfile );
 use File::Copy;
+use File::Basename qw( dirname );
 
 use PS::IPP::Metadata::Config;
@@ -20,5 +21,5 @@
 use PS::IPP::Config qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $req_type, $product, $dbname, $verbose, $save_temps );
+my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output );
 
 GetOptions(
@@ -27,8 +28,10 @@
            'req_file=s' => \$req_file,
            'req_type=s' => \$req_type,
+           'out_dir=s'  => \$out_dir,
            'product=s'  => \$product,
 	   'dbname=s'   => \$dbname,
 	   'verbose'    => \$verbose,
 	   'save-temps' => \$save_temps,
+           'redirect-output' => \$redirect_output,
 ) or pod2usage( 2 );
 
@@ -42,6 +45,18 @@
 $err .= "--req_name is required\n" if !$req_name;
 $err .= "--product is required\n" if !$product;
+# $err .= "--out_dir is required\n" if !$out_dir;
 
 die "$err" if $err;
+
+
+if (!$out_dir) {
+    $out_dir = dirname($req_file);
+}
+
+if ($redirect_output) {
+    my $logDest = "$out_dir/reqfinish.$req_id.log";
+    my $ipprc = PS::IPP::Config->new();
+    $ipprc->redirect_output($logDest);
+}
 
 my $missing_tools;
@@ -63,5 +78,5 @@
 }
 if ($finish_cmd) {
-    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product";
+    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir";
     $command   .= " --dbname $dbname" if $dbname;
     $command   .= " --verbose" if $verbose;