Index: /tags/ipp-20111222/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm
===================================================================
--- /tags/ipp-20111222/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 33173)
+++ /tags/ipp-20111222/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 33174)
@@ -487,10 +487,10 @@
     my $command = "$difftool -dbname $imagedb -pstamp_order";
     
-    my $listrun = 0;
+    my $choose_components = 0;
     if ($byid) {
         if ($skycell_id and ($skycell_id ne 'all')) {
             $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id";
         } else {
-            $listrun = 1;
+            $choose_components = 1;
             $command .= " -listrun -diff_id $id";
             # the following is a work around for the problem reported in ticket #1394
@@ -512,5 +512,5 @@
 
     my $images;
-    if ($listrun) {
+    if ($choose_components) {
         $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose);
     } else {
@@ -603,5 +603,5 @@
             my @imageList = ($image);
 
-            setRowRefs($rowList, \@imageList);
+            setRowRefs($rowList, \@imageList) unless $choose_components;
             # the $image is going to be returned directly in this case so we need to duplicate
             # some of processing that lookup does for other img_types
Index: /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl
===================================================================
--- /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl	(revision 33173)
+++ /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl	(revision 33174)
@@ -41,5 +41,5 @@
 my $EXTVER = 1.0;
 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
-my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
 GetOptions(
@@ -47,4 +47,5 @@
     'output=s'        =>      \$output,
     'workdir=s'       =>      \$workdir,
+    'job_id=s'        =>      \$job_id,
     'dbname=s'        =>      \$dbname,
     'dbserver=s'      =>      \$dbserver,
@@ -298,5 +299,5 @@
 			$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
 		    }
-		    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
+		    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
 			$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
 		    }
@@ -324,41 +325,47 @@
 my %processing_request;
 
-open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
-foreach my $fpa_id (keys %query) {
-    for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
-	print WISDOM "$fpa_id\t";
-	foreach my $key (keys %{ $query{$fpa_id} }) {
-	    print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
-	}
-	print WISDOM "\n";
-	if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
-	    @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
-		($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
-		 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
-	    push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
-	}
-    }
-}
-close(WISDOM);
-
-# If there is anything that needs to be updated, create the update request list, and then exit the program.
-my $exit_code = 0;
-my $update_request_file = "${workdir}/update_request.dat";
-open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
-foreach my $images (keys %update_request) {
-    foreach my $fault (keys %{ $update_request{$images} }) {
-	if ($fault == 25) {
-	    $exit_code = 25;
-	}
-	elsif ($fault != 0) {
-	    $exit_code = 21;
-	}
-	my $update_request = join ' ', @{ $update_request{$images}{$fault} };
-	print UPDATE_REQUEST "$update_request\n";
-    }
-}
-close(UPDATE_REQUEST);
-if ($exit_code != 0) {
-    exit($exit_code);
+if (!$job_id) {
+    open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
+    foreach my $fpa_id (keys %query) {
+        for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
+            print WISDOM "$fpa_id\t";
+            foreach my $key (keys %{ $query{$fpa_id} }) {
+                print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
+            }
+            print WISDOM "\n";
+            my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
+            if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
+                if ($data_state ne 'full') {
+                    
+                    @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
+                        ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
+                         $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+                }
+                push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+            }
+        }
+    }
+    close(WISDOM);
+
+    # If there is anything that needs to be updated, create the update request list, and then exit the program.
+    my $exit_code = 0;
+    my $update_request_file = "${workdir}/update_request.dat";
+    open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
+    foreach my $images (keys %update_request) {
+        foreach my $fault (keys %{ $update_request{$images} }) {
+            if ($fault == 25) {
+                $exit_code = 25;
+            }
+            elsif ($fault != 0) {
+                $exit_code = 21;
+            }
+            my $update_request = join ' ', @{ $update_request{$images}{$fault} };
+            print UPDATE_REQUEST "$update_request\n";
+        }
+    }
+    close(UPDATE_REQUEST);
+    if ($exit_code != 0) {
+        exit($exit_code);
+    }
 }
 
@@ -710,4 +717,7 @@
     $status = 0;
 #    print "$output\n";
+    if (-e $output) {
+        unlink $output or die "failed to unlink existing response file $output\n";
+    }
     my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
     check_fitsio( $status );
Index: /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl
===================================================================
--- /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl	(revision 33173)
+++ /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl	(revision 33174)
@@ -66,5 +66,5 @@
 if ($product eq "NULL") {
     # parsing failed just with fault = 0 (this leaves previously set fault in place
-    stop_request($req_id, 0, $verbose);
+    update_request($req_id, 0, $verbose);
     exit 0;
 }
@@ -73,5 +73,5 @@
 if (! -e $prod_dir ) {
     # something must have gone wrong at the parse stage
-    stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     die "product directory does not exist $prod_dir";
 }
@@ -82,9 +82,9 @@
     print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
     if (! mkdir $outdir ) {
-        stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+        update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
         die "cannot create output directory $outdir";
     }
 } elsif (! -d $outdir) {
-    stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     die "output fileset directory $outdir exists but is not a directory";
 }
@@ -121,6 +121,7 @@
 
 foreach my $job (@jobs) {
+    next if $job->{parent_id};
     my $job_id = $job->{job_id};
-    my $response_file = "response${job_id}.fits";
+    my $response_file = "response.${job_id}.fits";
     my $response_path = "$outdir/$response_file";
 
@@ -157,15 +158,19 @@
 }
 
-stop_request($req_id, $request_fault, $verbose);
+update_request($req_id, $request_fault, $verbose);
 
 exit 0;
 
-sub stop_request {
+sub update_request {
     my $req_id = shift;
     my $fault = shift;
     my $verbose = shift;
     
-    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
-    $command   .= " -set_fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id";
+    if ($fault) {
+        $command .= " -set_fault $fault";
+    } else {
+        $command .= " -set_state stop";
+    }
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
Index: /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl
===================================================================
--- /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl	(revision 33173)
+++ /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl	(revision 33174)
@@ -115,7 +115,12 @@
 my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
-my $data_to_update = '';
+# my $data_to_update = '';
 {
     my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
+    if ($job_id) {
+        $command .= " --job_id $job_id";
+    } else {
+        $command .= " --ignore-wisdom";
+    }
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -132,4 +137,5 @@
 	    my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
 	}
+        warn("Some inputs are not available and must be updated.");
     }	
     
@@ -137,74 +143,113 @@
 
 my $result;
-# If we returned correctly with a valid response file, get a job ID
-# for the completed work, and move the response to a standardized name.
-if ($fault == 0) {
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-    $command .= " -job_type detect_query -state stop -fault 0";
-    $command .= " -rownum 1";
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    if ($success) {
-        $job_id = join "", @$stdout_buf;
-        chomp $job_id;
-        if ($job_id && -e $response_file) {
-            rename $response_file, "$outdir/response${job_id}.fits";
+unless ($job_id) {
+    # We are running as a parse job
+    # If we returned correctly with a valid response file, get a job ID
+    # for the completed work, and move the response to a standardized name.
+    if ($fault == 0) {
+        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
+        $command .= " -job_type detect_query -state stop -fault 0";
+        $command .= " -rownum 1"; 
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        if ($success) {
+            $job_id = join "", @$stdout_buf;
+            chomp $job_id;
+            if ($job_id && -e $response_file) {
+                rename $response_file, "$outdir/response.${job_id}.fits";
+            }
+            $result = 0;
+        } else {
+            warn("Unable to perform $command error code: $error_code");
+            $result = $error_code >> 8;
         }
-        $result = 0;
-    } else {
-        warn("Unable to perform $command error code: $error_code");
-        $result = $error_code >> 8;
-    }
-}
-elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
-    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
-    # Get the dependency id for the data we're requesting be updated.
-    open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
-    while (<UPDATE_REQUEST>) {
-	my $data_to_update = $_;
-	chomp($data_to_update);
-	my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
-
-	# Link this request to a job and link that job to any dependency
-	my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-	$command .= " -job_type detect_query -state run -fault 0";
-	$command .= " -rownum 1";
-	$command .= " -dep_id $dep_id" if $dep_id;
-
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run(command => $command, verbose => $verbose);
-
-	if ($success) {
-	    $job_id = join "", @$stdout_buf;
-	    chomp $job_id;
-	    if ($job_id && -e $response_file) {
-		# We shouldn't have a response file at this stage.
-		rename $response_file, "$outdir/response${job_id}.fits";
-	    }
-	    $result = 0;
-	} else {
-	    warn("Unable to perform $command error code: $error_code");
-	    $result = $error_code >> 8;
-	}
-    }
-    close(UPDATE_REQUEST);
-}
-else {
-    my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR);
-}
-
-# This does not set the request state to stop.  That will happen with the request_finish.pl script, 
-# which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
-{
-    my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
-    $command .= " -set_fault $result" if $result;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    }
+    elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
+        # Failed to run correctly, which means that we need to queue a job and flag data for updating.
+        # first create a parent job for the actual detectabilty query
+        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/1_"; 
+        $command .= " -job_type detect_query -state run -fault 0 -is_parent";
+        $command .= " -rownum 1";
+
+        my $parent_job_id;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        if ($success) {
+            # pstamptool returns the job_id of the new job
+            $parent_job_id = join "", @$stdout_buf;
+            chomp $parent_job_id;
+            if ($parent_job_id) {
+                $result = 0;
+            } else {
+                print STDERR "pstamptool did not return a valid job_id for parent job\n";
+                $result = $PS_EXIT_UNKNOWN_ERROR;
+            }
+        } else {
+            warn("Unable to perform $command error code: $error_code");
+            $result = $error_code >> 8;
+        }
+        # now create child jobs for each dependent. All these jobs do is finish once the dependency
+        # is satisfied
+        if (!$result) {
+            # Get the dependency id for the data we need to have updated.
+            my $job_num = 2;
+            open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
+            while (<UPDATE_REQUEST>) {
+                my $data_to_update = $_;
+                chomp($data_to_update);
+                my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
+
+                # Link this request to a job and link that job to the dependent
+                my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/${job_num}_"; 
+                $command .= " -job_type child -state run -fault 0 -parent_id $parent_job_id";
+                $command .= " -rownum 1"; # XXX: we should choose a correct rownum
+                $command .= " -dep_id $dep_id" if $dep_id;
+
+                $job_num++;
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+
+                if ($success) {
+                    $job_id = join "", @$stdout_buf;
+                    chomp $job_id;
+                    $result = 0;
+                } else {
+                    warn("Unable to perform $command error code: $error_code");
+                    $result = $error_code >> 8;
+                    last;
+                }
+            }
+            close(UPDATE_REQUEST);
+        }
+    }
+    else {
+        my_die ("Parse fault!!", $fault);
+    }
+    # This does not set the request state to stop.  That will happen with the request_finish.pl script, 
+    # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
+    {
+        my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
+        $command .= " -set_fault $result" if $result;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+} else {
+    # We are running as a job presumably because an input needed
+    # to be regenerated
+    if ($fault) {
+        # in some cases we will need to finish off the request
+        my_die ("Run fault!!", $fault);
+    }
+    if (-e $response_file) {
+        rename $response_file, "$outdir/response.${job_id}.fits";
+    }
+}
+
  
 exit 0;
Index: /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl	(revision 33173)
+++ /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl	(revision 33174)
@@ -90,8 +90,8 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $params = read_params_file($outputBase);
 
 my $jobStatus;
 if ($jobType eq "stamp") {
+    my $params = read_params_file($outputBase);
 
     my $argString;
@@ -283,4 +283,6 @@
 
     my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+
+    my $params = read_params_file($outputBase);
     my $imagedb = $params->{imagedb};
 
@@ -304,5 +306,6 @@
     # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
     # run it again the same way.
-    my $argslist = "$outputBase/parse.args";
+    my $outdir = dirname($outputBase);
+    my $argslist = "$outdir/parse.args";
     open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     my $argString = <ARGSLIST>;
@@ -317,4 +320,5 @@
     $command .= " --dbserver $dbserver" if $dbserver;
     $command .= " --verbose" if $verbose;
+    $command .= " --save-temps" if $save_temps;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
@@ -327,4 +331,7 @@
 	my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
     }
+} elsif ($jobType eq "child") {
+    # the only thing jobs of jobType child is to finish
+    $jobStatus = 0;
 } else {
     my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
@@ -611,4 +618,8 @@
 
     $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
+    if ($exit_code > 100) {
+        carp ("invalid exit code: $exit_code changing to $PS_EXIT_UNKNOWN_ERROR");
+        $exit_code = $PS_EXIT_UNKNOWN_ERROR;
+    }
     $job_state = 'run' unless $job_state;
 
Index: /tags/ipp-20111222/pstamp/scripts/pstamp_parser_run.pl
===================================================================
--- /tags/ipp-20111222/pstamp/scripts/pstamp_parser_run.pl	(revision 33173)
+++ /tags/ipp-20111222/pstamp/scripts/pstamp_parser_run.pl	(revision 33174)
@@ -294,4 +294,8 @@
     carp($msg);
 
+    if (!$req_id) {
+        exit $PS_EXIT_CONFIG_ERROR;
+    }
+
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;