Index: trunk/ippScripts/scripts/sc_remote_exec.pl
===================================================================
--- trunk/ippScripts/scripts/sc_remote_exec.pl	(revision 37151)
+++ trunk/ippScripts/scripts/sc_remote_exec.pl	(revision 37187)
@@ -18,7 +18,6 @@
 my $SEC_HOST = 'mu-fe4';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
-#my $remote_root  = '/scratch3/watersc1/';
-my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
-$remote_root = '/scratch3/watersc1/';
+my $remote_root  = '/scratch3/watersc1/';
+
 # tools
 my $missing_tools;
@@ -31,5 +30,4 @@
     exit($PS_EXIT_CONFIG_ERROR);
 }
-
 
 # Options
@@ -56,5 +54,4 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
-
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    
 
@@ -152,8 +149,11 @@
 my $poll_sleep = 60;
 my $poll_response = 0;
+my $poll_word  = "nothing";
+
 if ($force) {
     $poll_response = 1;
-}
-my $poll_word  = "nothing";
+    $poll_word     = "Completed";
+}
+
 while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
     if ($verbose) {
@@ -171,5 +171,4 @@
 	warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
 	$poll_response = 1; # To fool the check later into proceding
-#	&my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR);
     }
     $poll_count++;
@@ -181,6 +180,5 @@
     &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
 }
-# Retrieve validation summary
-my $fault = 0;
+
 
 # Initialize the remote side to sync things
@@ -193,5 +191,5 @@
 #scp_get($remote_return,$ipp_return);
 
-# Check to see if we can resume.  Pack the files into an array
+# Construct list of dbinfo files we'll need to execute.
 my @returned_files = ();
 my @dbinfo_files   = ();
@@ -211,9 +209,4 @@
 close(RETURN);
 print STDERR "Expect $#dbinfo_files dbinfo files\n";
-# Binary search it.  BUT: We're binary searching binary values.  So,
-# this has to be far more careful than a fully sorted case would be.
-# Only scan the dbinfo files as those are the things that are
-# generated after the processing, and so must be the last things to
-# exist.
 
 my $first = 0;
@@ -222,36 +215,9 @@
 my $range = 20;  # How wide of a chunk to look through to catch threading drops
 my $offset = 0;  # Offset to start from.
-# This doesn't work as well as I'd like, and with the rsync, it's less needed.
-# if (( stat($dbinfo_files[$last]) )[7] != 0) { $offset = $#dbinfo_files;}
-# else {
-#     my $count = 0;
-#     while ($first < $last - $range) {
-
-# 	$mid = int(($last + $first)/2);
-# 	for (my $i = 0; $i <= $range; $i++) {
-# 	    my $size = (stat($dbinfo_files[$mid + $i]))[7];
-# #	    print STDERR "$first $last $mid $i $size $dbinfo_files[$mid + $i]\n";
-# 	    if ($size == 0) { 
-# 		if ($last == $mid + $i) { 
-# 		    $first = $last - 1;
-# 		}
-# 		$last = $mid + $i; 
-# 		$i = $range + 10;
-# 	    }
-# 	}
-# 	if ($last > $mid + $range) { $first = $mid + $range; }
-# 	$count++;
-# 	if ($count > 200) { 
-# 	    &my_die("The iteration loop couldn't converge after 200 rounds, suggesting something has gone wrong.",
-# 		    $remote_id,4,"error_run");
-# 	}
-#     }
-#     $offset = $mid - $range; # Because I want to make sure I get to the beginning of the thread group.
-#     $offset = $dbindex[$offset]; # Because we need to convert from dbinfo space to returned file space.
-# }
-
-# Feed teh return list into the transfer tool
+
+# Feed the return list into the transfer tool
 # If this fails, we should go to an auth wait point.
-if ($poll_word eq 'Completed') { 
+if ($poll_word eq 'Completed') {  
+    # We're complete.  All files should exist on disk
     my $iter = 0;
     my $transfer_success = 1;
@@ -267,4 +233,5 @@
 }
 else {
+    # Just try and fetch things, because we didn't complete correctly, it seems.
     my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
     my $return_push_output_print = join "\n", @{ $return_push_output };
@@ -282,9 +249,12 @@
 	run(command => $cmd, verbose => $verbose);
     unless ($success) {
+
 	# This shouldn't fail, but we also can't suddenly abort if something does fail.  
 	# Now we're going to drop the component that owns this dbinfo, which should force a retry.
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
 	warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
-	drop_component($remote_id,$cmd,$file);
+	if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
+	    drop_component($remote_id,$cmd,$file);
+	}
     }
 
@@ -342,4 +312,5 @@
     my $cmd = shift;
     $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $cmd, verbose => $verbose);