Index: trunk/ippScripts/scripts/sc_transfer_tool.pl
===================================================================
--- trunk/ippScripts/scripts/sc_transfer_tool.pl	(revision 36970)
+++ trunk/ippScripts/scripts/sc_transfer_tool.pl	(revision 37013)
@@ -12,12 +12,14 @@
 my $local_raw = '/scratch3/watersc1/raw/';
 my $local_tmp = '/scratch3/watersc1/tmp/';
-my $threads = 8;
+my $threads = 10;
 my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
 	     'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
 	     'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
-	     'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu');
+	     'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
+@hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
 my $input_file;
 my $verbose = 0;
 my $fetch = 0;
+my $offset = 0;
 
 GetOptions(
@@ -25,4 +27,5 @@
     'input=s'     => \$input_file,
     'fetch=s'     => \$fetch,
+    'offset=s'    => \$offset,
     'verbose'     => \$verbose,
     ) or pod2usage( 2 );
@@ -45,4 +48,5 @@
 my @filehandles;
 my $i;
+my $line = 0;
 for ($i = 0; $i < $threads; $i++) {
     open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
@@ -51,5 +55,8 @@
 while(<I>) {
     chomp;
-    $i = int(rand($#filehandles));
+    $line++;
+    if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
+
+    $i = int(rand($#filehandles + 1));
 
     my $fline;
@@ -59,17 +66,8 @@
     if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
 	print { $filehandles[$i] } $fline;
-#	$i++;
     }
     elsif (!($fetch)) { # We are pushing
-#	if (-e "${local_tmp}/$_") { 
-	    print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
-#	    $i++;
-#	}
-#	else { # But somehow couldn't find the file we expected
-#	    print STDERR "Expected file ${local_tmp}/$_ not found!\n";
-#	    die;
-#	}
+	print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
     }
-#    if ($i >= $threads) { $i = 0; }
 }
 close(I);
@@ -84,6 +82,6 @@
     $pids[$i] = fork();
     if ($pids[$i] == 0) {
-	my $host = $hosts[rand($#hosts)];
-	my $code;
+	my $host = $hosts[$i];
+	my $code = 0;
 	if ($fetch) { 
 	    $code = fetch_task($host,"${input_base}.${i}", 0);
@@ -98,8 +96,4 @@
     waitpid($pids[$i],0);
 }
-# die;
-# if ($fetch) {
-#     system("parallel -j $threads  ln -sf ${local_tmp}/{2} {1} :::: $input_file :::: $fetch");
-# }
 
 for ($i = 0; $i < $threads; $i++) {
@@ -117,11 +111,28 @@
     my $error = shift;
 
-    my $command = "tar cf - --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
+    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
+    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
     print STDERR "$command\n";
-#    die;
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or 4);
+
+	print STDERR "ERROR:  $error_code\n";
+# 	my $std = join "\n", @{ $stdout_buf };
+# 	print "STDOUT: $std\n";
+# 	$std = join "\n", @{ $stderr_buf };
+# 	print "STDERR: $std\n";
+
+	foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
+	    if ($line =~ /No such file or directory/) { 
+		my $file = (split /\s+/, $line)[1];
+		$file =~ s/:$//;
+		if ($file !~ /nebulous/) { next; }
+		print STDERR "Touching $file so tar can continue.\n";
+		system("touch $file");
+	    }
+	}
+
 	warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
 	$error++;
@@ -142,5 +153,5 @@
 
     system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
-    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
+    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
     print STDERR "$command\n";