Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl
===================================================================
--- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl	(revision 37270)
+++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl	(revision 37271)
@@ -14,14 +14,41 @@
 my $local_raw = "${remote_root}/raw/";
 my $local_tmp = "${remote_root}/tmp/";
-my $threads = 10;
-my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
-	     'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
-	     'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
-	     'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
+my $threads = 11;
+my @hosts = (
+    'ippc20.ipp.ifa.hawaii.edu',
+    'ippc21.ipp.ifa.hawaii.edu',
+    'ippc22.ipp.ifa.hawaii.edu',
+    'ippc23.ipp.ifa.hawaii.edu',
+    'ippc24.ipp.ifa.hawaii.edu',
+    'ippc25.ipp.ifa.hawaii.edu',
+    'ippc26.ipp.ifa.hawaii.edu',
+    'ippc27.ipp.ifa.hawaii.edu',
+    'ippc28.ipp.ifa.hawaii.edu',	
+#'ippc29.ipp.ifa.hawaii.edu', (mark's hi-mem test machine)
+#'ippc30.ipp.ifa.hawaii.edu', (postage stamp server)	
+    'ippc31.ipp.ifa.hawaii.edu',	
+    'ippc32.ipp.ifa.hawaii.edu'    
+    );
 @hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
+my %server_options = (
+    'ippc20.ipp.ifa.hawaii.edu' => '',
+    'ippc21.ipp.ifa.hawaii.edu' => '',
+    'ippc22.ipp.ifa.hawaii.edu' => '',
+    'ippc23.ipp.ifa.hawaii.edu' => '',
+    'ippc24.ipp.ifa.hawaii.edu' => '',
+    'ippc25.ipp.ifa.hawaii.edu' => '',
+    'ippc26.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc27.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc28.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc29.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc30.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc31.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes'
+    );
 my $input_file;
 my $verbose = 0;
 my $fetch = 0;
 my $offset = 0;
+my $quickfetch = 0;
 
 GetOptions(
@@ -31,4 +58,5 @@
     'offset=s'    => \$offset,
     'verbose'     => \$verbose,
+    'quickfetch'  => \$quickfetch,
     ) or pod2usage( 2 );
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
@@ -43,8 +71,13 @@
 unless(-d $local_raw) { system("mkdir -p $local_raw"); }
 
+# open the input file list
 open(I,$input_file) || die "Couldn't find input file specified\n";
+
+# open the fetch file list, if provided
 if ($fetch) {
     open(F,$fetch) || die "Couldn't find fetch file specified\n";
 }
+
+# generate N output files (to be fed to the N rsync / tar threads)
 my $input_base = basename($input_file);
 my @filehandles;
@@ -54,4 +87,5 @@
     open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
 }
+
 $i = 0;
 while(<I>) {
@@ -60,4 +94,8 @@
     if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
 
+    if ($line % 250 == 0)  {
+	print "line $line : $_\n";
+    }
+
     $i = int(rand($#filehandles + 1));
 
@@ -66,5 +104,11 @@
 	$fline = <F>;
     }
-    if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
+    
+    # if we are fetching, and 'quickfetch' is requested, skip file if it exists
+    # NOTE: this does not check that the transfer was complete or the file is current
+
+    if ($fetch) {
+	# We are fetching, and do not already have this file.
+	if ($quickfetch && (-e $_)) { next; }
 	print { $filehandles[$i] } $fline;
     }
@@ -72,5 +116,10 @@
 	# The rsync call expects to find files of a given name in the directory specified.
 #	print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
+#	if (-e "${local_tmp}/$_") {
 	print { $filehandles[$i] } "$_" . "\n";
+	# }
+	# else {
+	#     print "missing $_\n";
+	# }
     }
 }
@@ -90,13 +139,17 @@
 	if ($fetch) { 
 	    $code = fetch_task($host,"${input_base}.${i}", 0);
-	}
-	else {
+	} else {
 	    $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0);
 	}
+	print STDERR "$host $input_base $i $code\n";
 	exit($code);
     }
 }
+my $global_status = 0;
 for ($i = 0; $i < $threads; $i++) {
     waitpid($pids[$i],0);
+    my $this_status = $?;
+    print "exit status $hosts[$i] : $this_status\n";
+    if ($this_status) { $global_status = ($this_status >> 8); }
 }
 
@@ -105,4 +158,6 @@
 }
 
+print "global status: $global_status\n";
+exit ($global_status);
 
 # distribute bundles to nodes
@@ -116,37 +171,40 @@
 
     # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-#    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
-    my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
+    # my $command = "tar cf - --ignore-failed-read --dereference"
+    # $command = "$command --files-from=${transfer_filelist}"
+    # $command = "$command | /usr/projects/cosmo/amd6100/bin/ssh"
+    # $command = "$command -o NoneSwitch=yes -o NoneEnabled=yes
+    # $command = "$command $destination_host tar xf - -C /data/"
+    # $command = "$command --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
+
+    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
+    my $option = $server_options{$destination_host};
+    my $command = "rsync -Lt --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
-
+    
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
+
+    my $return_value = 0;
     unless ($success) {
-	print STDERR "ERROR:  $error_code\n";
-# 	my $std = join "\n", @{ $stdout_buf };
-# 	print "STDOUT: $std\n";
-# 	$std = join "\n", @{ $stderr_buf };
-# 	print "STDERR: $std\n";
-
-# 	foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
-# 	    if ($line =~ /No such file or directory/) { 
-# 		my $file = (split /\s+/, $line)[1];
-# 		$file =~ s/:$//;
-# 		if ($file !~ /nebulous/) { next; }
-# 		print STDERR "Touching $file so tar can continue.\n";
-# 		system("touch $file");
-# 	    }
-# 	}
-
-	warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-	$error++;
-	if ($error < 4) {
-	    $error_code = transfer_task($destination_host,$transfer_filelist,$error);
-	}
-	else {
-	    die("Failed too many times $error $destination_host $transfer_filelist");
-	}
-    }
-    return($error_code);
+	if ($error_code =~ /value 23/) {
+	    print STDERR "Some files failed to transfer.  Partial run?\n";
+	    $error = 23;
+	    $return_value = 23;
+	} else {
+	    print STDERR "ERROR:  $error_code\n";
+	    warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
+	    sleep(5);
+	    $error++;
+	    if ($error < 4) {
+		$error_code = transfer_task($destination_host,$transfer_filelist,$error);
+	    }
+	    else {
+		die("Failed too many times $error $destination_host $transfer_filelist");
+	    }
+	    $return_value = 1;
+	}
+    }
+    return($return_value);
 }
     
@@ -156,19 +214,48 @@
     my $error = shift;
 
-    system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
-    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
+    my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
+    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
+    my $option = $server_options{$destination_host};
+
+    # get the list of files to be transferred
+    # print "$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/\n";
+    # system("$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
+    # my $cmd1 = "rsync -e '$ssh_cmd $option' -auv ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/";
+    # print "$cmd1\n";
+    # system ("$cmd1");
+
+    # do the transfer 
+
+    # XXX EAM : fix this (use rsync not tar)
+    # my $command = "rsync -e '$ssh_cmd ${option}' --files-from=/tmp/${transfer_filelist} ${destination_host}:/ ${local_raw}";
+    my $command = "rsync -e '$ssh_cmd ${option}' --files-from=${local_tmp}/${transfer_filelist} ${destination_host}:/ ${local_raw}";
+
+    # my $command = "$ssh_cmd $option $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
     print STDERR "$command\n";
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    my $error_code = 0;
+    my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or 4);
-	warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-	$error++;
-	if ($error < 4) {
-	    $error_code = fetch_task($destination_host,$transfer_filelist,$error);
-	}
-	else {
-	    die("Failed too many times $error $destination_host $transfer_filelist");
+	print "raw error_msg: $error_msg...\n";
+	($error_code) = $error_msg =~ m/exited with value (\d+)/;
+	warn("Transfer of files in $local_tmp/$transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
+	print "*** stdout: *** \n";
+	foreach $line (@$stdout_buf) {
+	    print STDERR "stdout: $line\n";
+	}
+	print "*** stderr: *** \n";
+	foreach $line (@$stderr_buf) {
+	    print STDERR "stderr: $line\n";
+	}
+	if ($error_code == 23) {
+	    print STDERR "Some files failed to transfer\n";
+	} else {
+	    $error++;
+	    if ($error < 4) {
+		$error_code = fetch_task($destination_host,$transfer_filelist,$error);
+	    } else {
+		print STDERR "Failed too many times $error $destination_host $transfer_filelist\n";
+	    }
 	}
     }