Index: trunk/ippScripts/scripts/sc_transfer_tool.pl
===================================================================
--- trunk/ippScripts/scripts/sc_transfer_tool.pl	(revision 37125)
+++ trunk/ippScripts/scripts/sc_transfer_tool.pl	(revision 37833)
@@ -10,29 +10,67 @@
 use Sys::Hostname;
 
+my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
+my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
+
 my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
-$remote_root = '/scratch3/watersc1/';
-my $local_raw = "${remote_root}/raw/";
+   $remote_root = '/scratch3/watersc1/';
+
+#my $local_raw = "${remote_root}/tmp/";
 my $local_tmp = "${remote_root}/tmp/";
-my $threads = 10;
-my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
-	     'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
-	     'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
-	     'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
+my $threads = 11;
+my @hosts = (
+    'ippc20.ipp.ifa.hawaii.edu',
+    'ippc21.ipp.ifa.hawaii.edu',
+    'ippc22.ipp.ifa.hawaii.edu',
+    'ippc23.ipp.ifa.hawaii.edu',
+    'ippc24.ipp.ifa.hawaii.edu',
+    'ippc25.ipp.ifa.hawaii.edu',
+    'ippc26.ipp.ifa.hawaii.edu',
+    'ippc27.ipp.ifa.hawaii.edu',
+    'ippc28.ipp.ifa.hawaii.edu',	
+#'ippc29.ipp.ifa.hawaii.edu', (mark's hi-mem test machine)
+#'ippc30.ipp.ifa.hawaii.edu', (postage stamp server)	
+    'ippc31.ipp.ifa.hawaii.edu',	
+    'ippc32.ipp.ifa.hawaii.edu'    
+    );
 @hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
-my $input_file;
+my %server_options = (
+    'ippc20.ipp.ifa.hawaii.edu' => '',
+    'ippc21.ipp.ifa.hawaii.edu' => '',
+    'ippc22.ipp.ifa.hawaii.edu' => '',
+    'ippc23.ipp.ifa.hawaii.edu' => '',
+    'ippc24.ipp.ifa.hawaii.edu' => '',
+    'ippc25.ipp.ifa.hawaii.edu' => '',
+    'ippc26.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc27.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc28.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc29.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc30.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc31.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes'
+    );
+
+my $input_path;
+my $local_file;
+my $remote_file;
+
 my $verbose = 0;
 my $fetch = 0;
 my $offset = 0;
+my $retry = 0;
+my $quickfetch = 0;
 
 GetOptions(
     'threads=s'   => \$threads,
-    'input=s'     => \$input_file,
-    'fetch=s'     => \$fetch,
+    'input=s'     => \$input_path,
     'offset=s'    => \$offset,
+    'fetch'       => \$fetch,
     'verbose'     => \$verbose,
+    'retry'       => \$retry,
+    'quickfetch'  => \$quickfetch,
     ) or pod2usage( 2 );
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
 pod2usage( -msg => "Required options: --input", -exitval => 3) unless
-    defined($input_file);
+    defined($input_path);
 
 my $hostname = hostname;
@@ -41,45 +79,90 @@
 # split in input file list
 unless(-d $local_tmp) { system("mkdir -p $local_tmp"); }
-unless(-d $local_raw) { system("mkdir -p $local_raw"); }
-
-open(I,$input_file) || die "Couldn't find input file specified\n";
+#unless(-d $local_raw) { system("mkdir -p $local_raw"); }
+
+# we have two modes: fetch (-> LANL) and return (-> IPP)
+
 if ($fetch) {
-    open(F,$fetch) || die "Couldn't find fetch file specified\n";
-}
-my $input_base = basename($input_file);
-my @filehandles;
+    $local_file = "$input_path.check";
+    $remote_file = "$input_path.transfer";
+} else {
+    $local_file = "$input_path.generate";
+    $remote_file = "$input_path.return";
+}
+my $input_base = basename($local_file);
+
+unless ($retry) {
+  # open the input file list
+  open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n";
+  open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n";
+  
+  # generate N output files (to be fed to the N rsync / tar threads)
+  my @filehandles;
+  my $i;
+  my $line = 0;
+  for ($i = 0; $i < $threads; $i++) {
+      open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
+  }
+  
+  unless ($fetch) {
+      my $stat_file = "$input_path.stat";
+      print "STATFILE: $stat_file\n";
+      open (STATFILE, ">$stat_file");
+  }
+  
+  $i = 0;
+  while (my $Lname = <LOCAL>) {
+      my $Rname = <REMOTE>;
+  
+      chomp $Lname;
+      chomp $Rname;
+  
+      $line++;
+      if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
+  
+      if ($line % 250 == 0)  { print STDERR "line $line : $Lname\n"; }
+  
+      $i = int(rand($#filehandles + 1));
+      if ($i >= $threads) {
+  	print STDERR "HUH: impossible file handle?\n";
+  	die;
+      }
+  
+      # if we are fetching, and 'quickfetch' is requested, skip file if it exists
+      # NOTE: this does not check that the transfer was complete or the file is current
+  
+      if ($fetch) {
+  	# We are fetching, and do not already have this file.
+  	if ($quickfetch && (-e $Lname)) { next; }
+  	print { $filehandles[$i] } "$Rname\n";
+      } else { 
+  	# We are pushing, but only send back files we actually have
+  	# we cannot have the remote file unless the local file exists 
+  	# (it is a link to the local file and is generated after the local file)
+  	my $haveRemote = -e "$local_tmp/$Rname";
+  	if ($haveRemote) {
+  	    print { $filehandles[$i] } "$Rname\n";
+  	    print STATFILE "$Rname PASS\n";
+  	} else {
+  	    my $haveLocal = -e $Lname;
+  	    if ($haveLocal) {
+  		print STATFILE "$Rname PART\n";
+  	    } else {
+  		print STATFILE "$Rname FAIL\n";
+  	    }
+  	}
+      }
+  }
+  close(LOCAL);
+  close(REMOTE);
+  unless ($fetch) { close(STATFILE); }
+  
+  for ($i = 0; $i < $threads; $i++) {
+      close($filehandles[$i]);
+  }
+}
+
 my $i;
-my $line = 0;
-for ($i = 0; $i < $threads; $i++) {
-    open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
-}
-$i = 0;
-while(<I>) {
-    chomp;
-    $line++;
-    if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
-
-    $i = int(rand($#filehandles + 1));
-
-    my $fline;
-    if ($fetch) {
-	$fline = <F>;
-    }
-    if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
-	print { $filehandles[$i] } $fline;
-    }
-    elsif (!($fetch)) { # We are pushing
-	# The rsync call expects to find files of a given name in the directory specified.
-#	print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
-	print { $filehandles[$i] } "$_" . "\n";
-    }
-}
-close(I);
-
-for ($i = 0; $i < $threads; $i++) {
-    close($filehandles[$i]);
-}
-
-# fork the tars
+# fork the rsync
 my @pids = ();
 for ($i = 0; $i < $threads; $i++) {
@@ -89,14 +172,19 @@
 	my $code = 0;
 	if ($fetch) { 
-	    $code = fetch_task($host,"${input_base}.${i}", 0);
-	}
-	else {
+	    $code = fetch_task($host,"${local_tmp}/${input_base}.${i}", 0);
+	} else {
 	    $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0);
 	}
+	print STDERR "$host $input_base $i $code\n";
 	exit($code);
     }
 }
+
+my $global_status = 0;
 for ($i = 0; $i < $threads; $i++) {
     waitpid($pids[$i],0);
+    my $this_status = $?;
+    print "exit status $hosts[$i] : $this_status\n";
+    if ($this_status) { $global_status = ($this_status >> 8); }
 }
 
@@ -105,4 +193,11 @@
 }
 
+# my $scp_command = "$scp_cmd $option ${local_tmp}/${input_base}.stat $hosts[0]:/tmp/";
+# print "$scp_command\n";
+# system("$scp_command");
+# XXX check return status
+
+print "global status: $global_status\n";
+exit ($global_status);
 
 # distribute bundles to nodes
@@ -115,38 +210,33 @@
     my $error = shift;
 
-    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-#    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
-    my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
+    my $option = $server_options{$destination_host};
+
+    my $command = "rsync -L --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
-
+    
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
+
+    my $return_value = 0;
     unless ($success) {
-	print STDERR "ERROR:  $error_code\n";
-# 	my $std = join "\n", @{ $stdout_buf };
-# 	print "STDOUT: $std\n";
-# 	$std = join "\n", @{ $stderr_buf };
-# 	print "STDERR: $std\n";
-
-# 	foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
-# 	    if ($line =~ /No such file or directory/) { 
-# 		my $file = (split /\s+/, $line)[1];
-# 		$file =~ s/:$//;
-# 		if ($file !~ /nebulous/) { next; }
-# 		print STDERR "Touching $file so tar can continue.\n";
-# 		system("touch $file");
-# 	    }
-# 	}
-
-	warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-	$error++;
-	if ($error < 4) {
-	    $error_code = transfer_task($destination_host,$transfer_filelist,$error);
-	}
-	else {
-	    die("Failed too many times $error $destination_host $transfer_filelist");
+	if ($error_code =~ /value 23/) {
+	    print STDERR "Some files failed to transfer.  Partial run?\n";
+	    $error = 23;
+	    $return_value = 23;
+	} else {
+	    print STDERR "ERROR:  $error_code\n";
+	    warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
+	    sleep(5);
+	    $error++;
+	    if ($error < 4) {
+		$error_code = transfer_task($destination_host,$transfer_filelist,$error);
+	    }
+	    else {
+		die("Failed too many times $error $destination_host $transfer_filelist");
+	    }
+	    $return_value = 1;
 	}
     }
-    return($error_code);
+    return($return_value);
 }
     
@@ -156,19 +246,33 @@
     my $error = shift;
 
-    system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
-    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
+    my $option = $server_options{$destination_host};
+
+    my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_tmp}";
     print STDERR "$command\n";
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    my $error_code = 0;
+    my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => $verbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or 4);
-	warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-	$error++;
-	if ($error < 4) {
-	    $error_code = fetch_task($destination_host,$transfer_filelist,$error);
-	}
-	else {
-	    die("Failed too many times $error $destination_host $transfer_filelist");
+	print "raw error_msg: $error_msg...\n";
+	($error_code) = $error_msg =~ m/exited with value (\d+)/;
+	warn("Transfer of files in $transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
+	print "*** stdout: *** \n";
+	foreach my $line (@$stdout_buf) {
+	    print STDERR "stdout: $line\n";
+	}
+	print "*** stderr: *** \n";
+	foreach my $line (@$stderr_buf) {
+	    print STDERR "stderr: $line\n";
+	}
+	if ($error_code == 23) {
+	    print STDERR "Some files failed to transfer\n";
+	} else {
+	    $error++;
+	    if ($error < 4) {
+		$error_code = fetch_task($destination_host,$transfer_filelist,$error);
+	    } else {
+		print STDERR "Failed too many times $error $destination_host $transfer_filelist\n";
+	    }
 	}
     }