Changeset 37287

Timestamp:

Aug 22, 2014, 9:13:32 AM (12 years ago)

Author:

eugene

Message:

rework transfer script for better symmetry and checking of local (LANL) files

File:

• tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl (modified) (12 diffs)

tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl

@@ -10 +10 @@
 use Sys::Hostname;
 
+my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
+my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
+
 my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
-$remote_root = '/scratch3/watersc1/';
+   $remote_root = '/scratch3/watersc1/';
+
 my $local_raw = "${remote_root}/raw/";
 my $local_tmp = "${remote_root}/tmp/";
@@ -46 +50 @@
     'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes'
     );
-my $input_file;
+
+my $input_path;
+my $local_file;
+my $remote_file;
+
 my $verbose = 0;
 my $fetch = 0;
@@ -54 +62 @@
 GetOptions(
     'threads=s'   => \$threads,
-    'input=s'     => \$input_file,
-    'fetch=s'     => \$fetch,
+    'input=s'     => \$input_path,
     'offset=s'    => \$offset,
+    'fetch'       => \$fetch,
     'verbose'     => \$verbose,
     'quickfetch'  => \$quickfetch,
@@ -62 +70 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
 pod2usage( -msg => "Required options: --input", -exitval => 3) unless
-    defined($input_file);
+    defined($input_path);
 
 my $hostname = hostname;
@@ -71 +79 @@
 unless(-d $local_raw) { system("mkdir -p $local_raw"); }
 
+# we have two modes: fetch (-> LANL) and return (-> IPP)
+
+if ($fetch) {
+    $local_file = "$input_path.check";
+    $remote_file = "$input_path.transfer";
+} else {
+    $local_file = "$input_path.generate";
+    $remote_file = "$input_path.return";
+}
+
 # open the input file list
-open(I,$input_file) || die "Couldn't find input file specified\n";
-
-# open the fetch file list, if provided
-if ($fetch) {
-    open(F,$fetch) || die "Couldn't find fetch file specified\n";
-}
+open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n";
+open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n";
 
 # generate N output files (to be fed to the N rsync / tar threads)
-my $input_base = basename($input_file);
+my $input_base = basename($local_file);
 my @filehandles;
 my $i;
@@ -88 +102 @@
 }
 
-open (STATFILE, ">${local_tmp}/${input_base}.stat");
+my $stat_file = "$input_path.stat";
+print "STATFILE: $stat_file\n";
+open (STATFILE, ">$stat_file");
 
 $i = 0;
-while(<I>) {
-    chomp;
+while (my $Lname = <LOCAL>) {
+    my $Rname = <REMOTE>;
+
+    chomp $Lname;
+    chomp $Rname;
+
     $line++;
     if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
 
-    if ($line % 250 == 0)  {
-        print "line $line : $_\n";
-    }
+    if ($line % 250 == 0)  { print "line $line : $Lname\n"; }
 
     $i = int(rand($#filehandles + 1));
-
-    my $fline;
-    if ($fetch) {
-        $fline = <F>;
-    }
     
     # if we are fetching, and 'quickfetch' is requested, skip file if it exists
@@ -112 +125 @@
     if ($fetch) {
         # We are fetching, and do not already have this file.
-        if ($quickfetch && (-e $_)) { next; }
-        print { $filehandles[$i] } $fline;
+        if ($quickfetch && (-e $Lname)) { next; }
+        print { $filehandles[$i] } $Rname;
     } else { 
         # We are pushing, but only send back files we actually have
-        if (-e "$_") {
-            print { $filehandles[$i] } "$_" . "\n";
-            print STATFILE, "$_ PASS\n";
-        } else {
-            print STATFILE, "$_ FAIL\n";
-        }
-    }
-}
-close(I);
+        if (-e $Lname) {
+            print { $filehandles[$i] } "$Rname\n";
+            print STATFILE "$Rname PASS\n";
+        } else {
+            print STATFILE "$Rname FAIL\n";
+        }
+    }
+}
+close(LOCAL);
+close(REMOTE);
 close(STATFILE);
 
@@ -139 +153 @@
         my $code = 0;
         if ($fetch) { 
-            $code = fetch_task($host,"${input_base}.${i}", 0);
+            $code = fetch_task($host,"${local_tmp}/${input_base}.${i}", 0);
         } else {
             $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0);
@@ -160 +174 @@
 }
 
-my $scp_command = "$scp_cmd $option ${local_tmp}/${input_base}.stat $hosts[0]:/tmp/";
-print "$scp_command\n";
-system("$scp_command");
+# my $scp_command = "$scp_cmd $option ${local_tmp}/${input_base}.stat $hosts[0]:/tmp/";
+# print "$scp_command\n";
+# system("$scp_command");
 # XXX check return status
 
@@ -177 +191 @@
     my $error = shift;
 
-    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-    # my $command = "tar cf - --ignore-failed-read --dereference"
-    # $command = "$command --files-from=${transfer_filelist}"
-    # $command = "$command | /usr/projects/cosmo/amd6100/bin/ssh"
-    # $command = "$command -o NoneSwitch=yes -o NoneEnabled=yes
-    # $command = "$command $destination_host tar xf - -C /data/"
-    # $command = "$command --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
-
-    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
     my $option = $server_options{$destination_host};
+
     my $command = "rsync -Lt --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
@@ -221 +227 @@
     my $error = shift;
 
-    my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
-    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
     my $option = $server_options{$destination_host};
 
-    # get the list of files to be transferred
-    # print "$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/\n";
-    # system("$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
-    # my $cmd1 = "rsync -e '$ssh_cmd $option' -auv ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/";
-    # print "$cmd1\n";
-    # system ("$cmd1");
-
-    # do the transfer 
-
-    # XXX EAM : fix this (use rsync not tar)
-    # my $command = "rsync -e '$ssh_cmd ${option}' --files-from=/tmp/${transfer_filelist} ${destination_host}:/ ${local_raw}";
-    my $command = "rsync -e '$ssh_cmd ${option}' --files-from=${local_tmp}/${transfer_filelist} ${destination_host}:/ ${local_raw}";
-
-    # my $command = "$ssh_cmd $option $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
+    my $command = "rsync -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_raw}";
     print STDERR "$command\n";
 
@@ -247 +238 @@
         print "raw error_msg: $error_msg...\n";
         ($error_code) = $error_msg =~ m/exited with value (\d+)/;
-        warn("Transfer of files in $local_tmp/$transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
+        warn("Transfer of files in $transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
         print "*** stdout: *** \n";
         foreach $line (@$stdout_buf) {