Changeset 37102 for trunk/ippScripts/scripts/sc_transfer_tool.pl

Timestamp:

Jul 22, 2014, 3:50:46 PM (12 years ago)

Author:

watersc1

Message:

path change, trace drop, config typo fix, stack psf fix, exec force poll addition, exec drop/fails (likely needs further tuning), transfer shift to rsync return handler.

File:

• trunk/ippScripts/scripts/sc_transfer_tool.pl (modified) (5 diffs)

trunk/ippScripts/scripts/sc_transfer_tool.pl

@@ -10 +10 @@
 use Sys::Hostname;
 
-my $local_raw = '/scratch3/watersc1/raw/';
-my $local_tmp = '/scratch3/watersc1/tmp/';
+my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
+
+my $local_raw = "${remote_root}/raw/";
+my $local_tmp = "${remote_root}/tmp/";
 my $threads = 10;
 my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
@@ -68 +70 @@
     }
     elsif (!($fetch)) { # We are pushing
-        print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
+        # The rsync call expects to find files of a given name in the directory specified.
+#       print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
+        print { $filehandles[$i] } "$_" . "\n";
     }
 }
@@ -112 +116 @@
 
     # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
+#    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
+    my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
 
@@ -118 +123 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-
         print STDERR "ERROR:  $error_code\n";
 #       my $std = join "\n", @{ $stdout_buf };
@@ -125 +129 @@
 #       print "STDERR: $std\n";
 
-        foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
-            if ($line =~ /No such file or directory/) { 
-                my $file = (split /\s+/, $line)[1];
-                $file =~ s/:$//;
-                if ($file !~ /nebulous/) { next; }
-                print STDERR "Touching $file so tar can continue.\n";
-                system("touch $file");
-            }
-        }
+#       foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
+#           if ($line =~ /No such file or directory/) { 
+#               my $file = (split /\s+/, $line)[1];
+#               $file =~ s/:$//;
+#               if ($file !~ /nebulous/) { next; }
+#               print STDERR "Touching $file so tar can continue.\n";
+#               system("touch $file");
+#           }
+#       }
 
         warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");