IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37324


Ignore:
Timestamp:
Aug 27, 2014, 4:04:55 PM (12 years ago)
Author:
eugene
Message:

rsync only uses size to check validity; add retry option

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl

    r37298 r37324  
    5858my $fetch = 0;
    5959my $offset = 0;
     60my $retry = 0;
    6061my $quickfetch = 0;
    6162
    … …  
    6667    'fetch'       => \$fetch,
    6768    'verbose'     => \$verbose,
     69    'retry'       => \$retry,
    6870    'quickfetch'  => \$quickfetch,
    6971    ) or pod2usage( 2 );
    … …  
    8890    $remote_file = "$input_path.return";
    8991}
    90 
    91 # open the input file list
    92 open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n";
    93 open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n";
    94 
    95 # generate N output files (to be fed to the N rsync / tar threads)
    9692my $input_base = basename($local_file);
    97 my @filehandles;
     93
     94unless ($retry) {
     95  # open the input file list
     96  open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n";
     97  open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n";
     98 
     99  # generate N output files (to be fed to the N rsync / tar threads)
     100  my @filehandles;
     101  my $i;
     102  my $line = 0;
     103  for ($i = 0; $i < $threads; $i++) {
     104      open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
     105  }
     106 
     107  unless ($fetch) {
     108      my $stat_file = "$input_path.stat";
     109      print "STATFILE: $stat_file\n";
     110      open (STATFILE, ">$stat_file");
     111  }
     112 
     113  $i = 0;
     114  while (my $Lname = <LOCAL>) {
     115      my $Rname = <REMOTE>;
     116 
     117      chomp $Lname;
     118      chomp $Rname;
     119 
     120      $line++;
     121      if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
     122 
     123      if ($line % 250 == 0)  { print STDERR "line $line : $Lname\n"; }
     124 
     125      $i = int(rand($#filehandles + 1));
     126      if ($i >= $threads) {
     127        print STDERR "HUH: impossible file handle?\n";
     128        die;
     129      }
     130 
     131      # if we are fetching, and 'quickfetch' is requested, skip file if it exists
     132      # NOTE: this does not check that the transfer was complete or the file is current
     133 
     134      if ($fetch) {
     135        # We are fetching, and do not already have this file.
     136        if ($quickfetch && (-e $Lname)) { next; }
     137        print { $filehandles[$i] } "$Rname\n";
     138      } else {
     139        # We are pushing, but only send back files we actually have
     140        # we cannot have the remote file unless the local file exists
     141        # (it is a link to the local file and is generated after the local file)
     142        my $haveRemote = -e "$local_tmp/$Rname";
     143        if ($haveRemote) {
     144            print { $filehandles[$i] } "$Rname\n";
     145            print STATFILE "$Rname PASS\n";
     146        } else {
     147            my $haveLocal = -e $Lname;
     148            if ($haveLocal) {
     149                print STATFILE "$Rname PART\n";
     150            } else {
     151                print STATFILE "$Rname FAIL\n";
     152            }
     153        }
     154      }
     155  }
     156  close(LOCAL);
     157  close(REMOTE);
     158  unless ($fetch) { close(STATFILE); }
     159 
     160  for ($i = 0; $i < $threads; $i++) {
     161      close($filehandles[$i]);
     162  }
     163}
     164
    98165my $i;
    99 my $line = 0;
    100 for ($i = 0; $i < $threads; $i++) {
    101     open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
    102 }
    103 
    104 unless ($fetch) {
    105     my $stat_file = "$input_path.stat";
    106     print "STATFILE: $stat_file\n";
    107     open (STATFILE, ">$stat_file");
    108 }
    109 
    110 $i = 0;
    111 while (my $Lname = <LOCAL>) {
    112     my $Rname = <REMOTE>;
    113 
    114     chomp $Lname;
    115     chomp $Rname;
    116 
    117     $line++;
    118     if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
    119 
    120     if ($line % 250 == 0)  { print STDERR "line $line : $Lname\n"; }
    121 
    122     $i = int(rand($#filehandles + 1));
    123     if ($i >= $threads) {
    124         print STDERR "HUH: impossible file handle?\n";
    125         die;
    126     }
    127 
    128     # if we are fetching, and 'quickfetch' is requested, skip file if it exists
    129     # NOTE: this does not check that the transfer was complete or the file is current
    130 
    131     if ($fetch) {
    132         # We are fetching, and do not already have this file.
    133         if ($quickfetch && (-e $Lname)) { next; }
    134         print { $filehandles[$i] } $Rname;
    135     } else {
    136         # We are pushing, but only send back files we actually have
    137         # we cannot have the remote file unless the local file exists
    138         # (it is a link to the local file and is generated after the local file)
    139         my $haveRemote = -e "$local_tmp/$Rname";
    140         if ($haveRemote) {
    141             print { $filehandles[$i] } "$Rname\n";
    142             print STATFILE "$Rname PASS\n";
    143         } else {
    144             my $haveLocal = -e $Lname;
    145             if ($haveLocal) {
    146                 print STATFILE "$Rname PART\n";
    147             } else {
    148                 print STATFILE "$Rname FAIL\n";
    149             }
    150         }
    151     }
    152 }
    153 close(LOCAL);
    154 close(REMOTE);
    155 unless ($fetch) { close(STATFILE); }
    156 
    157 for ($i = 0; $i < $threads; $i++) {
    158     close($filehandles[$i]);
    159 }
    160 
    161166# fork the rsync
    162167my @pids = ();
    … …  
    207212    my $option = $server_options{$destination_host};
    208213
    209     my $command = "rsync -Lt --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     214    my $command = "rsync -Lt --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
    210215    print STDERR "$command\n";
    211216   
    … …  
    243248    my $option = $server_options{$destination_host};
    244249
    245     my $command = "rsync -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_raw}";
     250    my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_raw}";
    246251    print STDERR "$command\n";
    247252
    … …  
    254259        warn("Transfer of files in $transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
    255260        print "*** stdout: *** \n";
    256         foreach $line (@$stdout_buf) {
     261        foreach my $line (@$stdout_buf) {
    257262            print STDERR "stdout: $line\n";
    258263        }
    259264        print "*** stderr: *** \n";
    260         foreach $line (@$stderr_buf) {
     265        foreach my $line (@$stderr_buf) {
    261266            print STDERR "stderr: $line\n";
    262267        }
Note: See TracChangeset for help on using the changeset viewer.