IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37271


Ignore:
Timestamp:
Aug 20, 2014, 12:03:58 PM (12 years ago)
Author:
eugene
Message:

unspool list of hosts for readibility; remove NoneSwitch (not working for 6.6.hpn?); add quickfetch option (default to rsync all); replace tar with rsync for fetch; add comments; add some verbosity; catch exit status of threads and set script exit status accordingingly; report fetch errors, allow for return failures (really?)

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl

    r37126 r37271  
    1414my $local_raw = "${remote_root}/raw/";
    1515my $local_tmp = "${remote_root}/tmp/";
    16 my $threads = 10;
    17 my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
    18              'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
    19              'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
    20              'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
     16my $threads = 11;
     17my @hosts = (
     18    'ippc20.ipp.ifa.hawaii.edu',
     19    'ippc21.ipp.ifa.hawaii.edu',
     20    'ippc22.ipp.ifa.hawaii.edu',
     21    'ippc23.ipp.ifa.hawaii.edu',
     22    'ippc24.ipp.ifa.hawaii.edu',
     23    'ippc25.ipp.ifa.hawaii.edu',
     24    'ippc26.ipp.ifa.hawaii.edu',
     25    'ippc27.ipp.ifa.hawaii.edu',
     26    'ippc28.ipp.ifa.hawaii.edu',       
     27#'ippc29.ipp.ifa.hawaii.edu', (mark's hi-mem test machine)
     28#'ippc30.ipp.ifa.hawaii.edu', (postage stamp server)   
     29    'ippc31.ipp.ifa.hawaii.edu',       
     30    'ippc32.ipp.ifa.hawaii.edu'   
     31    );
    2132@hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
     33my %server_options = (
     34    'ippc20.ipp.ifa.hawaii.edu' => '',
     35    'ippc21.ipp.ifa.hawaii.edu' => '',
     36    'ippc22.ipp.ifa.hawaii.edu' => '',
     37    'ippc23.ipp.ifa.hawaii.edu' => '',
     38    'ippc24.ipp.ifa.hawaii.edu' => '',
     39    'ippc25.ipp.ifa.hawaii.edu' => '',
     40    'ippc26.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     41    'ippc27.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     42    'ippc28.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     43    'ippc29.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     44    'ippc30.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     45    'ippc31.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
     46    'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes'
     47    );
    2248my $input_file;
    2349my $verbose = 0;
    2450my $fetch = 0;
    2551my $offset = 0;
     52my $quickfetch = 0;
    2653
    2754GetOptions(
    … …  
    3158    'offset=s'    => \$offset,
    3259    'verbose'     => \$verbose,
     60    'quickfetch'  => \$quickfetch,
    3361    ) or pod2usage( 2 );
    3462pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
    … …  
    4371unless(-d $local_raw) { system("mkdir -p $local_raw"); }
    4472
     73# open the input file list
    4574open(I,$input_file) || die "Couldn't find input file specified\n";
     75
     76# open the fetch file list, if provided
    4677if ($fetch) {
    4778    open(F,$fetch) || die "Couldn't find fetch file specified\n";
    4879}
     80
     81# generate N output files (to be fed to the N rsync / tar threads)
    4982my $input_base = basename($input_file);
    5083my @filehandles;
    … …  
    5487    open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
    5588}
     89
    5690$i = 0;
    5791while(<I>) {
    … …  
    6094    if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
    6195
     96    if ($line % 250 == 0)  {
     97        print "line $line : $_\n";
     98    }
     99
    62100    $i = int(rand($#filehandles + 1));
    63101
    … …  
    66104        $fline = <F>;
    67105    }
    68     if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
     106   
     107    # if we are fetching, and 'quickfetch' is requested, skip file if it exists
     108    # NOTE: this does not check that the transfer was complete or the file is current
     109
     110    if ($fetch) {
     111        # We are fetching, and do not already have this file.
     112        if ($quickfetch && (-e $_)) { next; }
    69113        print { $filehandles[$i] } $fline;
    70114    }
    … …  
    72116        # The rsync call expects to find files of a given name in the directory specified.
    73117#       print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
     118#       if (-e "${local_tmp}/$_") {
    74119        print { $filehandles[$i] } "$_" . "\n";
     120        # }
     121        # else {
     122        #     print "missing $_\n";
     123        # }
    75124    }
    76125}
    … …  
    90139        if ($fetch) {
    91140            $code = fetch_task($host,"${input_base}.${i}", 0);
    92         }
    93         else {
     141        } else {
    94142            $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0);
    95143        }
     144        print STDERR "$host $input_base $i $code\n";
    96145        exit($code);
    97146    }
    98147}
     148my $global_status = 0;
    99149for ($i = 0; $i < $threads; $i++) {
    100150    waitpid($pids[$i],0);
     151    my $this_status = $?;
     152    print "exit status $hosts[$i] : $this_status\n";
     153    if ($this_status) { $global_status = ($this_status >> 8); }
    101154}
    102155
    … …  
    105158}
    106159
     160print "global status: $global_status\n";
     161exit ($global_status);
    107162
    108163# distribute bundles to nodes
    … …  
    116171
    117172    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
    118 #    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference";
    119     my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     173    # my $command = "tar cf - --ignore-failed-read --dereference"
     174    # $command = "$command --files-from=${transfer_filelist}"
     175    # $command = "$command | /usr/projects/cosmo/amd6100/bin/ssh"
     176    # $command = "$command -o NoneSwitch=yes -o NoneEnabled=yes
     177    # $command = "$command $destination_host tar xf - -C /data/"
     178    # $command = "$command --transform '" . 's,^.\*/data/,,' . "' --dereference";
     179
     180    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
     181    my $option = $server_options{$destination_host};
     182    my $command = "rsync -Lt --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
    120183    print STDERR "$command\n";
    121 
     184   
    122185    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    123186        run(command => $command, verbose => $verbose);
     187
     188    my $return_value = 0;
    124189    unless ($success) {
    125         print STDERR "ERROR:  $error_code\n";
    126 #       my $std = join "\n", @{ $stdout_buf };
    127 #       print "STDOUT: $std\n";
    128 #       $std = join "\n", @{ $stderr_buf };
    129 #       print "STDERR: $std\n";
    130 
    131 #       foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
    132 #           if ($line =~ /No such file or directory/) {
    133 #               my $file = (split /\s+/, $line)[1];
    134 #               $file =~ s/:$//;
    135 #               if ($file !~ /nebulous/) { next; }
    136 #               print STDERR "Touching $file so tar can continue.\n";
    137 #               system("touch $file");
    138 #           }
    139 #       }
    140 
    141         warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
    142         $error++;
    143         if ($error < 4) {
    144             $error_code = transfer_task($destination_host,$transfer_filelist,$error);
    145         }
    146         else {
    147             die("Failed too many times $error $destination_host $transfer_filelist");
    148         }
    149     }
    150     return($error_code);
     190        if ($error_code =~ /value 23/) {
     191            print STDERR "Some files failed to transfer.  Partial run?\n";
     192            $error = 23;
     193            $return_value = 23;
     194        } else {
     195            print STDERR "ERROR:  $error_code\n";
     196            warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
     197            sleep(5);
     198            $error++;
     199            if ($error < 4) {
     200                $error_code = transfer_task($destination_host,$transfer_filelist,$error);
     201            }
     202            else {
     203                die("Failed too many times $error $destination_host $transfer_filelist");
     204            }
     205            $return_value = 1;
     206        }
     207    }
     208    return($return_value);
    151209}
    152210   
    … …  
    156214    my $error = shift;
    157215
    158     system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
    159     my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
     216    my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
     217    my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
     218    my $option = $server_options{$destination_host};
     219
     220    # get the list of files to be transferred
     221    # print "$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/\n";
     222    # system("$scp_cmd $option ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
     223    # my $cmd1 = "rsync -e '$ssh_cmd $option' -auv ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/";
     224    # print "$cmd1\n";
     225    # system ("$cmd1");
     226
     227    # do the transfer
     228
     229    # XXX EAM : fix this (use rsync not tar)
     230    # my $command = "rsync -e '$ssh_cmd ${option}' --files-from=/tmp/${transfer_filelist} ${destination_host}:/ ${local_raw}";
     231    my $command = "rsync -e '$ssh_cmd ${option}' --files-from=${local_tmp}/${transfer_filelist} ${destination_host}:/ ${local_raw}";
     232
     233    # my $command = "$ssh_cmd $option $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
    160234    print STDERR "$command\n";
    161235
    162     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     236    my $error_code = 0;
     237    my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) =
    163238        run(command => $command, verbose => $verbose);
    164239    unless ($success) {
    165         $error_code = (($error_code >> 8) or 4);
    166         warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
    167         $error++;
    168         if ($error < 4) {
    169             $error_code = fetch_task($destination_host,$transfer_filelist,$error);
    170         }
    171         else {
    172             die("Failed too many times $error $destination_host $transfer_filelist");
     240        print "raw error_msg: $error_msg...\n";
     241        ($error_code) = $error_msg =~ m/exited with value (\d+)/;
     242        warn("Transfer of files in $local_tmp/$transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
     243        print "*** stdout: *** \n";
     244        foreach $line (@$stdout_buf) {
     245            print STDERR "stdout: $line\n";
     246        }
     247        print "*** stderr: *** \n";
     248        foreach $line (@$stderr_buf) {
     249            print STDERR "stderr: $line\n";
     250        }
     251        if ($error_code == 23) {
     252            print STDERR "Some files failed to transfer\n";
     253        } else {
     254            $error++;
     255            if ($error < 4) {
     256                $error_code = fetch_task($destination_host,$transfer_filelist,$error);
     257            } else {
     258                print STDERR "Failed too many times $error $destination_host $transfer_filelist\n";
     259            }
    173260        }
    174261    }
Note: See TracChangeset for help on using the changeset viewer.