IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41172 for trunk/Nebulous/bin


Ignore:
Timestamp:
Nov 27, 2019, 12:04:14 PM (7 years ago)
Author:
eugene
Message:

merging updates to Nebulous from EAM dev branch ipp-20191011

Location:
trunk/Nebulous/bin
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/Nebulous/bin/neb-cat

    r13080 r41172  
    1818use constant CHUNK_SIZE => 1024;
    1919
    20 my ($server);
     20my ($server, $help);
    2121
    2222$server = $ENV{'NEB_SERVER'} unless $server;
    … …  
    2424GetOptions(
    2525    'server|s=s'    => \$server,
     26    'help|h'        => \$help,
    2627) || pod2usage( 2 );
    2728
    28 pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 )
    29         unless defined $server;
     29# explicitly defined -h / --help:
     30if ($help) { pod2usage( -verbose => 1 ); }
     31
     32pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 ) unless defined $server;
    3033
    3134my $neb = Nebulous::Client->new(
    … …  
    9598=over 4
    9699
    97 =item * --server|-s <URL>
     100=item --server|-s <URL>
    98101
    99102URL of the Nebulous server to connect to.

  • trunk/Nebulous/bin/neb-cp

    r16182 r41172  
    1616use Pod::Usage qw( pod2usage );
    1717
    18 my ($volume, $server);
     18my ($volume, $server, $help);
    1919
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    … …  
    2323    'volume=s'      => \$volume,
    2424    'server|s=s'    => \$server,
     25    'help|h'        => \$help,
    2526) || pod2usage( 2 );
     27
     28# explicitly defined -h / --help:
     29if ($help) { pod2usage( -verbose => 1 ); }
    2630
    2731my $src = shift;

  • trunk/Nebulous/bin/neb-cull

    r30236 r41172  
    1717
    1818my ($min_copies, $server, $volume, $one_only);
    19 my ($soft_volume);
     19my ($soft_volume, $help);
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    2121
    … …  
    2626    'one_only|o'    => \$one_only,
    2727    'soft_volume'   => \$soft_volume,
     28    'help|h'        => \$help,
    2829) || pod2usage( 2 );
     30
     31# explicitly defined -h / --help:
     32if ($help) { pod2usage( -verbose => 1 ); }
    2933
    3034my $key = shift;
    … …  
    5963# print "@cull_args\n";
    6064
    61 
    6265if ($one_only) {
    6366    my $status = $neb->there_can_be_only_one(@cull_args);
     67    if (not defined $status) {
     68        die "not enough instances of Nebulous key: $key";
     69    }
    6470    if ($status == 0) {
    6571        die "not enough instances of Nebulous key: $key";

  • trunk/Nebulous/bin/neb-insert

    r25939 r41172  
    9595=head1 SYNOPSIS
    9696
    97     neb-insert --server <neb-server> <neb-key> <local-file> [--copies <n>] [--destroy]
     97    neb-insert [--server <neb-server>] <neb-key> <local-file> [--copies <n>] [--destroy]
    9898
    9999#=head1 DESCRIPTION

  • trunk/Nebulous/bin/neb-ls

    r28581 r41172  
    1919    $ls,
    2020    $column,
     21    $help
    2122);
    2223
    2324$server = $ENV{'NEB_SERVER'} unless $server;
    24 
    2525
    2626GetOptions(
    … …  
    2929    'l'             => \$ls,
    3030    'column|c'      => \$column,
     31    'help|h'        => \$help,
    3132) || pod2usage( 2 );
     33
     34# explicitly defined -h / --help:
     35if ($help) { pod2usage( -verbose => 1 ); }
    3236
    3337# twiddle column so that it does it by default.
    … …  
    3539$column = not $column;
    3640
     41if ($ls) { $path = 1; }
     42
    3743my $pattern = shift;
    3844
    3945pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    40 pod2usage( -msg => "Required options: --server", -exitval => 2 )
    41     unless $server;
     46pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
    4247
    4348my $neb = Nebulous::Client->new(
    … …  
    96101=head1 SYNOPSIS
    97102
    98     neb-ls [--server <URL>] [-c] [-p] <pattern>
     103    neb-ls [--server <URL>] [-c] [-p] [-l] <pattern>
    99104
    100105=head1 DESCRIPTION
    101106
    102107This program list Nebulous keys matched by C<<pattern>>.  Call it with no
    103 arguments is equivalent to searching with the pattern C<neb://>.  SQL like
     108arguments is equivalent to searching with the pattern C<neb://>.  SQL-like
    104109wildcards are supported to select out certain files.
    105110
    … …  
    108113=over 4
    109114
    110 =item * --path|-p
     115=item --path|-p
    111116
    112117Find the disk files corresponding to the keys found.
    113118
    114 =item * --column|-c
     119=item --column|-c
    115120
    116121Disable column formatting, and output results one to a line.
    117122
     123=item --l|-l
     124
     125Give disk file information using ls (implies -p).
     126
    118127Optional
    119128
    120 =item * --server|-s <URL>
     129=item --server|-s <URL>
    121130
    122131URL of the Nebulous server to connect to.

  • trunk/Nebulous/bin/neb-migrate

    r28446 r41172  
    2323
    2424$server = $ENV{'NEB_SERVER'} unless $server;
     25
     26die "this tool is probably too inefficient unless used very carefully";
    2527
    2628GetOptions(

  • trunk/Nebulous/bin/neb-repair

    r39926 r41172  
    4545);
    4646
    47 die "can't connected to Nebulous Server: $server"
     47die "can't connect to Nebulous Server: $server"
    4848    unless defined $neb;
    4949
    5050my $stat = $neb->stat($key);
    5151die "nebulous key: $key not found" unless $stat;   
     52
    5253my $instances = $neb->find_instances($key, 'any');
    5354die "no instances found" unless $instances;   
    … …  
    7172
    7273for (my $i = 0; $i <= $#files; $i++) {
     74    # count existing files and get the MD5 sums
    7375    if (-e $files[$i]) {
    7476        $existance[$i] = 1;
    … …  
    8991        push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i];
    9092        $last_f = $files[$i];
    91     }
    92     else {
     93    } else {
    9394        $existance[$i] = 0;
    9495        $md5sums[$i] = 'NON-EXISTANT';
    … …  
    9899}
    99100
     101# check number of copies and number of unique MD5 sum values
    100102my $N_mds = scalar(keys(%md5sum_uniq));
    101103my $N_files = scalar(@files);
     104
     105# seems like we should not call neb-replicate until *after* we have
     106# removed 0-byte copies
    102107if ($N_files < $user_copies) {
    103108    print "Insufficient copies ($N_files) of $key, replicating!\n";
    104109    system("neb-replicate $key");
    105110}
     111
    106112if ($N_mds == 1) {
    107113    print "Nebulous key $key is consistent.\n";
    108 }
    109 else {
     114} else {
    110115    my $valid_sum = '';
    111116    my $valid_file = '';
    112117    my $m;
    113118    foreach $m (keys %md5sum_valid) {
    114         if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) {
     119        if (($m ne 'd41d8cd98f00b204e9800998ecf8427e') && ($m ne 'NON-EXISTANT')) {
    115120            if ($valid_sum ne '') {
    116121                die "Too many acceptable files for key $key\n";
    … …  
    122127    if ($valid_sum eq '') {
    123128        die "No acceptable file for key $key\n";
    124     }
    125     else {
     129    } else {
    126130        print "Repairing instances\n";
    127131        foreach $m (keys %md5sum_valid) {
    … …  
    130134                    print "\tcp $valid_file $f\n";
    131135                    system("cp $valid_file $f");
     136
     137                    # if we failed to make a copy, the target ($f) could be inaccessible
    132138                    if ($? >> 8 != 0) {
    133139                        if ($existance[0] == 1) {
    134                             # We can fix this with culls and replications.
     140
     141                            # The target is a valid file 0-byte file
     142                            # but we cannot replace it with the
     143                            # correct file.  We can fix this with
     144                            # culls and replications.
     145
    135146                            print "Attempting to fix this.\n";
    136147                            print("neb-replicate $key\n");
    137148                            system("neb-replicate $key");
    138149                            if ($? >> 8 == 0) {
     150                                # if the neb-replicate succeeded, remove the bad copy with neb-cull
    139151                                my $bad_volume = (split /\//, $f)[2];
    140152                                print("neb-cull --volume $bad_volume $key\n");
    141153                                system("neb-cull --volume $bad_volume $key");
    142                             }
    143                             else {
     154                            } else {
    144155                                die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n";
    145156                            }
    146                         }
    147                         else {
     157                        } else {
    148158                            die "Failed to copy file $valid_file to $f!\n";
    149159                        }
    … …  
    187197=head1 DESCRIPTION
    188198
    189 This program repairs instances and corrects instance counts for a nebulous key. 
     199This program repairs instances and corrects instance counts for a nebulous key:
     200
     201If some of the existing instances have the MD5 sum corresponding to a
     2020-byte file, but the rest have the same, non-zero byte md5sum, then
     203replace the 0-byte files with the non-zero byte version.
     204
     205At the end, if the number of instances is less than the value of
     206user.copies, then additional copies are generated.
    190207
    191208=head1 OPTIONS

  • trunk/Nebulous/bin/neb-replicate

    r29476 r41172  
    4242);
    4343
     44# Nebulous::Client programs can use this to set the log level:
     45# $neb->set_log_level ('DEBUG');
     46
    4447die "can't connected to Nebulous Server: $server"
    4548    unless defined $neb;
    … …  
    5053    for (my $i = 0; $i < $copies; $i++) {
    5154        if (defined $volume) {
     55            # this is coded so that all copies end up on the specified volume which seems wrong
    5256            $neb->replicate($key, $volume)
    5357                or die "failed to replicate Nebulous key: $key";

  • trunk/Nebulous/bin/neb-shift

    r31964 r41172  
    1717
    1818my ($volume, $server);
     19
     20die "this tool may fail since find_instances will return an instance which is not on the hard volume";
    1921
    2022$server = $ENV{'NEB_SERVER'} unless $server;

  • trunk/Nebulous/bin/neb-stat

    r32045 r41172  
    1919use URI;
    2020
    21 my ($validate,$server,$nouris);
     21my ($validate,$server,$nouris,$help);
    2222
    2323$server = $ENV{'NEB_SERVER'} unless $server;
    2424
    2525GetOptions(
    26     'server|s=s'     => \$server,
    27     'validate|v'         => \$validate,
    28     'files'          => \$nouris,
     26    'server|s=s'    => \$server,
     27    'validate|v'    => \$validate,
     28    'files|f'       => \$nouris,
     29    'help|h'        => \$help,
    2930) || pod2usage( 2 );
    3031
     32# explicitly defined -h / --help:
     33if ($help) { pod2usage( -verbose => 1 ); }
     34
    3135my $key = shift;
    3236
    3337pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    34 pod2usage( -msg => "Required options: --server", -exitval => 2 )
    35     unless $server;
    36 pod2usage( -msg => "missing key", exitval => 2 )
    37     unless defined $key;
     38pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
     39pod2usage( -msg => "missing key", exitval => 2 ) unless defined $key;
    3840
    3941my $neb = Nebulous::Client->new(
    … …  
    5254eval {
    5355    $user_copies = $neb->getxattr($key, "user.copies");
     56    die ("user.copies not found") unless $user_copies;
    5457};
    55 unless(defined($user_copies)) {
     58unless(defined($user_copies) and $user_copies ne "") {
    5659    $user_copies = 1;
    5760}
    … …  
    7477            $md5sum_uniq{$md5sums[$i]} = 1;
    7578            close($fh);
    76         }
    77         else {
     79        } else {
    7880            $existance[$i] = 0;
    7981            $md5sums[$i] = 'NON-EXISTANT';
    8082            $md5sum_uniq{$md5sums[$i]} = 1;
    8183        }
     84
    8285        if (defined($nouris)) {
    8386            $validation[$i] = sprintf("% 3d %32s %s",
    … …  
    8588                                      $md5sums[$i],
    8689                                      $files[$i]);
    87         }
    88         else {
     90        } else {
    8991            $validation[$i] = sprintf("% 3d %32s %s",
    9092                                      $existance[$i],
    … …  
    112114        "instance location:\n", " " x 4,
    113115        join("\n" . " " x 4, @validation), "\n";
     116} else {
     117   
     118    if (defined($nouris)) {
     119        my @files = map {URI->new($_)->file if $_} @$instances;
     120
     121        print
     122            "object id:             ", @$stat[0], "\n",
     123            "key:                   ", @$stat[1], "\n",
     124            "read lock:             ", @$stat[2], "\n",
     125            "write lock:            ", @$stat[3], "\n",
     126            "epoch:                 ", @$stat[4], "\n",
     127            "mtime:                 ", @$stat[5], "\n",
     128            "available instances:   ", @$stat[6], "\n",
     129            "requested instances:   ", $user_copies, "\n",
     130            "total instances:       ", @$stat[7], "\n",
     131           
     132            "instance location:\n", " " x 4,
     133            join("\n" . " " x 4, @files), "\n";
     134    } else {
     135        print
     136            "object id:             ", @$stat[0], "\n",
     137            "key:                   ", @$stat[1], "\n",
     138            "read lock:             ", @$stat[2], "\n",
     139            "write lock:            ", @$stat[3], "\n",
     140            "epoch:                 ", @$stat[4], "\n",
     141            "mtime:                 ", @$stat[5], "\n",
     142            "available instances:   ", @$stat[6], "\n",
     143            "requested instances:   ", $user_copies, "\n",
     144            "total instances:       ", @$stat[7], "\n",
     145           
     146            "instance location:\n", " " x 4,
     147            join("\n" . " " x 4, @$instances), "\n";
     148    }
     149
    114150}
    115 else {
    116     print
    117         "object id:             ", @$stat[0], "\n",
    118         "key:                   ", @$stat[1], "\n",
    119         "read lock:             ", @$stat[2], "\n",
    120         "write lock:            ", @$stat[3], "\n",
    121         "epoch:                 ", @$stat[4], "\n",
    122         "mtime:                 ", @$stat[5], "\n",
    123         "available instances:   ", @$stat[6], "\n",
    124         "requested instances:   ", $user_copies, "\n",
    125         "total instances:       ", @$stat[7], "\n",
    126        
    127         "instance location:\n", " " x 4,
    128         join("\n" . " " x 4, @$instances), "\n";
    129 }
    130151
    131152
    … …  
    150171=over 4
    151172
    152 =item * --server|-s <URL>
     173=item --server|-s <URL>
    153174
    154175URL of the Nebulous server to connect to.
    … …  
    156177Optional if the appropriate environment variable is set.
    157178
    158 =item * --validate|-v
     179=item --validate|-v
    159180
    160181Optional - prints out additional information including md5sum.
    161182
    162 =item * --files
     183=item --files
    163184
    164185Optional - prints out disk files instead of uris during validate stage.
    … …  
    174195=over 4
    175196
    176 =item * C<NEB_SERVER>
     197=item C<NEB_SERVER>
    177198
    178199Equivalent to --server|-s
Note: See TracChangeset for help on using the changeset viewer.