Index: trunk/Nebulous/bin/neb-cat =================================================================== --- trunk/Nebulous/bin/neb-cat (revision 40387) +++ trunk/Nebulous/bin/neb-cat (revision 41172) @@ -18,5 +18,5 @@ use constant CHUNK_SIZE => 1024; -my ($server); +my ($server, $help); $server = $ENV{'NEB_SERVER'} unless $server; @@ -24,8 +24,11 @@ GetOptions( 'server|s=s' => \$server, + 'help|h' => \$help, ) || pod2usage( 2 ); -pod2usage( -msg => "Required options: --server ", -exitval => 2 ) - unless defined $server; +# explicitly defined -h / --help: +if ($help) { pod2usage( -verbose => 1 ); } + +pod2usage( -msg => "Required options: --server ", -exitval => 2 ) unless defined $server; my $neb = Nebulous::Client->new( @@ -95,5 +98,5 @@ =over 4 -=item * --server|-s +=item --server|-s URL of the Nebulous server to connect to. Index: trunk/Nebulous/bin/neb-cp =================================================================== --- trunk/Nebulous/bin/neb-cp (revision 40387) +++ trunk/Nebulous/bin/neb-cp (revision 41172) @@ -16,5 +16,5 @@ use Pod::Usage qw( pod2usage ); -my ($volume, $server); +my ($volume, $server, $help); $server = $ENV{'NEB_SERVER'} unless $server; @@ -23,5 +23,9 @@ 'volume=s' => \$volume, 'server|s=s' => \$server, + 'help|h' => \$help, ) || pod2usage( 2 ); + +# explicitly defined -h / --help: +if ($help) { pod2usage( -verbose => 1 ); } my $src = shift; Index: trunk/Nebulous/bin/neb-cull =================================================================== --- trunk/Nebulous/bin/neb-cull (revision 40387) +++ trunk/Nebulous/bin/neb-cull (revision 41172) @@ -17,5 +17,5 @@ my ($min_copies, $server, $volume, $one_only); -my ($soft_volume); +my ($soft_volume, $help); $server = $ENV{'NEB_SERVER'} unless $server; @@ -26,5 +26,9 @@ 'one_only|o' => \$one_only, 'soft_volume' => \$soft_volume, + 'help|h' => \$help, ) || pod2usage( 2 ); + +# explicitly defined -h / --help: +if ($help) { pod2usage( -verbose => 1 ); } my $key = shift; @@ -59,7 +63,9 @@ # print "@cull_args\n"; - if ($one_only) { my $status = $neb->there_can_be_only_one(@cull_args); + if (not defined $status) { + die "not enough instances of Nebulous key: $key"; + } if ($status == 0) { die "not enough instances of Nebulous key: $key"; Index: trunk/Nebulous/bin/neb-insert =================================================================== --- trunk/Nebulous/bin/neb-insert (revision 40387) +++ trunk/Nebulous/bin/neb-insert (revision 41172) @@ -95,5 +95,5 @@ =head1 SYNOPSIS - neb-insert --server [--copies ] [--destroy] + neb-insert [--server ] [--copies ] [--destroy] #=head1 DESCRIPTION Index: trunk/Nebulous/bin/neb-ls =================================================================== --- trunk/Nebulous/bin/neb-ls (revision 40387) +++ trunk/Nebulous/bin/neb-ls (revision 41172) @@ -19,8 +19,8 @@ $ls, $column, + $help ); $server = $ENV{'NEB_SERVER'} unless $server; - GetOptions( @@ -29,5 +29,9 @@ 'l' => \$ls, 'column|c' => \$column, + 'help|h' => \$help, ) || pod2usage( 2 ); + +# explicitly defined -h / --help: +if ($help) { pod2usage( -verbose => 1 ); } # twiddle column so that it does it by default. @@ -35,9 +39,10 @@ $column = not $column; +if ($ls) { $path = 1; } + my $pattern = shift; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --server", -exitval => 2 ) - unless $server; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server; my $neb = Nebulous::Client->new( @@ -96,10 +101,10 @@ =head1 SYNOPSIS - neb-ls [--server ] [-c] [-p] + neb-ls [--server ] [-c] [-p] [-l] =head1 DESCRIPTION This program list Nebulous keys matched by C<>. Call it with no -arguments is equivalent to searching with the pattern C. SQL like +arguments is equivalent to searching with the pattern C. SQL-like wildcards are supported to select out certain files. @@ -108,15 +113,19 @@ =over 4 -=item * --path|-p +=item --path|-p Find the disk files corresponding to the keys found. -=item * --column|-c +=item --column|-c Disable column formatting, and output results one to a line. +=item --l|-l + +Give disk file information using ls (implies -p). + Optional -=item * --server|-s +=item --server|-s URL of the Nebulous server to connect to. Index: trunk/Nebulous/bin/neb-migrate =================================================================== --- trunk/Nebulous/bin/neb-migrate (revision 40387) +++ trunk/Nebulous/bin/neb-migrate (revision 41172) @@ -23,4 +23,6 @@ $server = $ENV{'NEB_SERVER'} unless $server; + +die "this tool is probably too inefficient unless used very carefully"; GetOptions( Index: trunk/Nebulous/bin/neb-repair =================================================================== --- trunk/Nebulous/bin/neb-repair (revision 40387) +++ trunk/Nebulous/bin/neb-repair (revision 41172) @@ -45,9 +45,10 @@ ); -die "can't connected to Nebulous Server: $server" +die "can't connect to Nebulous Server: $server" unless defined $neb; my $stat = $neb->stat($key); die "nebulous key: $key not found" unless $stat; + my $instances = $neb->find_instances($key, 'any'); die "no instances found" unless $instances; @@ -71,4 +72,5 @@ for (my $i = 0; $i <= $#files; $i++) { + # count existing files and get the MD5 sums if (-e $files[$i]) { $existance[$i] = 1; @@ -89,6 +91,5 @@ push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; $last_f = $files[$i]; - } - else { + } else { $existance[$i] = 0; $md5sums[$i] = 'NON-EXISTANT'; @@ -98,19 +99,23 @@ } +# check number of copies and number of unique MD5 sum values my $N_mds = scalar(keys(%md5sum_uniq)); my $N_files = scalar(@files); + +# seems like we should not call neb-replicate until *after* we have +# removed 0-byte copies if ($N_files < $user_copies) { print "Insufficient copies ($N_files) of $key, replicating!\n"; system("neb-replicate $key"); } + if ($N_mds == 1) { print "Nebulous key $key is consistent.\n"; -} -else { +} else { my $valid_sum = ''; my $valid_file = ''; my $m; foreach $m (keys %md5sum_valid) { - if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) { + if (($m ne 'd41d8cd98f00b204e9800998ecf8427e') && ($m ne 'NON-EXISTANT')) { if ($valid_sum ne '') { die "Too many acceptable files for key $key\n"; @@ -122,6 +127,5 @@ if ($valid_sum eq '') { die "No acceptable file for key $key\n"; - } - else { + } else { print "Repairing instances\n"; foreach $m (keys %md5sum_valid) { @@ -130,20 +134,26 @@ print "\tcp $valid_file $f\n"; system("cp $valid_file $f"); + + # if we failed to make a copy, the target ($f) could be inaccessible if ($? >> 8 != 0) { if ($existance[0] == 1) { - # We can fix this with culls and replications. + + # The target is a valid file 0-byte file + # but we cannot replace it with the + # correct file. We can fix this with + # culls and replications. + print "Attempting to fix this.\n"; print("neb-replicate $key\n"); system("neb-replicate $key"); if ($? >> 8 == 0) { + # if the neb-replicate succeeded, remove the bad copy with neb-cull my $bad_volume = (split /\//, $f)[2]; print("neb-cull --volume $bad_volume $key\n"); system("neb-cull --volume $bad_volume $key"); - } - else { + } else { die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n"; } - } - else { + } else { die "Failed to copy file $valid_file to $f!\n"; } @@ -187,5 +197,12 @@ =head1 DESCRIPTION -This program repairs instances and corrects instance counts for a nebulous key. +This program repairs instances and corrects instance counts for a nebulous key: + +If some of the existing instances have the MD5 sum corresponding to a +0-byte file, but the rest have the same, non-zero byte md5sum, then +replace the 0-byte files with the non-zero byte version. + +At the end, if the number of instances is less than the value of +user.copies, then additional copies are generated. =head1 OPTIONS Index: trunk/Nebulous/bin/neb-replicate =================================================================== --- trunk/Nebulous/bin/neb-replicate (revision 40387) +++ trunk/Nebulous/bin/neb-replicate (revision 41172) @@ -42,4 +42,7 @@ ); +# Nebulous::Client programs can use this to set the log level: +# $neb->set_log_level ('DEBUG'); + die "can't connected to Nebulous Server: $server" unless defined $neb; @@ -50,4 +53,5 @@ for (my $i = 0; $i < $copies; $i++) { if (defined $volume) { + # this is coded so that all copies end up on the specified volume which seems wrong $neb->replicate($key, $volume) or die "failed to replicate Nebulous key: $key"; Index: trunk/Nebulous/bin/neb-shift =================================================================== --- trunk/Nebulous/bin/neb-shift (revision 40387) +++ trunk/Nebulous/bin/neb-shift (revision 41172) @@ -17,4 +17,6 @@ my ($volume, $server); + +die "this tool may fail since find_instances will return an instance which is not on the hard volume"; $server = $ENV{'NEB_SERVER'} unless $server; Index: trunk/Nebulous/bin/neb-stat =================================================================== --- trunk/Nebulous/bin/neb-stat (revision 40387) +++ trunk/Nebulous/bin/neb-stat (revision 41172) @@ -19,21 +19,23 @@ use URI; -my ($validate,$server,$nouris); +my ($validate,$server,$nouris,$help); $server = $ENV{'NEB_SERVER'} unless $server; GetOptions( - 'server|s=s' => \$server, - 'validate|v' => \$validate, - 'files' => \$nouris, + 'server|s=s' => \$server, + 'validate|v' => \$validate, + 'files|f' => \$nouris, + 'help|h' => \$help, ) || pod2usage( 2 ); +# explicitly defined -h / --help: +if ($help) { pod2usage( -verbose => 1 ); } + my $key = shift; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --server", -exitval => 2 ) - unless $server; -pod2usage( -msg => "missing key", exitval => 2 ) - unless defined $key; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server; +pod2usage( -msg => "missing key", exitval => 2 ) unless defined $key; my $neb = Nebulous::Client->new( @@ -52,6 +54,7 @@ eval { $user_copies = $neb->getxattr($key, "user.copies"); + die ("user.copies not found") unless $user_copies; }; -unless(defined($user_copies)) { +unless(defined($user_copies) and $user_copies ne "") { $user_copies = 1; } @@ -74,10 +77,10 @@ $md5sum_uniq{$md5sums[$i]} = 1; close($fh); - } - else { + } else { $existance[$i] = 0; $md5sums[$i] = 'NON-EXISTANT'; $md5sum_uniq{$md5sums[$i]} = 1; } + if (defined($nouris)) { $validation[$i] = sprintf("% 3d %32s %s", @@ -85,6 +88,5 @@ $md5sums[$i], $files[$i]); - } - else { + } else { $validation[$i] = sprintf("% 3d %32s %s", $existance[$i], @@ -112,20 +114,39 @@ "instance location:\n", " " x 4, join("\n" . " " x 4, @validation), "\n"; +} else { + + if (defined($nouris)) { + my @files = map {URI->new($_)->file if $_} @$instances; + + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n", + "read lock: ", @$stat[2], "\n", + "write lock: ", @$stat[3], "\n", + "epoch: ", @$stat[4], "\n", + "mtime: ", @$stat[5], "\n", + "available instances: ", @$stat[6], "\n", + "requested instances: ", $user_copies, "\n", + "total instances: ", @$stat[7], "\n", + + "instance location:\n", " " x 4, + join("\n" . " " x 4, @files), "\n"; + } else { + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n", + "read lock: ", @$stat[2], "\n", + "write lock: ", @$stat[3], "\n", + "epoch: ", @$stat[4], "\n", + "mtime: ", @$stat[5], "\n", + "available instances: ", @$stat[6], "\n", + "requested instances: ", $user_copies, "\n", + "total instances: ", @$stat[7], "\n", + + "instance location:\n", " " x 4, + join("\n" . " " x 4, @$instances), "\n"; + } + } -else { - print - "object id: ", @$stat[0], "\n", - "key: ", @$stat[1], "\n", - "read lock: ", @$stat[2], "\n", - "write lock: ", @$stat[3], "\n", - "epoch: ", @$stat[4], "\n", - "mtime: ", @$stat[5], "\n", - "available instances: ", @$stat[6], "\n", - "requested instances: ", $user_copies, "\n", - "total instances: ", @$stat[7], "\n", - - "instance location:\n", " " x 4, - join("\n" . " " x 4, @$instances), "\n"; -} @@ -150,5 +171,5 @@ =over 4 -=item * --server|-s +=item --server|-s URL of the Nebulous server to connect to. @@ -156,9 +177,9 @@ Optional if the appropriate environment variable is set. -=item * --validate|-v +=item --validate|-v Optional - prints out additional information including md5sum. -=item * --files +=item --files Optional - prints out disk files instead of uris during validate stage. @@ -174,5 +195,5 @@ =over 4 -=item * C +=item C Equivalent to --server|-s