Changeset 32476

Timestamp:

Oct 3, 2011, 3:16:23 PM (15 years ago)

Author:

watersc1

Message:

Trunk merge of nebulous code. neb-shift: require an explicit volume. neb-stat: add validation code. neb-repair: automatically repair bad instances from other instances.

Location:

tags/ipp-20110622/Nebulous

Files:

• . (modified) (1 prop)
• Build.PL (modified) (1 diff)
• MANIFEST (modified) (1 diff)
• bin/neb-repair (copied) (copied from trunk/Nebulous/bin/neb-repair )
• bin/neb-shift (modified) (2 diffs)
• bin/neb-stat (modified) (5 diffs)

tags/ipp-20110622/Nebulous/Build.PL

@@ -116 +116 @@
         bin/neb-mv
         bin/neb-replicate
+        bin/neb-repair
         bin/neb-rm
         bin/neb-shift

tags/ipp-20110622/Nebulous/MANIFEST

@@ -17 +17 @@
 bin/neb-mv
 bin/neb-replicate
+bin/neb-repair
 bin/neb-rm
 bin/neb-shift

tags/ipp-20110622/Nebulous/bin/neb-shift

@@ -29 +29 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --server <source key> <source volume>",
+pod2usage( -msg => "Required options: --server --volume <source key> <source volume>",
         -exitval => 2 )
-        unless defined $server and defined $src and defined $abandon_vol;
+        unless defined $server and defined $src and defined $abandon_vol and defined $volume;
 
 my $neb = Nebulous::Client->new(
@@ -45 +45 @@
 }
 
+my $all_uris = $neb->find_instances($src);
+
 my $status;
-if ($volume) {
-    $status = $neb->replicate( $src, $volume);
+$status = $neb->replicate( $src, $volume);
+die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status;
+
+$uris = $neb->find_instances($src, "~$volume");
+unless (defined $uris) {
+    die "Replicate phase failed for $src ($abandon_vol) ($volume)";
 }
-else {
-    $status = $neb->replicate($src);
-}
-die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status;
 
 $status = $neb->cull($src,$abandon_vol);

tags/ipp-20110622/Nebulous/bin/neb-stat

@@ -16 +16 @@
 use Pod::Usage qw( pod2usage );
 
-my ($server);
+use Digest::MD5;
+use URI;
+
+my ($validate,$server,$nouris);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -22 +25 @@
 GetOptions(
     'server|s=s'     => \$server,
+    'validate|v'         => \$validate,
+    'files'          => \$nouris,
 ) || pod2usage( 2 );
 
@@ -44 +49 @@
 die "no instances found" unless $instances;   
 
+my $user_copies;
+eval {
+    $user_copies = $neb->getxattr($key, "user.copies");
+};
+unless(defined($user_copies)) {
+    $user_copies = 1;
+}
+
+my @validation;
+my %md5sum_uniq;
+my $existing_copies = 0;
+if (defined $validate) {
+    my @existance;
+    my @md5sums;
+    my @files = map {URI->new($_)->file if $_} @$instances;
+
+    for (my $i = 0; $i <= $#files; $i++) {
+        if (-e $files[$i]) {
+            $existance[$i] = 1;
+            $existing_copies++;
+            my $fh;
+            open($fh,$files[$i]);
+            $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest;
+            $md5sum_uniq{$md5sums[$i]} = 1;
+            close($fh);
+        }
+        else {
+            $existance[$i] = 0;
+            $md5sums[$i] = 'NON-EXISTANT';
+            $md5sum_uniq{$md5sums[$i]} = 1;
+        }
+        if (defined($nouris)) {
+            $validation[$i] = sprintf("% 3d %32s %s",
+                                      $existance[$i],
+                                      $md5sums[$i],
+                                      $files[$i]);
+        }
+        else {
+            $validation[$i] = sprintf("% 3d %32s %s",
+                                      $existance[$i],
+                                      $md5sums[$i],
+                                      ${ $instances }[$i]);
+        }
+    }
+}
+
 no warnings qw(uninitialized);
-print
-    "object id:             ", @$stat[0], "\n",
-    "key:                   ", @$stat[1], "\n",
-    "read lock:             ", @$stat[2], "\n",
-    "write lock:            ", @$stat[3], "\n",
-    "epoch:                 ", @$stat[4], "\n",
-    "mtime:                 ", @$stat[5], "\n",
-    "available instances:   ", @$stat[6], "\n",
-    "total instances:       ", @$stat[7], "\n",
-    "instance location:\n", " " x 4,
-    join("\n" . " " x 4, @$instances), "\n";
+if (defined $validate) {
+    print
+        "object id:             ", @$stat[0], "\n",
+        "key:                   ", @$stat[1], "\n",
+        "read lock:             ", @$stat[2], "\n",
+        "write lock:            ", @$stat[3], "\n",
+        "epoch:                 ", @$stat[4], "\n",
+        "mtime:                 ", @$stat[5], "\n",
+        "md5sum count:          ", scalar(keys %md5sum_uniq), "\n",
+        "requested instances:   ", $user_copies, "\n",
+        "available instances:   ", @$stat[6], "\n",
+        "existing instances:    ", $existing_copies, "\n",
+        "total instances:       ", @$stat[7], "\n",
+        
+        "instance location:\n", " " x 4,
+        join("\n" . " " x 4, @validation), "\n";
+}
+else {
+    print
+        "object id:             ", @$stat[0], "\n",
+        "key:                   ", @$stat[1], "\n",
+        "read lock:             ", @$stat[2], "\n",
+        "write lock:            ", @$stat[3], "\n",
+        "epoch:                 ", @$stat[4], "\n",
+        "mtime:                 ", @$stat[5], "\n",
+        "available instances:   ", @$stat[6], "\n",
+        "requested instances:   ", $user_copies, "\n",
+        "total instances:       ", @$stat[7], "\n",
+        
+        "instance location:\n", " " x 4,
+        join("\n" . " " x 4, @$instances), "\n";
+}
+
 
 __END__
@@ -67 +140 @@
 =head1 SYNOPSIS
 
-    neb-stat [--server <URL>] <key>
+    neb-stat [--server <URL>] [--validate] <key> 
 
 =head1 DESCRIPTION
@@ -82 +155 @@
 
 Optional if the appropriate environment variable is set.
+
+=item * --validate|-v
+
+Optional - prints out additional information including md5sum.
+
+=item * --files
+
+Optional - prints out disk files instead of uris during validate stage.
 
 =back