Index: /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server.pm	(revision 23570)
+++ /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server.pm	(revision 23571)
@@ -64,19 +64,33 @@
 }
 
-sub db
+# EAM : In the 'distributed' version of Nebulous, there is a collection of N databases
+# the db_index uniquely defines the db used by a given key
+sub db_index_for_key
 {
     my ($self, $key) = @_;
+
+    my $config  = $self->config;
+
+    my $db_index = 0;
+    die "key not defined" unless defined $key;
+
+    # hash the key to select the correct database instance
+    # only use the first 8 hex chars... have to be careful to avoid an int
+    # overflow here
+    # XXX this needs to be modified to follow my key prescription:
+    # neb://host/path/path/filename.ext -> use path/path/filename (ignore neb://host and .ext)
+    # EAM : test to double-check hashing behavior
+    $db_index = unpack("h8", sha1_hex("$key")) % $config->n_db;
+
+    return $db_index;
+}
+
+sub db_for_index
+{
+    my ($self, $db_index) = @_;
 
     my $log     = $self->log;
     my $sql     = $self->sql;
     my $config  = $self->config;
-
-    my $db_index = 0;
-    if (defined $key) {
-        # hash the key to select the correct database instance
-        # only use the first 8 hex chars... have to be careful to avoid an int
-        # overflow here
-        $db_index = unpack("h8", sha1_hex("$key")) % $config->n_db;
-    }
 
     # lookup to see if we have a stored dbh for this database
@@ -128,4 +142,20 @@
 }
 
+sub db
+{
+    my ($self, $key) = @_;
+
+    my $log     = $self->log;
+    my $sql     = $self->sql;
+    my $config  = $self->config;
+
+    die "key not defined" unless defined $key;
+    my $db_index = $self->db_index_for_key($key);
+
+    my $dbh = $self->db_for_index($db_index);
+    return $dbh;
+}
+
+# XXX need a function to loop over index values
 
 sub create_object
@@ -322,224 +352,121 @@
 }
 
-
-# sub swap_objects
-# {
-#     my $self = shift;
-# 
-#     my ($key1, $key2) = validate_pos(@_,
-#         {
-#             type        => SCALAR,
-#             callbacks   => {
-#                 'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
-#             },
-#         },
-#         {
-#             type        => SCALAR,
-#             callbacks   => {
-#                 'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
-#             },
-#         },
-#     );
-# 
-#     my $log  = $self->log;
-#     my $sql  = $self->sql;
-#     my $dbh1 = $self->db($key1);
-#     my $dbh2 = $self->db($key2);
-# 
-#     $log->debug("entered - @_");
-# 
-#     # ignore volumes
-#     $key1 = parse_neb_key($key1);
-#     $key2 = parse_neb_key($key2);
-# 
-#     # order of operations for the swap with a single db is:
-#     # key1 -> key1.swap
-#     # key2 -> key1
-#     # key1.swap -> key2
-# 
-#     # XXX this cmp will only work if ->db() returns the same exact (cached) dbh
-#     if ($dbh1 == $dbh2) {
-#         my $dbh = $dbh1;
-#         eval {
-#             {
-#                 # key1 -> key1.swap
-#                 my $query = $dbh->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key1->path . ".swap", $key1->path);
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             {
-#                 # key2 -> key1
-#                 my $query = $dbh->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key1->path, $key2->path);
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             {
-#                 # key1.swap -> key2
-#                 my $query = $dbh->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key2->path, $key1->path . ".swap");
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             $dbn->commit;
-#             $log->debug("commit");
-#         };
-#         if ($@) {
-#             $dbh->rollback;
-#             $log->debug("rollback");
-#             $log->logdie("database error: $@");
-#         }
-#     }
-# 
-#     # order of operations for the swap between two dbs is:
-#     # key1 start transaction
-#     # key1 -> read all instances
-#     # key1 -> remove all instances
-#     # key2 start transaction
-#     # key2 -> read all instances
-#     # key2 -> remove all instances
-#     # key1 -> insert key 2 instances
-#     # key2 -> insert key 1 instances
-#     # key1,2 commit
-# 
-#     eval {
-#         {
-#             # key1 -> read all instances
-#             my $query = $dbh->prepare_cached($sql->rename_object); 
-#             # this SQL statment takes the new key name as the first param
-#             my $rows = $query->execute($key1->path . ".swap", $key1->path);
-# 
-#             # if we affected more then one row something very bad has happened.
-#             unless ($rows == 1) {
-#                 $query->finish;
-#                 $log->logdie("affected row count is $rows instead of 1");
-#             }
-#         }
-# 
-#         {
-#             # key2 -> key1
-#             my $query = $db->prepare_cached($sql->rename_object); 
-#             # this SQL statment takes the new key name as the first param
-#             my $rows = $query->execute($key1->path, $key2->path);
-# 
-#             # if we affected more then one row something very bad has happened.
-#             unless ($rows == 1) {
-#                 $query->finish;
-#                 $log->logdie("affected row count is $rows instead of 1");
-#             }
-#         }
-# 
-#         {
-#             # key1.swap -> key2
-#             my $query = $db->prepare_cached($sql->rename_object); 
-#             # this SQL statment takes the new key name as the first param
-#             my $rows = $query->execute($key2->path, $key1->path . ".swap");
-# 
-#             # if we affected more then one row something very bad has happened.
-#             unless ($rows == 1) {
-#                 $query->finish;
-#                 $log->logdie("affected row count is $rows instead of 1");
-#             }
-#         }
-# 
-#         $db->commit;
-#         $log->debug("commit");
-#     };
-#         if ($@) {
-#             $db->rollback;
-#             $log->debug("rollback");
-#             $log->logdie("database error: $@");
-#         }
-# 
-# 
-#     $log->debug("leaving");
-# 
-#     return 1;
-# }
-# 
-
-# ### this block below comes from the single branch, and implements
-#      the transactions in the new way (two passes?).  the above handles
-#      the case of multiple dbs
-# 
-# TRANS: while (1) {
-#         eval {
-#             {
-#                 # key1 -> key1.swap
-#                 my $query = $db->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key1->path . ".swap", $key1->path);
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             {
-#                 # key2 -> key1
-#                 my $query = $db->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key1->path, $key2->path);
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             {
-#                 # key1.swap -> key2
-#                 my $query = $db->prepare_cached($sql->rename_object); 
-#                 # this SQL statment takes the new key name as the first param
-#                 my $rows = $query->execute($key2->path, $key1->path . ".swap");
-# 
-#                 # if we affected more then one row something very bad has happened.
-#                 unless ($rows == 1) {
-#                     $query->finish;
-#                     $log->logdie("affected row count is $rows instead of 1");
-#                 }
-#             }
-# 
-#             $db->commit;
-#             $log->debug("commit");
-#         };
-#         if ($@) {
-#             $db->rollback;
-#             $log->debug("rollback");
-#             if ($@ =~ /Deadlock found/) {
-#                 $log->warn("database deadlock retrying transaction: $@");
-#                 redo TRANS;
-#             }
-#             $log->logdie("database error: $@");
-#         }
-#         last;
-#     }
-# 
-#     $log->debug("leaving");
-# 
-#     return 1;
-# }
+sub swap_objects
+{
+    my $self = shift;
+
+    my ($key1, $key2) = validate_pos(@_,
+				     {
+					 type        => SCALAR,
+					 callbacks   => {
+					     'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+					 },
+				     },
+				     {
+					 type        => SCALAR,
+					 callbacks   => {
+					     'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+					 },
+				     },
+	);
+
+    my $log  = $self->log;
+    my $sql  = $self->sql;
+
+    my $dbidx1 = $self->db_index_for_key($key1);
+    my $dbidx2 = $self->db_index_for_key($key2);
+    die "cannot swap keys not stored on the same database" unless ($dbidx1 == $dbidx2);
+
+    my $dbh1 = $self->db_for_index($dbidx1);
+    my $dbh2 = $self->db_for_index($dbidx2);
+    die "different db handles for the same db?" unless ($dbh1 == $dbh2);
+
+    $log->debug("entered - @_");
+
+    # ignore volumes
+    $key1 = parse_neb_key($key1);
+    $key2 = parse_neb_key($key2);
+
+    # order of operations for the swap with a single db is:
+    # key1 -> key1.swap
+    # key2 -> key1
+    # key1.swap -> key2
+
+    my $dbh = $dbh1;
+  TRANS: while (1) {
+      eval {
+	  {
+	      # key1 -> key1.swap
+	      my $query = $db->prepare_cached($sql->rename_object); 
+	      # this SQL statment takes the new key name as the first param
+	      my $rows = $query->execute($key1->path . ".swap", $key1->path);
+
+	      # if we affected more then one row something very bad has happened.
+	      unless ($rows == 1) {
+		  $query->finish;
+		  $log->logdie("affected row count is $rows instead of 1");
+	      }
+	  }
+
+	  {
+	      # key2 -> key1
+	      my $query = $db->prepare_cached($sql->rename_object); 
+	      # this SQL statment takes the new key name as the first param
+	      my $rows = $query->execute($key1->path, $key2->path);
+
+	      # if we affected more then one row something very bad has happened.
+	      unless ($rows == 1) {
+		  $query->finish;
+		  $log->logdie("affected row count is $rows instead of 1");
+	      }
+	  }
+
+	  {
+	      # key1.swap -> key2
+	      my $query = $db->prepare_cached($sql->rename_object); 
+	      # this SQL statment takes the new key name as the first param
+	      my $rows = $query->execute($key2->path, $key1->path . ".swap");
+
+	      # if we affected more then one row something very bad has happened.
+	      unless ($rows == 1) {
+		  $query->finish;
+		  $log->logdie("affected row count is $rows instead of 1");
+	      }
+	  }
+
+	  $db->commit;
+	  $log->debug("commit");
+      };
+      if ($@) {
+	  $db->rollback;
+	  $log->debug("rollback");
+	  if ($@ =~ /Deadlock found/) {
+	      $log->warn("database deadlock retrying transaction: $@");
+	      redo TRANS;
+	  }
+	  $log->logdie("database error: $@");
+      }
+      last;
+  }
+
+    $log->debug("leaving");
+
+    return 1;
+}
+
+# EAM : from JH, below is a possible option to swap instances across
+# dbs.  I recommend that this action be disallowed, and instead such
+# operations be implemented only as a combination of copy and delete
+
+# order of operations for the swap between two dbs is:
+# key1 start transaction
+# key1 -> read all instances
+# key1 -> remove all instances
+# key2 start transaction
+# key2 -> read all instances
+# key2 -> remove all instances
+# key1 -> insert key 2 instances
+# key2 -> insert key 1 instances
+# key1,2 commit
 
 sub replicate_object
@@ -644,4 +571,6 @@
 
             # TODO add some stuff here to retry if unsucessful
+	    # XXX if this fails, it should try to generate the
+	    # instance on another volume (unless !$soft_volume) 
             $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
 
@@ -1137,9 +1066,7 @@
 }
 
-
+# loop over all db_index values, passing db_index to each call
 sub find_objects
 {
-    # XXX: this will only search one db
-
     my $self = shift;
 
@@ -1153,5 +1080,4 @@
     my $log = $self->log;
     my $sql = $self->sql;
-    my $db  = $self->db;
 
     $log->debug( "entered - @_" );
@@ -1164,4 +1090,27 @@
     # attempt to strip off neb:// if it exists
     $pattern =~ s|^neb:||;
+
+    my @keys = ();
+    my $n_dbs = $self->n_db();
+    for (my $index = 0; $index < $n_dbs; $index ++) {
+	
+	my $newkeys = $self->find_objects_for_index($index, $pattern);
+	push @keys, @$newkeys;
+    }
+    $log->logdie("no keys found") unless ( scalar @keys );
+
+    $log->debug( "leaving" );
+
+    return \@keys;
+}
+
+# find matching objects from the given server
+sub find_objects_for_index
+{
+
+    my $self = shift;
+    my $index = shift;
+    my $pattern = shift;
+    my $db  = $self->db_for_index($index);
 
     my @keys;
@@ -1177,11 +1126,6 @@
     $log->logdie("database error: $@") if $@;
 
-    $log->logdie("no keys found") unless ( scalar @keys );
-
-    $log->debug( "leaving" );
-
     return \@keys;
 }
-
 
 sub find_instances
@@ -1416,8 +1360,10 @@
 }
 
-
+# this should have a 'db_index' as an argument
 sub mounts
 {
     # XXX: this will only pull the mounts from one db
+    # XXX: loop over db_index and generate a single unique list
+    # XXX: or report mount info for each db server
     my $self = shift;
 
@@ -1426,5 +1372,5 @@
     my $log = $self->log;
     my $sql = $self->sql;
-    my $db  = $self->db;
+    my $db  = $self->db_for_index(0); # XXX fix as above
 
     $log->debug("entered - @_");
@@ -1772,4 +1718,5 @@
     $log->debug( "entered" );
 
+# XXX do we need to loop over db_index?
 #    $self->db->disconnect;