Index: trunk/Nebulous-Server/Build.PL
===================================================================
--- trunk/Nebulous-Server/Build.PL	(revision 40618)
+++ trunk/Nebulous-Server/Build.PL	(revision 41172)
@@ -43,6 +43,4 @@
     },
     script_files        => [qw(
-        bin/neb-admin
-        bin/neb-fsck
         bin/neb-cabadd
         bin/neb-voladd
@@ -54,2 +52,6 @@
     )],
 )->create_build_script;
+
+# these scripts are no longer installed (2019.10.31)
+#       bin/neb-admin
+#       bin/neb-fsck
Index: trunk/Nebulous-Server/MANIFEST
===================================================================
--- trunk/Nebulous-Server/MANIFEST	(revision 40618)
+++ trunk/Nebulous-Server/MANIFEST	(revision 41172)
@@ -4,7 +4,4 @@
 MANIFEST
 README
-bin/neb-admin
-bin/neb-fsck
-bin/neb-initdb
 bin/neb-voladd
 bin/neb-cabadd
Index: trunk/Nebulous-Server/bin/neb-voladd
===================================================================
--- trunk/Nebulous-Server/bin/neb-voladd	(revision 40618)
+++ trunk/Nebulous-Server/bin/neb-voladd	(revision 41172)
@@ -43,4 +43,10 @@
 
 my $dbh = DBI->connect(
+    # if you want to work with the localhost database but get socket path errors,
+    # you should set the server host as 127.0.0.1 (not localhost)
+
+    # the following tells DBI to read the mysql/my.cnf files
+    #  "DBI:mysql:database=$db:host=$dbhost:mysql_read_default_group=dbd_mysql",
+
     "DBI:mysql:database=$db:host=$dbhost",
     $dbuser,
Index: trunk/Nebulous-Server/docs/apache2-mod_perl-startup.pl
===================================================================
--- trunk/Nebulous-Server/docs/apache2-mod_perl-startup.pl	(revision 40618)
+++ trunk/Nebulous-Server/docs/apache2-mod_perl-startup.pl	(revision 41172)
@@ -28,17 +28,24 @@
 use Nebulous::Server;
 
-my $dsn         = 'DBI:mysql:database=nebulous:host=ipp019.ifa.hawaii.edu';
+my $dsn         = 'DBI:mysql:database=nebulous:host=nebulous.ipp.ifa.hawaii.edu';
 my $dbuser      = 'nebulous';
-my $dbpasswd    = '@neb@';
+my $dbpasswd    = 'XXXX';
 
 #$Apache::DBI::DEBUG = 1;
 #Apache::DBI->connect_on_init( $dsn, $dbuser, $dbpasswd );
 Apache::DBI->setPingTimeOut($dsn, 10);
-Nebulous::Server::SOAP->new_on_init(
+
+my $config = Nebulous::Server::Config->new(
+    trace       => 'warn',
+					   );
+$config->add_db(
+    dbindex     => 0,
     dsn         => $dsn,
     dbuser      => $dbuser,
     dbpasswd    => $dbpasswd,
-    log_level   => 'all',
-);
+		);
+
+Nebulous::Server::SOAP->new_on_init($config);
+
 
 1;
Index: trunk/Nebulous-Server/docs/install.txt
===================================================================
--- trunk/Nebulous-Server/docs/install.txt	(revision 40618)
+++ trunk/Nebulous-Server/docs/install.txt	(revision 41172)
@@ -55,11 +55,12 @@
     export NEB_PASS=@neb@
 
+    *** this is very dangerous and should only be run in test examples:
     neb-initdb
 
-    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous
-    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous
-    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous
-    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous
-    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous
+    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous --mountpoint /data/ipp000.0/nebulous
+    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous --mountpoint /data/ipp000.1/nebulous
+    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous --mountpoint /data/ipp002.0/nebulous
+    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous --mountpoint /data/ipp003.0/nebulous
+    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous --mountpoint /data/ipp022.0/nebulous
 
     # or in bulk:
Index: trunk/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 40618)
+++ trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 41172)
@@ -13,7 +13,15 @@
 use base qw( Class::Accessor::Fast );
 
+use Carp;
+
 use Cache::Memcached;
 use DBI;
+
+## for gentoo:
 use Digest::SHA1 qw( sha1_hex );
+
+## for ubuntu:
+# use Digest::SHA qw( sha1_hex );
+
 use Fcntl ':mode';
 use File::Basename qw( basename dirname fileparse );
@@ -77,6 +85,6 @@
         Cache::Memcached->new({
 	    servers => $config->memcached_servers,
-       })
-    );
+			      })
+	);
 #    $self->cache->set("foo", "bar") or die "set failed";
 #    $self->cache->get("foo") or die "get failed";
@@ -215,4 +223,6 @@
 }
 
+my $get_storage_volume_calls = 0;
+
 sub create_object
 {
@@ -252,6 +262,5 @@
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
-    if (defined $vol_name
-        and not $self->_is_valid_volume_name($key, $key->volume)) {
+    if (defined $vol_name and not $self->_is_valid_volume_name($key, $key->volume)) {
         unless ($key->hard_volume) {
             $log->warn( "$vol_name is not a known volume name" );
@@ -262,4 +271,6 @@
     }
         
+    $get_storage_volume_calls = 0;
+
     my ($vol_id, $vol_host, $vol_path, $vol_xattr)
         = $self->_get_storage_volume($key, $vol_name, $key->hard_volume);
@@ -273,5 +284,7 @@
                 # create storage_object
                 my $query = $db->prepare_cached( $sql->new_object ); 
-                $query->execute('NULL', $key->path, basename($key->path), $parent_id);
+# bad syntax    $query->execute('NULL', $key->path, basename($key->path), $parent_id);
+                $query->execute(0, $key->path, basename($key->path), $parent_id);
+                $query->finish;
             }
 
@@ -292,4 +305,5 @@
                 my $query = $db->prepare_cached( $sql->new_object_attr ); 
                 $query->execute($so_id);
+                $query->finish;
             }
 
@@ -300,4 +314,5 @@
                 my $query = $db->prepare_cached( $sql->new_object_instance );
                 $query->execute($vol_id);
+                $query->finish;
             }
 
@@ -327,4 +342,5 @@
                 # vol_id, uri, ins_id
                 $query->execute($vol_id, "$uri", $ins_id);
+                $query->finish;
             }
 
@@ -421,4 +437,5 @@
                     }
                     $query->finish;
+		    $db->commit;
                 }
 
@@ -441,4 +458,6 @@
                     my $query = $db->prepare_cached($sql->new_directory);
                     $query->execute($dir, $parent_id);
+                    $query->finish;
+		    $db->commit;
                 }
 
@@ -452,9 +471,8 @@
                     ($parent_id) = $query->fetchrow_array;
                     $query->finish;
+		    $db->commit;
                 }
                 die("failed to get LAST_INSERT_ID()")
                     unless $parent_id;
-
-                $db->commit;
             }
         };
@@ -712,11 +730,4 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -726,15 +737,20 @@
 
     # if a volume name is explicity specified then we should make the
-    # replication onto that volume (even if there is alread an instance on that
+    # replication onto that volume (even if there is already an instance on that
     # volume) if at all possible and throw an error if we can not.
+
     # if a volume name IS NOT specified then we should make the replication
     # onto any (hopefully the best) volume that DOES NOT already have an
-    # instance on it.  If all avilable volume already have an instance on them
+    # instance on it.  If all available volumes already have an instance on them
     # then we should throw an error 
+
     # volume names implied as part of the key are *IGNORED* as the source and
     # *SHOULD NOT* be used as the destination either
 
     eval {
-        $key = parse_neb_key($key);
+	# EAM 2019.10.31 : if I supply dest_vol_name here, then
+	# $key->volume will point at that volume and the section below
+	# will correctly be testing the validity of dest_vol_name
+        $key = parse_neb_key($key, $dest_vol_name);
     };
     $log->logdie("$@") if $@;
@@ -742,8 +758,14 @@
     my $db  = $self->db($key);
 
+    ## Old comment:
     # puke if the source volume is bogus, we may want to actually use this as
     # the instance to be copied later
-    if (defined $key->volume
-        and not $self->_is_valid_volume_name($key, $key->volume)) {
+
+    # EAM 2019.10.31 : in the past, the key-implied volume was
+    # supplied here, which was inconsistent since above it says the
+    # key-implied volume is ignored. By supplying $dest_vol_name to
+    # parse_neb_key above, the key->volume now refers to the
+    # dest_volume
+    if (defined $key->volume and not $self->_is_valid_volume_name($key, $key->volume)) {
         unless ($key->hard_volume) {
             $log->warn($key->volume . " not a known volume name");
@@ -752,19 +774,26 @@
         }
     }
-    # puke if the source volume is bogus, we may want to actually use this as
-    # the instance to be copied later
-    if (defined $dest_vol_name
-        and not $self->_is_valid_volume_name($key, $dest_vol_name)) {
-           $log->logdie($key->volume . " is not a valid volume name");
-    }
-        
+
+    # puke if the destination volume is bogus
+    ## XXX if I supply dest_vol_name to parse_neb_key above, this section below is redundant with the above
+    ## if (defined $dest_vol_name
+    ##     and not $self->_is_valid_volume_name($key, $dest_vol_name)) {
+    ##        $log->logdie($key->volume . " is not a valid volume name");
+    ## }
+
+    $get_storage_volume_calls = 0;
+
     my ($vol_id, $vol_host, $vol_path, $vol_xattr);
     if (defined $dest_vol_name) {
+	# if we supplied dest_vol_name, then the call to parse_neb_key above
+	# will have set $key->volume to that value and hard_volume to match
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
-            = $self->_get_storage_volume($key, $dest_vol_name);
+            = $self->_get_storage_volume($key, $key->volume, $key->hard_volume);
     } else {
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
             = $self->_get_replication_volume($key);
     }
+
+    # print "selected target: $vol_id, $vol_host\n";
 
     my $uri;
@@ -784,4 +813,5 @@
                 $so_id = $query->fetchrow_hashref->{ 'so_id' };
                 $query->finish;
+		$db->commit;
             }
 
@@ -800,4 +830,5 @@
                 # time LAST_INSERT_ID() is invoked
                 $query->finish;
+		$db->commit;
             }
 
@@ -841,6 +872,6 @@
     eval {
         my $mode = $self->getxattr_object("$key", 'user.mode');
-        if (defined $mode) {
-            $self->chmod_object("$key", $mode);
+        if (defined $mode && $mode ne "") {
+	    $self->chmod_object("$key", $mode);
         }
     };
@@ -852,5 +883,4 @@
 
     $log->debug("leaving");
-
     return "$uri";
 }
@@ -1323,4 +1353,5 @@
             die( "xattr $key:$name does not exist" );
         }
+
         # if we go more then one row bad something very bad has happened.
         unless ($rows == 1) {
@@ -1335,7 +1366,10 @@
         $value = $row->{ 'value' };
     };
+    $db->commit;
+
     if ($@) {
         if ($@ =~ /user\..*? does not exist/) {
             # do not log xattr does not exist messages
+	    # NOTE: the client is using the reported message to interpret the errors
             die $@;
         }
@@ -1386,4 +1420,5 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->debug("leaving");
@@ -1442,5 +1477,4 @@
                 die( "affected row count is $rows instead of 1" );
             }
-
             $db->commit;
             $log->debug("commit");
@@ -1545,6 +1579,5 @@
 	$work_dir = $pattern;
 	$file_pattern = '%';
-    }
-    else {
+    } else {
 	my $dir_plain = dirname($pattern);
 	if ($dir_plain eq 'neb:') {
@@ -1575,4 +1608,5 @@
         my $query = $db->prepare_cached( $sql->find_dir_by_parent_id . " AND dirname LIKE ? ");
         $query->execute( $dir_id, $file_pattern );
+	$db->commit;
 
         while ( my $row = $query->fetchrow_hashref ) {
@@ -1595,4 +1629,5 @@
         my $query = $db->prepare_cached( $sql->find_object_by_dir_id . " AND ext_id_basename LIKE ? ");
         $query->execute( $dir_id , $file_pattern);
+	$db->commit;
 
         while ( my $row = $query->fetchrow_hashref ) {
@@ -1624,5 +1659,4 @@
     my $sql = $self->sql;
     my $db  = $self->_db_for_index($index);
-
 
     # first check to see if the key is an exact match
@@ -1645,4 +1679,5 @@
         $log->logdie("database error: $@");
     }
+    $db->commit;
 
     if (scalar @keys) {
@@ -1678,4 +1713,5 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     # find files under dir
@@ -1692,4 +1728,5 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->debug( "leaving" );
@@ -1753,4 +1790,5 @@
 }
 
+# sub find_instances
 sub find_instances_old
 {
@@ -1769,11 +1807,4 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -1787,8 +1818,6 @@
     my $sql = $self->sql;
 
-#    unless ($key) {
-#        $log->warn("key was undefined after validate_pos(), trying again...");
-#        return $self->find_instances(@_);
-#    }
+    # default value for find_invalid is false
+    if (not defined $find_invalid) { $find_invalid = 0; }
 
     # vol_name overrides the key implied volume
@@ -1828,5 +1857,6 @@
 	    my $rows;
             # ext_id, available
-	    if (defined($find_invalid)) {
+	    if (defined($find_invalid) and $find_invalid) {
+		# XXX returns instances which are NOT available
 		$rows = $query->execute($key->path, 0);
 	    }
@@ -1865,5 +1895,5 @@
 }
 
-#sub find_instances_by_proximity
+# sub find_instances_by_proximity
 sub find_instances
 {
@@ -1882,11 +1912,4 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -1897,11 +1920,11 @@
         }, 
     );
-
     my $sql = $self->sql;
 
-#    unless ($key) {
-#        $log->warn("key was undefined after validate_pos(), trying again...");
-#        return $self->find_instances(@_);
-#    }
+    if (not defined $find_invalid) { $find_invalid = 0; }
+
+    if ($find_invalid eq "find them all") {
+	$log->logdie("old neb-cull --one_only, neb-rm --invalid both disabled by new Server.pm");
+    }
 
     # vol_name overrides the key implied volume
@@ -1932,6 +1955,4 @@
 	    }
 	}
-
-
     }
 
@@ -1972,17 +1993,25 @@
             # ext_id, name, available
 	    # host_vol_id host_cab_id host_site_id ext_id available
-            my $rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 1);
+
+	    my $rows;
+	    if (defined($find_invalid) and $find_invalid) {
+		# returns instances which are NOT available (volume not available)
+		$rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 0);
+	    } else {
+		$rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 1);
+	    }		
             unless ($rows > 0) {
                 $query->finish;
-                die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name");
+                die("no instances on storage volume or volume is not available for key: $key volume: $vol_name");
             }
         } else {
             $query = $db->prepare_cached( $sql->get_object_instances );
+            # ext_id, available
+
 	    my $rows;
-            # ext_id, available
-	    if (defined($find_invalid)) {
+	    if (defined($find_invalid) and $find_invalid) {
+		# returns instances which are NOT available (volume not available)
 		$rows = $query->execute($key->path, 0);
-	    }
-	    else {
+	    } else {
 		$rows = $query->execute($key->path, 1);
 	    }
@@ -1992,8 +2021,13 @@
             }
         }
+	$db->commit;
+	## if we do not call commit here, the transaction stays
+	## open blocking some operations below. 
 
         while (my $row = $query->fetchrow_hashref) {
             my $instance = $row->{ 'uri' };
             push @locations, $instance if $instance;
+
+	    # Carp::carp ("instance 2: $row->{ 'uri' }, $row->{ 'vol_id' }, $row->{ 'cab_id' }, $row->{ 'vol_idx' }\n");
         }
     };
@@ -2003,5 +2037,5 @@
         if (defined $vol_name and not defined $key->hard_volume) {
             $log->debug("retrying with 'any' volume");
-            return $self->find_instances($key->path, 'any');
+            return $self->find_instances($key->path, 'any', $find_invalid);
         }
         $log->logdie("database error: $@");
@@ -2018,4 +2052,8 @@
 }
 
+# this method returns instances on the specified volume
+# or a list of all available instances
+# it does NOT choose instances based on proximity
+# and it does NOT return invalid instances
 sub find_instances_for_cull
 {
@@ -2061,4 +2099,8 @@
     my $db  = $self->db($key);
 
+    ## XXX this method is missing the alias deference section (see find_instances)
+    ## if aliases are intended to be compute nodes in the same cabinet / site as
+    ## the storage nodes, then this is not needed.
+
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
@@ -2082,5 +2124,5 @@
             unless ($rows > 0) {
                 $query->finish;
-                die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name");
+                die("no instances on storage volume or volume is not available for key: $key volume: $vol_name");
             }
         } else {
@@ -2105,10 +2147,13 @@
         $db->rollback;
         # handle soft volumes
-        if (defined $vol_name and not defined $key->hard_volume) {
+	# we need to call find_intances_for_cull here so the returned structure 
+	# is the same (find_instances just returns an array of strings)
+        if (defined $vol_name and not defined $key->hard_volume and ($vol_name ne "any")) {
             $log->debug("retrying with 'any' volume");
-            return $self->find_instances($key->path, 'any');
+            return $self->find_instances_for_cull($key->path, 'any');
         }
         $log->logdie("database error: $@");
     }
+    $db->commit;
 
     # XXX remove this?
@@ -2222,5 +2267,4 @@
                 }
             }
-
             $db->commit;
             $log->debug("commit");
@@ -2282,4 +2326,5 @@
         $stat = $query->fetchrow_arrayref;
         $query->finish;
+	$db->commit;
     };
     $log->logdie("database error: $@") if $@;
@@ -2318,4 +2363,5 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->logdie("no mounted volumes found") unless (scalar @$stats);
@@ -2415,5 +2461,9 @@
     my ($key, $name, $hard_volume) = @_;
     
-#    $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<");
+    # track the number of calls to this function and
+    # give up after 10 attempts
+    $get_storage_volume_calls ++;
+
+#   $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<");
     my $sql = $self->sql;
     my $db  = $self->db($key);
@@ -2427,7 +2477,8 @@
             # %free, name, avaiable, allocate
             $rows = $query->execute($max_used_space, $name, 1, 1);
-            # XXX destinguish between non-existant and unavailable
+            # XXX distinguish between non-existant and unavailable
             unless ($rows > 0) {
                 $query->finish;
+		$db->commit;
 
                 # if a volume name was specified, and is soft, and we failed to
@@ -2442,4 +2493,5 @@
             if ($rows > 1) {
                 $query->finish;
+		$db->commit;
                 die("affected row count is $rows instead of 1");
             }
@@ -2449,8 +2501,12 @@
             $rows = $query->execute($max_used_space, 1, 1, $topfew_count);
 #	    $log->warn("Storage_volume: $rows $topfew_count");
-            # there has to be atleast one storage volume
+            # there has to be at least one storage volume
             unless ($rows > 0) {
                 $query->finish;
-                die("no storage volume is available for key: $key volume: $name hard_volume: $hard_volume");
+		$db->commit;
+		# prevent failure in the die due to undefined variables
+		my $hard_volume_str = defined $hard_volume ? $hard_volume : "undefined";
+		my $name_str = defined $name ? $name : "undefined";
+                die("no storage volume is available for key: $key volume: $name_str hard_volume: $hard_volume_str");
             }
         }
@@ -2465,5 +2521,9 @@
             $log->error($@);
             $log->debug("retrying...");
-            return $self->_get_storage_volume(@_);
+	    if ($get_storage_volume_calls < 10) {
+		return $self->_get_storage_volume(@_);
+	    }
+	    # else
+	    $log->logdie("no available storage volume: $@");
         } 
         # else
@@ -2502,5 +2562,5 @@
 	unless ($rows > 0) {
 	    $query->finish;
-	    die("Requested key $key does not exist");
+	    die("Requested key $key does not exist (or cabinet is undefined)");
 	}
 	if ($rows == 1) {
@@ -2513,16 +2573,18 @@
 	    }
 	    $query->finish;
-	}
-	else {
+	} else {
+	    # if instances are spread across multiple cabinets, allow the next instance to go anywhere
+	    # NOTE: this does not prevent the new instance from going to the same volume as one of the existing instances
 	    $forbidden_cabinet = 0;
 	    $forbidden_site    = 0;
 	    $query->finish;
 	}
-	    
+	$db->commit;
 
         $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id );
-        # ext_id, %free, avaiable, allocate
+        # ext_id, %free, available, allocate
+
         $rows = $query->execute($key->path, $max_used_space, 1, 1, $forbidden_cabinet, $forbidden_site, $topfew_count);
-        # XXX destinguish between non-existant and unaviable
+        # XXX distinguish between non-existant and unavailable
         unless ($rows > 0) {
             $query->finish;
@@ -2544,4 +2606,5 @@
         ($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array;
         $query->finish;
+	$db->commit;
     };
     $log->logdie("database error: $@") if $@;
@@ -2590,4 +2653,6 @@
         ($ext_id) = $query->fetchrow_array;
         $query->finish;
+        $db->commit;
+	## if we do not call commit here, the transaction stays open blocking some operations below
     };
     if ($@) {
@@ -2612,5 +2677,6 @@
 }
 
-
+# in the past, _is_valid_volume_name returned 'undef'.  
+# now it either returns 1 (success) or 0 (failure)
 sub _is_valid_volume_name
 {
@@ -2650,4 +2716,5 @@
         ($vol_id, $vol_path, $free) = $query->fetchrow_array;
         $query->finish;
+        $db->commit;
     };
     $log->logdie("database error: $@") if $@;
Index: trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 40618)
+++ trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 41172)
@@ -574,4 +574,7 @@
 DROP TABLE IF EXISTS directory;
 DROP TABLE IF EXISTS deleted;
+DROP TABLE IF EXISTS cabinet;
+DROP TABLE IF EXISTS aliasvol;
+DROP TABLE IF EXISTS lost_instances;
 SET FOREIGN_KEY_CHECKS=1
 END
Index: trunk/Nebulous-Server/lib/Test/Nebulous.pm
===================================================================
--- trunk/Nebulous-Server/lib/Test/Nebulous.pm	(revision 40618)
+++ trunk/Nebulous-Server/lib/Test/Nebulous.pm	(revision 41172)
@@ -19,7 +19,9 @@
 our @EXPORT = qw( $NEB_DB $NEB_USER $NEB_PASS $NEB_MEMCACHED_SERVERS);
 
-our $NEB_DB     = $ENV{'NEB_DB'}   || "DBI:mysql:database=test:host=localhost";
-our $NEB_USER   = $ENV{'NEB_USER'} || "test";
-our $NEB_PASS   = $ENV{'NEB_PASS'} || '';
+# require use of the test database server:
+
+our $NEB_DB     = "DBI:mysql:database=test:host=localhost:mysql_socket=/var/run/mysqld/mysqld.sock";
+our $NEB_USER   = "test";
+our $NEB_PASS   = '';
 our $NEB_MEMCACHED_SERVERS = ['127.0.0.1:11211'];
 
@@ -35,4 +37,12 @@
 my $dir6 = "";
 my $dir7 = "";
+my $dir8 = "";
+
+sub show_setup {
+    my $self = shift;
+
+    print "DB: $NEB_DB, USER: $NEB_USER, PASS: $NEB_PASS\n";
+}
+
 
 sub setup {
@@ -49,4 +59,5 @@
     $dir6 = tempdir( CLEANUP => 0 );
     $dir7 = tempdir( CLEANUP => 0 );
+    $dir8 = tempdir( CLEANUP => 0 );
 
     foreach my $statement (@{ $sql->get_db_schema }) {
@@ -54,44 +65,48 @@
     }
 
-    # node01/node02: allocate = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 1);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1);
+    # generate 2 cabinets so we can replicate to a valid location
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab01", 1, "main") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab02", 1, "main") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab03", 2, "back") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab04", 2, "back") });
 
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 2);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2);
+    # node01 : allocate = TRUE (cab01, site 1)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (1, 1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e7, note FROM volume WHERE vol_id = ? }, undef, 1);
 
-    # node03: allocate = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 3);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3);
+    # node02 : allocate = TRUE (cab02, site 1)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (2, 2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 2);
+
+    # node03: allocate = TRUE (cab03, site 2)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (3, 3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 3);
 
     # node04: allocate = FALSE, available = FALSE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (4, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 4);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (4, 1, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 4);
 
     # node05: allocate = FALSE, available = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (5, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 5);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (5, 1, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 5);
 
     # node06: allocate = TRUE, available = FALSE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (6, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 6);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (6, 1, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 6);
 
     # node07: full
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (7, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e10 FROM volume WHERE vol_id = ? }, undef, 7);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (7, 1, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e10, note FROM volume WHERE vol_id = ? }, undef, 7);
+    
+    # node08: allocate = TRUE, (cab04, site 2)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (8, 4, 'node08', 'node08', ?, '/', TRUE, TRUE) }, undef, $dir8);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 8);
 
-#    $dbh->do(qq{ call getmountedvol() });
+#   $dbh->do(qq{ call getmountedvol() });
 }
 
 sub cleanup {
     # memcached needs to be emptied between test runs
-        Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS)->flush_all;
+    Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS)->flush_all;
     # if flush_all() turns out to be useless as feared
 #   my $memd = Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS);
@@ -106,4 +121,5 @@
 #   $query->finish;
 
+    
     foreach my $statement (@{ $sql->get_db_clear }) {
         $dbh->do( $statement );
@@ -118,4 +134,19 @@
     rmtree([$dir6], 0, 1) if -e $dir6;
     rmtree([$dir7], 0, 1) if -e $dir7;
+    rmtree([$dir8], 0, 1) if -e $dir8;
+}
+
+# function to change state of one node for tests:
+sub switch_node_state {
+    my $self = shift;
+
+    $dbh->do(qq{ UPDATE volume set available = FALSE where name = 'node02'});
+    $dbh->do(qq{ UPDATE mountedvol set available = FALSE where name = 'node02'});
+}
+
+sub get_node_dirs {
+    my $self = shift;
+
+    return ($dir1, $dir2, $dir3, $dir4, $dir5, $dir6, $dir7, $dir8);
 }
 
Index: trunk/Nebulous-Server/scripts/mkdirs.mana
===================================================================
--- trunk/Nebulous-Server/scripts/mkdirs.mana	(revision 41172)
+++ trunk/Nebulous-Server/scripts/mkdirs.mana	(revision 41172)
@@ -0,0 +1,26 @@
+#!/usr/bin/env mana
+
+macro mk.neb.dirs
+  if ($0 != 2)
+    echo "USAGE: mk.neb.dirs (path)"
+    exit 1
+  end
+
+  for i 0 256
+    for j 0 256
+      sprintf dir %02x/%02x $i $j
+      mkdir $1/$dir
+    end
+    echo $i
+  end
+end
+
+if ($SCRIPT)
+  if ($argv:n != 1)
+    echo "USAGE: mkdirs.mana (path)"
+    exit 1
+  end
+
+  mk.neb.dirs $argv:0
+  exit 0
+end
Index: trunk/Nebulous-Server/scripts/stest.sh
===================================================================
--- trunk/Nebulous-Server/scripts/stest.sh	(revision 41172)
+++ trunk/Nebulous-Server/scripts/stest.sh	(revision 41172)
@@ -0,0 +1,7 @@
+#!/bin/csh -f
+
+# script to run tests of the cmd-line interfaces
+
+# neb-touch
+# neb-mv
+# neb-mv
Index: trunk/Nebulous-Server/t/03_server_setup.t
===================================================================
--- trunk/Nebulous-Server/t/03_server_setup.t	(revision 40618)
+++ trunk/Nebulous-Server/t/03_server_setup.t	(revision 41172)
@@ -14,4 +14,7 @@
 use Nebulous::Server;
 use Test::Nebulous;
+
+Test::Nebulous->show_setup;
+die "test";
 
 Test::Nebulous->setup;
Index: trunk/Nebulous-Server/t/04_server_create_object.t
===================================================================
--- trunk/Nebulous-Server/t/04_server_create_object.t	(revision 40618)
+++ trunk/Nebulous-Server/t/04_server_create_object.t	(revision 41172)
@@ -30,5 +30,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -40,12 +39,11 @@
     uri_scheme_ok($uri, 'file');
 
-SKIP: {
-    skip "requires xattr support", 1 unless $test_xattr;
-    is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
-}
-}
-
-Test::Nebulous->setup;
-
+  SKIP: {
+      skip "requires xattr support", 1 unless $test_xattr;
+      is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
+    }
+}
+
+Test::Nebulous->setup;
 {
     # key
@@ -57,8 +55,8 @@
     uri_scheme_ok($uri, 'file');
 
-SKIP: {
-    skip "requires xattr support", 1 unless $test_xattr;
-    is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
-}
+  SKIP: {
+      skip "requires xattr support", 1 unless $test_xattr;
+      is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
+    }
 }
 
@@ -528,5 +526,5 @@
     $neb->create_object("foo", '~node07');
 };
-like($@, qr/node07 is not available/, "request volume this is full");
+like($@, qr/node07 is not available/, "request volume which is full");
 
 Test::Nebulous->setup;
@@ -535,5 +533,5 @@
     $neb->create_object("neb://~node07/foo");
 };
-like($@, qr/node07 is not available/, "request volume this is full");
+like($@, qr/node07 is not available/, "request volume which is full");
 
 Test::Nebulous->setup;
Index: trunk/Nebulous-Server/t/05_server_replicate_object.t
===================================================================
--- trunk/Nebulous-Server/t/05_server_replicate_object.t	(revision 40618)
+++ trunk/Nebulous-Server/t/05_server_replicate_object.t	(revision 41172)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 25;
+use Test::More tests => 43;
 
 use lib qw( ./t ./lib );
@@ -28,5 +28,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -45,5 +44,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -62,5 +60,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume is undef
@@ -79,5 +76,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -147,4 +143,166 @@
 }
 
+# hard_volume tests:
+
+# replication with hard_volume:
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object: hard_volume as node target
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    like($loc2, qr/$nodedirs[1]/, "instance matches expected node");
+}
+
+# replication to unknown volume: should succeed
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "dummy");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    uri_scheme_ok($loc2, 'file');
+}
+
+# replication to unknown volume with hard_volume: should fail
+Test::Nebulous->setup;
+eval {
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~dummy");
+
+    my $locations = $neb->find_instances("foo");
+};
+like($@, qr/is not a valid volume name/, "replicate to unknown node with hard_volume should fail");
+
+# replication to unavailable volume: should succeed
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "node04");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    uri_scheme_ok($loc2, 'file');
+}
+
+# replication to unavailable volume with hard_volume: should fail
+Test::Nebulous->setup;
+eval {
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node04");
+
+    my $locations = $neb->find_instances("foo");
+};
+like($@, qr/is not available/, "replicate to unknown node with hard_volume should fail");
+
+### END hard_volume tests
+
+### location-aware tests (do we respect the cab_id / vol_id exclusions?)
+
+# here are the volume definitions (Test/Nebulous.pm):
+# name   : vol_id : cab_id : site_id : xattr
+# node01 :      1 :      1 :       1 :     0
+# node02 :      2 :      2 :       1 :     0
+# node03 :      3 :      3 :       2 :     0
+# node04 :      4 :      1 :       1 :     0
+# node05 :      5 :      1 :       1 :     0
+# node06 :      6 :      1 :       1 :     0
+# node07 :      7 :      1 :       1 :     0
+# node08 :      8 :      4 :       2 :     0
+
+# replication without volume 
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object: hard_volume as node target
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    # if the volume is not explicitly specified, then the first copy
+    # should go to a different volume, cabinet, and site
+    # if first is one node01, replication should go to node03 or node08
+    # since those both have different site_id values
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+
+    my $isnode03 = $loc2 =~ m/$nodedirs[2]/;
+    my $isnode08 = $loc2 =~ m/$nodedirs[7]/;
+    ok($isnode03 || $isnode08, "instance matches expected node");
+}
+
+# replication without volume
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node03");
+    my $uri2 = $neb->replicate_object("foo");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    # if the volume is not explicitly specified, then the first copy
+    # should go to a different volume, cabinet, and site
+    # if first is one node03, replication should go to node01 or node02
+    # since those both have different site_id values
+    like($loc1, qr/$nodedirs[2]/, "instance matches expected node");
+
+    my $isnode01 = $loc2 =~ m/$nodedirs[0]/;
+    my $isnode02 = $loc2 =~ m/$nodedirs[1]/;
+    ok($isnode01 || $isnode02, "instance matches expected node");
+}
+
 Test::Nebulous->setup;
 
Index: trunk/Nebulous-Server/t/08_server_find_instances.t
===================================================================
--- trunk/Nebulous-Server/t/08_server_find_instances.t	(revision 40618)
+++ trunk/Nebulous-Server/t/08_server_find_instances.t	(revision 41172)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 19;
+use Test::More tests => 44;
 
 use lib qw( ./t ./lib );
@@ -22,6 +22,19 @@
 );
 
-Test::Nebulous->setup;
-
+# XXX previously, the logic was: if we request a hard volume location
+# using ~node, then we would give an error if no data is available except on that node.
+
+# the code has been modified so that the requested storage object is provided if possible, 
+# even if the apparently "hard" target did not exist.
+
+Test::Nebulous->setup;
+Test::Nebulous->cleanup; # make sure the database is reset before running any tests
+
+# for (my $i = 0; $i < $Nloc; $i++) {
+#	my $loc = $locations->[$i];
+#	print "location: $loc\n";
+#}
+
+Test::Nebulous->setup;
 {
     # key
@@ -35,5 +48,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -49,5 +61,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -61,5 +72,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -73,5 +83,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -85,5 +94,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -98,14 +106,158 @@
 }
 
-# object exists but instance is on a different volume
-Test::Nebulous->setup;
-
+# request from a hard_volume returns an instance which is not on the hard volume if available
+Test::Nebulous->setup;
 eval {
     # key, volume
     my $uri = $neb->create_object('foo', '~node01');
-
     my $locations = $neb->find_instances('foo', '~node02');
-};
-like($@, qr/no instances on storage volume/, 'instances on a different volume');
+
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on different node (as hard_volume) returned");
+
+# request from a hard_volume returns an instance which is not on the hard volume if available
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node01');
+    my $locations = $neb->find_instances('foo', '~node02');
+
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on different node (as hard_volume) returned");
+
+### TESTS FOR INSTANCE ON INVALID NODE
+diag ("TESTS FOR INSTANCE ON INVALID NODE");
+
+# request from a soft_volume for an instance which is on an invalid node (find_invalid = 1) [SOFT, 1]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02', 1);
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on invalid node (as soft_volume) returned with find_invalid = 1");
+
+# request from a soft_volume for an instance which is on an invalid node (find_invalid = 0) [SOFT, 0]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02', 0);
+};
+like($@, qr/database error/, "instance on invalid node (as soft_volume) not returned (find_invalid = 0)");
+
+# request from a hard_volume for an instance which is on an invalid node (find_instance not supplied) [SOFT, X]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02');
+};
+like($@, qr/no instances available for key/, "instance on invalid node (as soft_volume) not returned (find_invalid not set)");
+
+# request from a hard_volume for an instance which is on an invalid node (find_invalid = 1) [HARD, 1]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02', 1);
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on invalid node (as hard_volume) returned with find_invalid = 1");
+
+# request from a hard_volume for an instance which is on an invalid node (find_invalid = 0) [HARD, 0]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02', 0);
+};
+like($@, qr/no instances on storage volume/, "instance on invalid node (as hard_volume) not returned (find_invalid = 0)");
+
+# request from a hard_volume for an instance which is on an invalid node [HARD, X]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02');
+};
+like($@, qr/no instances on storage volume/, "instance on invalid node (as hard_volume) not returned (find_invalid not set)");
+
+#### TESTING the instance proximity analysis
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # place 4 instances on specific nodes (hard targets works):
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+    my $uri3 = $neb->replicate_object("foo", "~node03");
+    my $uri4 = $neb->replicate_object("foo", "~node08");
+
+    my $locations = $neb->find_instances("foo", "node01");
+    my $Nloc = @$locations;
+
+    is($Nloc, 4, "found 4 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+    my $loc3 = $locations->[2];
+    my $loc4 = $locations->[3];
+
+    # choosing instance relative to node01 yields:
+    like($loc1, qr/$nodedirs[0]/, "node01 is closest to node01");    
+    like($loc2, qr/$nodedirs[1]/, "node02 is 2nd closest to node01");
+    like($loc3, qr/$nodedirs[2]/, "node03 is 3rd closest to node01");
+    like($loc4, qr/$nodedirs[7]/, "node08 is 4th closest to node01");
+}
+
+#### TESTING the instance proximity analysis
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # place 4 instances on specific nodes (hard targets works):
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+    my $uri3 = $neb->replicate_object("foo", "~node03");
+    my $uri4 = $neb->replicate_object("foo", "~node08");
+
+    my $locations = $neb->find_instances("foo", "node03");
+    my $Nloc = @$locations;
+
+    is($Nloc, 4, "found 4 instances");
+    
+    my $loc1 = $locations->[0]; # 
+    my $loc2 = $locations->[1]; # 
+    my $loc3 = $locations->[2]; # 
+    my $loc4 = $locations->[3]; # 
+
+    # choosing instance relative to node01 yields:
+    like($loc1, qr/$nodedirs[2]/, "node03 is closest to node03");
+    like($loc2, qr/$nodedirs[7]/, "node08 is 2nd closest to node03");
+    like($loc3, qr/$nodedirs[1]/, "node02 is 3rd closest to node03");
+    like($loc4, qr/$nodedirs[0]/, "node08 is 4th closest to node03");
+}
+
+########
 
 Test::Nebulous->setup;
@@ -125,9 +277,9 @@
 
 Test::Nebulous->setup;
-
-eval {
-    $neb->find_instances();
-};
-like($@, qr/1 - 2 were expected/, "no params");
+eval {
+    my $locations = $neb->find_instances();
+};
+like($@, qr/1 - 3 were expected/, "no params");
+# note the new API expects 1-3 parameters
 
 Test::Nebulous->setup;
@@ -135,7 +287,8 @@
 eval {
     $neb->create_object('foo');
-    $neb->find_instances('foo', 'node01', 3);
-};
-like($@, qr/1 - 2 were expected/, "too many params");
+    $neb->find_instances('foo', 'node01', 3, 5);
+};
+like($@, qr/1 - 3 were expected/, "too many params");
+# note the new API expects 1-3 parameters
 
 Test::Nebulous->cleanup;
Index: trunk/Nebulous-Server/t/11_server_is_valid_volume_name.t
===================================================================
--- trunk/Nebulous-Server/t/11_server_is_valid_volume_name.t	(revision 40618)
+++ trunk/Nebulous-Server/t/11_server_is_valid_volume_name.t	(revision 41172)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 3;
+use Test::More tests => 5;
 
 use lib qw( ./t ./lib );
@@ -31,5 +31,15 @@
 Test::Nebulous->setup;
 
-is($neb->_is_valid_volume_name('foo', 'node99'), undef, "invalid volume name");
+is($neb->_is_valid_volume_name('foo', 'node99'), 0, "invalid volume name");
+
+Test::Nebulous->setup;
+
+# key does not need to exist (is hashed to select db server)
+ok($neb->_is_valid_volume_name('bax', 'node01'), "valid volume name");
+
+Test::Nebulous->setup;
+
+# 'any' should always be a valid name
+ok($neb->_is_valid_volume_name('foo', 'any'), "valid volume name");
 
 Test::Nebulous->cleanup;
Index: trunk/Nebulous-Server/t/15_server_xattr.t
===================================================================
--- trunk/Nebulous-Server/t/15_server_xattr.t	(revision 40618)
+++ trunk/Nebulous-Server/t/15_server_xattr.t	(revision 41172)
@@ -184,6 +184,5 @@
     $neb->getxattr_object('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
-    "get xattr from non-existant xattr key");
+like($@, qr|xattr neb:///foo:user.bar does not exist|, "get xattr from non-existant xattr key");
 
 Test::Nebulous->setup;
Index: trunk/Nebulous-Server/t/16_mounts.t
===================================================================
--- trunk/Nebulous-Server/t/16_mounts.t	(revision 40618)
+++ trunk/Nebulous-Server/t/16_mounts.t	(revision 41172)
@@ -28,5 +28,5 @@
     my $mounts = $neb->mounts();
 
-    is(scalar @$mounts, 7, "number of rows");
+    is(scalar @$mounts, 8, "number of rows");
 
     my %row;
@@ -35,12 +35,12 @@
         = @{$mounts->[0]};
     
-    is($row{total},     100000000000);
-    is($row{used},      100000000);
-    is($row{vol_id},    1);
-    is($row{name},      "node01");
-    is($row{host},      "node01");
-    is($row{allocate},  1);
-    is($row{available}, 1);
-    is($row{xattr},     0);
+    is($row{total},     100000000000, "total bytes");
+    is($row{used},      100000000, "used bytes");
+    is($row{vol_id},    1, "vol id");
+    is($row{name},      "node01", "name");
+    is($row{host},      "node01", "host");
+    is($row{allocate},  1, "is allocated");
+    is($row{available}, 1, "is available");
+    is($row{xattr},     0, "has no xattr");
 
     # 2nd row
@@ -48,12 +48,12 @@
         = @{$mounts->[1]};
 
-    is($row{total},     100000000000);
-    is($row{used},      1000000000);
-    is($row{vol_id},    2);
-    is($row{name},      "node02");
-    is($row{host},      "node02");
-    is($row{allocate},  1);
-    is($row{available}, 1);
-    is($row{xattr},     0);
+    is($row{total},     100000000000, "total bytes");
+    is($row{used},      1000000000, "used bytes");
+    is($row{vol_id},    2, "vol id");
+    is($row{name},      "node02", "name");
+    is($row{host},      "node02", "host");
+    is($row{allocate},  1, "is allocated");
+    is($row{available}, 1, "is available");
+    is($row{xattr},     0, "has no xattr");
 }
 
Index: trunk/Nebulous-Server/t/18_server_chmod_object.t
===================================================================
--- trunk/Nebulous-Server/t/18_server_chmod_object.t	(revision 40618)
+++ trunk/Nebulous-Server/t/18_server_chmod_object.t	(revision 41172)
@@ -38,4 +38,7 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path);
 }
 
@@ -50,6 +53,9 @@
     my $mode = $neb->chmod_object($key, 0440);
 
-    my $st1 = stat(URI->new($uri1)->path);
-    my $st2 = stat(URI->new($uri2)->path);
+    my $path1 = URI->new($uri1)->path;
+    my $path2 = URI->new($uri2)->path;
+
+    my $st1 = stat($path1);
+    my $st2 = stat($path2);
 
     is($st1->mode &07777, 0440, "chmod() first instance");
@@ -57,4 +63,8 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod (0600, $path1);
+    chmod (0600, $path2);
 }
 
Index: trunk/Nebulous-Server/t/20_server_find_objects_wildcard.t
===================================================================
--- trunk/Nebulous-Server/t/20_server_find_objects_wildcard.t	(revision 41172)
+++ trunk/Nebulous-Server/t/20_server_find_objects_wildcard.t	(revision 41172)
@@ -0,0 +1,109 @@
+#!/usr/bin/perl
+
+# Copryight (C) 2004-2005  Joshua Hoblitt
+#
+# $Id: 12_server_find_objects.t,v 1.4 2008-05-16 20:29:19 jhoblitt Exp $
+
+use strict;
+use warnings FATAL => qw( all );
+
+use Test::More tests => 22;
+
+use lib qw( ./t ./lib );
+
+use Nebulous::Server;
+use Test::Nebulous;
+
+my $neb = Nebulous::Server->new(
+    dsn         => $NEB_DB,
+    dbuser      => $NEB_USER,
+    dbpasswd    => $NEB_PASS,
+);
+
+Test::Nebulous->setup;
+Test::Nebulous->cleanup;
+
+&mktestfiles ($neb);
+{
+    my $keys = $neb->find_objects_wildcard("topdir");
+
+    is(scalar @$keys, 4, 'number of keys found');
+    is($keys->[0], "topdir/subdir/", "key name matches");
+    is($keys->[1], "topdir/object1", "key name matches");
+    is($keys->[2], "topdir/object2", "key name matches");
+    is($keys->[3], "topdir/objfoobar", "key name matches");
+}
+
+&mktestfiles ($neb);
+{
+    my $keys = $neb->find_objects_wildcard("topdir/%");
+
+    is(scalar @$keys, 4, 'number of keys found');
+    is($keys->[0], "topdir/subdir/", "key name matches");
+    is($keys->[1], "topdir/object1", "key name matches");
+    is($keys->[2], "topdir/object2", "key name matches");
+    is($keys->[3], "topdir/objfoobar", "key name matches");
+}
+
+&mktestfiles ($neb);
+{
+    my $keys = $neb->find_objects_wildcard("topdir/obj%");
+
+    is(scalar @$keys, 3, 'number of keys found');
+    is($keys->[0], "topdir/object1", "key name matches");
+    is($keys->[1], "topdir/object2", "key name matches");
+    is($keys->[2], "topdir/objfoobar", "key name matches");
+}
+
+&mktestfiles ($neb);
+{
+    my $keys = $neb->find_objects_wildcard("topdir/object%");
+
+    is(scalar @$keys, 2, 'number of keys found');
+    is($keys->[0], "topdir/object1", "key name matches");
+    is($keys->[1], "topdir/object2", "key name matches");
+}
+
+&mktestfiles ($neb);
+{
+    my $keys = $neb->find_objects_wildcard("obj%");
+
+    is(scalar @$keys, 3, 'number of keys found');
+    is($keys->[0], "object1", "key name matches");
+    is($keys->[1], "object2", "key name matches");
+    is($keys->[2], "objfoobar", "key name matches");
+}
+
+eval {
+    $neb->find_objects_wildcard("foo", 3);
+};
+like($@, qr/1 was expected/, "too many params");
+
+Test::Nebulous->cleanup;
+
+sub mktestfiles {
+    my ($neb) = $_[0];
+
+    Test::Nebulous->setup;
+    
+    # find_objects_wildcard allows these searches:
+    # -> dir : implies dir/* (dir/%)
+    # -> dir/%
+    # -> dir/sub%
+    # -> sub%
+    
+    # create examples matching the above
+    $neb->create_object("object1");
+    $neb->create_object("object2");
+    $neb->create_object("objfoobar");
+    
+    $neb->create_object("topdir/object1");
+    $neb->create_object("topdir/object2");
+    $neb->create_object("topdir/objfoobar");
+    
+    $neb->create_object("topdir/subdir/object1");
+    $neb->create_object("topdir/subdir/object2");
+    $neb->create_object("topdir/subdir/objfoobar");
+    
+    return 1;
+}