Index: trunk/Nebulous/Build.PL
===================================================================
--- trunk/Nebulous/Build.PL	(revision 40680)
+++ trunk/Nebulous/Build.PL	(revision 41172)
@@ -96,5 +96,6 @@
         'Test::Nebulous'            => 0,
         'Test::Cmd'                 => '1.05',
-        'Apache2::SOAP'             => 0,
+# gentoo: 'Apache2::SOAP'             => 0,
+        'Apache::SOAP'             => 0, # needed instead for ubuntu 18.04
         'Apache::DBI'               => '1.05',
         'Apache::Test'              => '1.27',  # bundles Apache::TestMB
Index: trunk/Nebulous/bin/neb-cat
===================================================================
--- trunk/Nebulous/bin/neb-cat	(revision 40680)
+++ trunk/Nebulous/bin/neb-cat	(revision 41172)
@@ -18,5 +18,5 @@
 use constant CHUNK_SIZE => 1024;
 
-my ($server);
+my ($server, $help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -24,8 +24,11 @@
 GetOptions(
     'server|s=s'    => \$server,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
 
-pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 )
-        unless defined $server;
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
+
+pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 ) unless defined $server;
 
 my $neb = Nebulous::Client->new(
@@ -95,5 +98,5 @@
 =over 4
 
-=item * --server|-s <URL>
+=item --server|-s <URL>
 
 URL of the Nebulous server to connect to.
Index: trunk/Nebulous/bin/neb-cp
===================================================================
--- trunk/Nebulous/bin/neb-cp	(revision 40680)
+++ trunk/Nebulous/bin/neb-cp	(revision 41172)
@@ -16,5 +16,5 @@
 use Pod::Usage qw( pod2usage );
 
-my ($volume, $server);
+my ($volume, $server, $help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -23,5 +23,9 @@
     'volume=s'      => \$volume,
     'server|s=s'    => \$server,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 my $src = shift;
Index: trunk/Nebulous/bin/neb-cull
===================================================================
--- trunk/Nebulous/bin/neb-cull	(revision 40680)
+++ trunk/Nebulous/bin/neb-cull	(revision 41172)
@@ -17,5 +17,5 @@
 
 my ($min_copies, $server, $volume, $one_only);
-my ($soft_volume);
+my ($soft_volume, $help);
 $server = $ENV{'NEB_SERVER'} unless $server;
 
@@ -26,5 +26,9 @@
     'one_only|o'    => \$one_only,
     'soft_volume'   => \$soft_volume,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 my $key = shift;
@@ -59,7 +63,9 @@
 # print "@cull_args\n";
 
-
 if ($one_only) {
     my $status = $neb->there_can_be_only_one(@cull_args);
+    if (not defined $status) {
+        die "not enough instances of Nebulous key: $key";
+    }
     if ($status == 0) {
         die "not enough instances of Nebulous key: $key";
Index: trunk/Nebulous/bin/neb-insert
===================================================================
--- trunk/Nebulous/bin/neb-insert	(revision 40680)
+++ trunk/Nebulous/bin/neb-insert	(revision 41172)
@@ -95,5 +95,5 @@
 =head1 SYNOPSIS
 
-    neb-insert --server <neb-server> <neb-key> <local-file> [--copies <n>] [--destroy]
+    neb-insert [--server <neb-server>] <neb-key> <local-file> [--copies <n>] [--destroy]
 
 #=head1 DESCRIPTION
Index: trunk/Nebulous/bin/neb-ls
===================================================================
--- trunk/Nebulous/bin/neb-ls	(revision 40680)
+++ trunk/Nebulous/bin/neb-ls	(revision 41172)
@@ -19,8 +19,8 @@
     $ls,
     $column,
+    $help
 );
 
 $server = $ENV{'NEB_SERVER'} unless $server;
-
 
 GetOptions(
@@ -29,5 +29,9 @@
     'l'             => \$ls,
     'column|c'      => \$column,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 # twiddle column so that it does it by default.
@@ -35,9 +39,10 @@
 $column = not $column;
 
+if ($ls) { $path = 1; }
+
 my $pattern = shift;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --server", -exitval => 2 )
-    unless $server;
+pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
 
 my $neb = Nebulous::Client->new(
@@ -96,10 +101,10 @@
 =head1 SYNOPSIS
 
-    neb-ls [--server <URL>] [-c] [-p] <pattern>
+    neb-ls [--server <URL>] [-c] [-p] [-l] <pattern>
 
 =head1 DESCRIPTION
 
 This program list Nebulous keys matched by C<<pattern>>.  Call it with no
-arguments is equivalent to searching with the pattern C<neb://>.  SQL like
+arguments is equivalent to searching with the pattern C<neb://>.  SQL-like
 wildcards are supported to select out certain files.
 
@@ -108,15 +113,19 @@
 =over 4
 
-=item * --path|-p
+=item --path|-p
 
 Find the disk files corresponding to the keys found.
 
-=item * --column|-c
+=item --column|-c
 
 Disable column formatting, and output results one to a line.
 
+=item --l|-l
+
+Give disk file information using ls (implies -p).
+
 Optional
 
-=item * --server|-s <URL>
+=item --server|-s <URL>
 
 URL of the Nebulous server to connect to.
Index: trunk/Nebulous/bin/neb-migrate
===================================================================
--- trunk/Nebulous/bin/neb-migrate	(revision 40680)
+++ trunk/Nebulous/bin/neb-migrate	(revision 41172)
@@ -23,4 +23,6 @@
 
 $server = $ENV{'NEB_SERVER'} unless $server;
+
+die "this tool is probably too inefficient unless used very carefully";
 
 GetOptions(
Index: trunk/Nebulous/bin/neb-repair
===================================================================
--- trunk/Nebulous/bin/neb-repair	(revision 40680)
+++ trunk/Nebulous/bin/neb-repair	(revision 41172)
@@ -45,9 +45,10 @@
 );
 
-die "can't connected to Nebulous Server: $server"
+die "can't connect to Nebulous Server: $server"
     unless defined $neb;
 
 my $stat = $neb->stat($key);
 die "nebulous key: $key not found" unless $stat;   
+
 my $instances = $neb->find_instances($key, 'any');
 die "no instances found" unless $instances;   
@@ -71,4 +72,5 @@
 
 for (my $i = 0; $i <= $#files; $i++) {
+    # count existing files and get the MD5 sums
     if (-e $files[$i]) {
 	$existance[$i] = 1;
@@ -89,6 +91,5 @@
 	push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i];
 	$last_f = $files[$i];
-    }
-    else {
+    } else {
 	$existance[$i] = 0;
 	$md5sums[$i] = 'NON-EXISTANT';
@@ -98,19 +99,23 @@
 }
 
+# check number of copies and number of unique MD5 sum values
 my $N_mds = scalar(keys(%md5sum_uniq));
 my $N_files = scalar(@files);
+
+# seems like we should not call neb-replicate until *after* we have
+# removed 0-byte copies
 if ($N_files < $user_copies) {
     print "Insufficient copies ($N_files) of $key, replicating!\n";
     system("neb-replicate $key");
 }
+
 if ($N_mds == 1) {
     print "Nebulous key $key is consistent.\n";
-}
-else {
+} else {
     my $valid_sum = '';
     my $valid_file = '';
     my $m;
     foreach $m (keys %md5sum_valid) {
-	if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) {
+	if (($m ne 'd41d8cd98f00b204e9800998ecf8427e') && ($m ne 'NON-EXISTANT')) {
 	    if ($valid_sum ne '') {
 		die "Too many acceptable files for key $key\n";
@@ -122,6 +127,5 @@
     if ($valid_sum eq '') {
 	die "No acceptable file for key $key\n";
-    }
-    else {
+    } else {
 	print "Repairing instances\n";
 	foreach $m (keys %md5sum_valid) {
@@ -130,20 +134,26 @@
 		    print "\tcp $valid_file $f\n";
 		    system("cp $valid_file $f");
+
+		    # if we failed to make a copy, the target ($f) could be inaccessible
 		    if ($? >> 8 != 0) {
 			if ($existance[0] == 1) {
-			    # We can fix this with culls and replications.
+
+			    # The target is a valid file 0-byte file
+			    # but we cannot replace it with the
+			    # correct file.  We can fix this with
+			    # culls and replications.
+
 			    print "Attempting to fix this.\n";
 			    print("neb-replicate $key\n");
 			    system("neb-replicate $key");
 			    if ($? >> 8 == 0) {
+				# if the neb-replicate succeeded, remove the bad copy with neb-cull
 				my $bad_volume = (split /\//, $f)[2];
 				print("neb-cull --volume $bad_volume $key\n");
 				system("neb-cull --volume $bad_volume $key");
-			    }
-			    else {
+			    } else {
 				die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n";
 			    }
-			}
-			else {
+			} else {
 			    die "Failed to copy file $valid_file to $f!\n";
 			}
@@ -187,5 +197,12 @@
 =head1 DESCRIPTION
 
-This program repairs instances and corrects instance counts for a nebulous key.  
+This program repairs instances and corrects instance counts for a nebulous key:
+
+If some of the existing instances have the MD5 sum corresponding to a
+0-byte file, but the rest have the same, non-zero byte md5sum, then
+replace the 0-byte files with the non-zero byte version.
+
+At the end, if the number of instances is less than the value of
+user.copies, then additional copies are generated.
 
 =head1 OPTIONS
Index: trunk/Nebulous/bin/neb-replicate
===================================================================
--- trunk/Nebulous/bin/neb-replicate	(revision 40680)
+++ trunk/Nebulous/bin/neb-replicate	(revision 41172)
@@ -42,4 +42,7 @@
 );
 
+# Nebulous::Client programs can use this to set the log level: 
+# $neb->set_log_level ('DEBUG');
+
 die "can't connected to Nebulous Server: $server"
     unless defined $neb;
@@ -50,4 +53,5 @@
     for (my $i = 0; $i < $copies; $i++) {
         if (defined $volume) {
+	    # this is coded so that all copies end up on the specified volume which seems wrong
             $neb->replicate($key, $volume)
                 or die "failed to replicate Nebulous key: $key";
Index: trunk/Nebulous/bin/neb-shift
===================================================================
--- trunk/Nebulous/bin/neb-shift	(revision 40680)
+++ trunk/Nebulous/bin/neb-shift	(revision 41172)
@@ -17,4 +17,6 @@
 
 my ($volume, $server);
+
+die "this tool may fail since find_instances will return an instance which is not on the hard volume";
 
 $server = $ENV{'NEB_SERVER'} unless $server;
Index: trunk/Nebulous/bin/neb-stat
===================================================================
--- trunk/Nebulous/bin/neb-stat	(revision 40680)
+++ trunk/Nebulous/bin/neb-stat	(revision 41172)
@@ -19,21 +19,23 @@
 use URI;
 
-my ($validate,$server,$nouris);
+my ($validate,$server,$nouris,$help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
 
 GetOptions(
-    'server|s=s'     => \$server,
-    'validate|v'         => \$validate,
-    'files'          => \$nouris,
+    'server|s=s'    => \$server,
+    'validate|v'    => \$validate,
+    'files|f'       => \$nouris,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
 
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
+
 my $key = shift;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --server", -exitval => 2 )
-    unless $server;
-pod2usage( -msg => "missing key", exitval => 2 )
-    unless defined $key;
+pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
+pod2usage( -msg => "missing key", exitval => 2 ) unless defined $key;
 
 my $neb = Nebulous::Client->new(
@@ -52,6 +54,7 @@
 eval {
     $user_copies = $neb->getxattr($key, "user.copies");
+    die ("user.copies not found") unless $user_copies;
 };
-unless(defined($user_copies)) {
+unless(defined($user_copies) and $user_copies ne "") {
     $user_copies = 1;
 }
@@ -74,10 +77,10 @@
 	    $md5sum_uniq{$md5sums[$i]} = 1;
 	    close($fh);
-	}
-	else {
+	} else {
 	    $existance[$i] = 0;
 	    $md5sums[$i] = 'NON-EXISTANT';
 	    $md5sum_uniq{$md5sums[$i]} = 1;
 	}
+
 	if (defined($nouris)) {
 	    $validation[$i] = sprintf("% 3d %32s %s",
@@ -85,6 +88,5 @@
 				      $md5sums[$i],
 				      $files[$i]);
-	}
-	else {
+	} else {
 	    $validation[$i] = sprintf("% 3d %32s %s",
 				      $existance[$i],
@@ -112,20 +114,39 @@
 	"instance location:\n", " " x 4,
 	join("\n" . " " x 4, @validation), "\n";
+} else {
+    
+    if (defined($nouris)) {
+	my @files = map {URI->new($_)->file if $_} @$instances;
+
+	print
+	    "object id:             ", @$stat[0], "\n",
+	    "key:                   ", @$stat[1], "\n",
+	    "read lock:             ", @$stat[2], "\n",
+	    "write lock:            ", @$stat[3], "\n",
+	    "epoch:                 ", @$stat[4], "\n",
+	    "mtime:                 ", @$stat[5], "\n",
+	    "available instances:   ", @$stat[6], "\n",
+	    "requested instances:   ", $user_copies, "\n",
+	    "total instances:       ", @$stat[7], "\n",
+	    
+	    "instance location:\n", " " x 4,
+	    join("\n" . " " x 4, @files), "\n";
+    } else {
+	print
+	    "object id:             ", @$stat[0], "\n",
+	    "key:                   ", @$stat[1], "\n",
+	    "read lock:             ", @$stat[2], "\n",
+	    "write lock:            ", @$stat[3], "\n",
+	    "epoch:                 ", @$stat[4], "\n",
+	    "mtime:                 ", @$stat[5], "\n",
+	    "available instances:   ", @$stat[6], "\n",
+	    "requested instances:   ", $user_copies, "\n",
+	    "total instances:       ", @$stat[7], "\n",
+	    
+	    "instance location:\n", " " x 4,
+	    join("\n" . " " x 4, @$instances), "\n";
+    }
+
 }
-else {
-    print
-	"object id:             ", @$stat[0], "\n",
-	"key:                   ", @$stat[1], "\n",
-	"read lock:             ", @$stat[2], "\n",
-	"write lock:            ", @$stat[3], "\n",
-	"epoch:                 ", @$stat[4], "\n",
-	"mtime:                 ", @$stat[5], "\n",
-	"available instances:   ", @$stat[6], "\n",
-	"requested instances:   ", $user_copies, "\n",
-	"total instances:       ", @$stat[7], "\n",
-	
-	"instance location:\n", " " x 4,
-	join("\n" . " " x 4, @$instances), "\n";
-}
 
 
@@ -150,5 +171,5 @@
 =over 4
 
-=item * --server|-s <URL>
+=item  --server|-s <URL>
 
 URL of the Nebulous server to connect to.
@@ -156,9 +177,9 @@
 Optional if the appropriate environment variable is set.
 
-=item * --validate|-v
+=item --validate|-v
 
 Optional - prints out additional information including md5sum.
 
-=item * --files
+=item --files
 
 Optional - prints out disk files instead of uris during validate stage.
@@ -174,5 +195,5 @@
 =over 4
 
-=item * C<NEB_SERVER>
+=item C<NEB_SERVER>
 
 Equivalent to --server|-s
Index: trunk/Nebulous/lib/Nebulous/Client.pm
===================================================================
--- trunk/Nebulous/lib/Nebulous/Client.pm	(revision 40680)
+++ trunk/Nebulous/lib/Nebulous/Client.pm	(revision 41172)
@@ -209,5 +209,5 @@
         },
     );
-
+    
     $log->debug( "entered - @_" );
 
@@ -310,4 +310,5 @@
     my $stats = $self->stat($key);
     my $instances = $stats->[6];
+
     if ((not defined $instances) or ($instances == 0)) {
         $self->set_err("can not cull - no instances");
@@ -315,8 +316,7 @@
         return;
     }
-    if ($instances < $min_copies) {
+    if ($instances <= $min_copies) {
         $self->set_err("can not cull - not enough instances");
         $log->debug("leaving");
-
         return;
     }
@@ -334,7 +334,5 @@
 
     unless (defined $locations) {
-        $self->set_err( "no instances" );
         $log->debug( "leaving" );
-
         return;
     }
@@ -351,4 +349,5 @@
 	}
     }
+
     # If we don't have two copies on one volume, see if we have two copies on a single 
     # cabinet. If so, delete the first of those copies.
@@ -362,4 +361,5 @@
 	}
     }
+
     # Fail-safe. We didn't have any duplicates (the instances are "well-mixed"), so 
     # delete the first.
@@ -403,13 +403,23 @@
     eval {
         # first - strip off any inaccesible instances
+	# these are not counted in the 'removed' number
         if (not defined $self->prune($key)) {
             # use the prune() method to also determine if $key is valid
             die "invalid key: $key";
         }
-
-        # check to see if there is an instance on $vol_name that should be the
-        # sole survivor
+	
+        # check to see if there is an instance on $vol_name that
+        # should be the sole survivor.  This is a bit dangerous : what
+        # if the only instance is on an invalid volume?  what if an
+        # invalid volume is specified?
+
         if (defined $vol_name) {
             $locations = $self->find_instances($key, $vol_name);
+            if (not defined $locations) {
+                die "no instances on requested node";
+            }
+            if (scalar @$locations == 0) {
+                die "no instances on requested node";
+            }
         }
 
@@ -418,17 +428,26 @@
 
         if (defined $locations) {
-            my $instances = $self->find_instances($key, undef, 'find them all');
-            if (scalar @$instances == 1) {
-                # only one instance nothing to do
-                return 0;
-            }
-            if (scalar @$instances == 0) {
+	    print "dev: looking for undef volume instances\n";
+	    # first delete the valid instances (except the one we own)
+            my $instances_valid = $self->find_instances($key, undef, 0); # find all valid instances
+            if (not defined $instances_valid) {
+		# there should have been at least one valid instance?
                 die "no instances";
             }
-            foreach my $victim (@$instances) {
-                next if $victim eq $locations->[0];
-                $self->delete_instance($key, $victim);
-                $removed++;
+            if (scalar @$instances_valid == 0) {
+		# there should have been at least one valid instance?
+                die "no instances";
             }
+
+	    # we have more than one instance, delete the others
+            if (scalar @$instances_valid > 1) {
+		foreach my $victim (@$instances_valid) {
+		    next if $victim eq $locations->[0];
+		    $self->delete_instance($key, $victim);
+		    $removed ++;
+		}
+	    }
+	    # the prune above should have deleted any invalid instances, so we only
+	    # need to worry about the valid instances;
         } else {
             # nuke whatever
@@ -448,5 +467,4 @@
             }
         }
-
     };
     if ($@) {
@@ -663,4 +681,17 @@
     if ($response->fault) {
         $self->set_err($response->faultstring);
+
+	## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+	    die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+	## invalid object key
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+	    die "invalid xattr key $name : fails 'xattr is in user. namespace' test\n";
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -690,4 +721,22 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
+	## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+	    die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+	## invalid xattr key (not in user. namespace)
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+	    die "xattr $name is invalid : xattr is in user. namespace\n";
+        }
+
+	## invalid xattr key
+        if ($response->faultstring =~ /user\..*? does not exist/) {
+            $log->debug( "leaving" );
+	    die "xattr key $name not found : xattr $name does not exist\n";
+        }
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -747,4 +796,23 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
+	## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+	    die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+	## invalid xattr key (not in user. namespace)
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+	    die "xattr $name is invalid : xattr is in user. namespace\n";
+        }
+
+	## invalid xattr key
+        if ($response->faultstring =~ /user\..*? does not exist/) {
+            $log->debug( "leaving" );
+	    die "xattr key $name not found : xattr $name does not exist\n";
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -865,5 +933,5 @@
     unless(defined($params[0])) {
 	$params[0] = hostname() . ".0";
-#	print STDERR "Setting host to $params[0]\n";
+	# print STDERR "Setting host to $params[0]\n";
     }
 
@@ -871,4 +939,5 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
         # check to see if this failure is because $key doesn't exist
         if ($response->faultstring =~ /is valid object key/) {
@@ -878,4 +947,14 @@
         # key is valid but no instances are on the specified volume
         if ($response->faultstring =~ /no instances on storage volume/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        # check to see if this failure is volume is unknown
+        if ($response->faultstring =~ /is not a valid volume name/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        # key is valid but no instances are on the specified volume
+        if ($response->faultstring =~ /no instances available for key/) {
             $log->debug( "leaving" );
             return;
@@ -949,5 +1028,7 @@
         $self->set_err($response->faultstring);
         # check to see if this failure is because $key doesn't exist
+	my $string = $response->faultstring;
         if ($response->faultstring =~ /is valid object key/) {
+	    $self->set_err("invalid object key");
             $log->debug( "leaving" );
             return;
@@ -955,4 +1036,5 @@
         # key is valid but no instances are on the specified volume
         if ($response->faultstring =~ /no instances on storage volume/) {
+	    $self->set_err("no instances on storage volume or volume is not available");
             $log->debug( "leaving" );
             return;
@@ -972,4 +1054,5 @@
 
 
+## this is used by PS:IPP:Config nebulous functions
 sub find
 {
@@ -1014,5 +1097,4 @@
     return $path;
 }
-
 
 sub open
@@ -1124,8 +1206,8 @@
     $log->debug( "entered - @_" );
 
-
     my $locations;
     if ($invalid) {
-	$locations = $self->find_instances( $key, 'any', 'find them all');
+	# delete the invalid instances:
+	$locations = $self->find_instances( $key, 'any', 1);
     }
     else {
@@ -1315,4 +1397,8 @@
         }
 
+        if ($response->faultstring =~ /is valid object key/) {
+	    $log->logdie("parameter #1 to delete_instance did not pass the 'is valid object key' callback");
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -1477,4 +1563,15 @@
 }
 
+sub set_log_level
+{
+    my $self = shift;
+
+    my $log_level = shift;
+
+    $log->level($log_level);
+
+    return $self;
+}
+
 
 1;
Index: trunk/Nebulous/lib/Nebulous/Util.pm
===================================================================
--- trunk/Nebulous/lib/Nebulous/Util.pm	(revision 40680)
+++ trunk/Nebulous/lib/Nebulous/Util.pm	(revision 41172)
@@ -78,4 +78,5 @@
     # The instance file attempting to be opened should always exist as it was
     # created by the Nebulous server.
+
     my $fh;
     for (my $i = 0; $i < 60; $i++) {
@@ -160,5 +161,5 @@
         die "can not process $arg because it is not in name:value form"
             unless defined $name and defined $value;
-        die "xattr name: $name is not in to the form user.name"
+        die "xattr name: $name is not in the form user.name"
             unless $name =~ /^user\./;
         $neb->setxattr($key, $name, $value, "replace")
Index: trunk/Nebulous/nebclient/tests/tap/INSTALL
===================================================================
--- trunk/Nebulous/nebclient/tests/tap/INSTALL	(revision 40680)
+++ 	(revision )
@@ -1,8 +1,0 @@
-Quick Installation
-
-    ./configure
-    make
-    make check
-    make install
-
-Run "configure --help" for additional options.
Index: trunk/Nebulous/nebclient/tests/tap/INSTALL.txt
===================================================================
--- trunk/Nebulous/nebclient/tests/tap/INSTALL.txt	(revision 41172)
+++ trunk/Nebulous/nebclient/tests/tap/INSTALL.txt	(revision 41172)
@@ -0,0 +1,8 @@
+Quick Installation
+
+    ./configure
+    make
+    make check
+    make install
+
+Run "configure --help" for additional options.
Index: trunk/Nebulous/nebclient/tests/tests.c
===================================================================
--- trunk/Nebulous/nebclient/tests/tests.c	(revision 40680)
+++ trunk/Nebulous/nebclient/tests/tests.c	(revision 41172)
@@ -16,9 +16,31 @@
     if (foo) nebFree(foo)
 
+int show_instances (nebServer *server, char *key) {
+
+  nebObjectInstances *locations = NULL;
+
+  locations = nebFindInstances(server, key, NULL);
+
+  for (int i = 0; i < locations->n; i++) {
+    fprintf (stderr, "# loc %d : %s\n", i, locations->URI[i]);
+  }
+  nebObjectInstancesFree(locations);
+  return 1;
+}
+
+int count_instances (nebServer *server, char *key) {
+
+  nebObjectInstances *locations = NULL;
+  locations = nebFindInstances(server, key, NULL);
+  int Ninstances = locations ? locations->n : 0;
+  nebObjectInstancesFree(locations);
+  return Ninstances;
+}
+
 int main (int argc, char **argv) {
     nebServer       *server = NULL;
     char            *key = "foobarbaz";
 
-    plan_tests(31);
+    plan_tests(37);
 
     if (getenv("NEB_SERVER")) {
@@ -37,4 +59,13 @@
         filename = nebCreate(server, key, NULL, &URI);
         diag("filename is %s", filename);
+
+	if (!filename) {
+	  fprintf (stdout, "not ok: no files in test server, exiting\n");
+	  exit (2);
+	}
+
+	show_instances (server, key);
+	ok (count_instances (server, key) == 1, "correct instance count"); 
+
         neb_ok(server, filename, "create object");
         ok(URI, "URI not NULL");
@@ -53,8 +84,11 @@
         neb_ok(server, fh > -1, "create new object filehandle");
         ok(URI, "URI not NULL");
+	ok (count_instances (server, key) == 1, "correct instance count"); 
 
         tst_nebFree(URI);
         close(fh);
     }
+
+    show_instances (server, key);
 
     {
@@ -80,12 +114,21 @@
     }
 
-
     neb_ok(server, nebReplicate(server, key, NULL, NULL), "replicate object");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 2, "correct instance count"); 
 
     neb_ok(server, nebOpen(server, key, NEB_WRITE) < 0, "write to object with multiple instances");
+    show_instances (server, key);
 
     neb_ok(server, nebCull(server, key), "cull object");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 1, "correct instance count"); 
+
+    neb_ok(server, nebCull(server, key) == 0, "cannot cull object with one instance");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 1, "correct instance count"); 
 
     neb_ok(server, nebStat(server, key), "stat object");
+    show_instances (server, key);
 
     neb_ok(server, nebLock(server, key, NEB_WRITE), "lock object write");
Index: trunk/Nebulous/scripts/cmdtests.sh
===================================================================
--- trunk/Nebulous/scripts/cmdtests.sh	(revision 41172)
+++ trunk/Nebulous/scripts/cmdtests.sh	(revision 41172)
@@ -0,0 +1,139 @@
+#!/bin/tcsh -f
+
+alias OK '\!*; \
+ if ($status == 0) then \
+   echo -n "OK : " \
+ else \
+   echo -n "NOT OK : " \
+ endif'
+
+alias IS 'if (\!*) then \
+   echo -n "OK : " \
+ else \
+   echo -n "NOT OK : " \
+ endif'
+
+# here is a set of tests of the user-level neb-XXX commands
+
+set myFile = foobar/test01
+set myAltF = foobar/test02
+
+neb-stat $myFile > /dev/null
+if ($status == 0) then
+   neb-rm $myFile
+   echo "# reseting test environment"
+endif
+
+neb-stat $myAltF > /dev/null
+if ($status == 0) then
+   neb-rm $myAltF
+   echo "# reseting test environment"
+endif
+
+OK neb-touch $myFile
+echo "neb-touch file"
+
+set nebname = `neb-ls $myFile`
+IS ($status == 0)
+echo "found nebfile"
+
+IS ("$nebname" == "$myFile")
+echo "name of nebfile"
+
+set diskname = `neb-ls -p $myFile`
+IS ($status == 0)
+echo "found diskname"
+
+IS (-e $diskname)
+echo "found diskfile $diskname"
+
+echo "lorem ipsum" > $diskname
+
+set result = `neb-cat $myFile`
+IS ("$result" == "lorem ipsum")
+echo "neb-cat"
+
+OK neb-stat $myFile > /dev/null
+echo neb-stat
+
+neb-stat $myFile | grep "object id" > /dev/null
+IS ($status == 0)
+echo neb-stat object id
+
+neb-stat $myFile | grep "read lock" > /dev/null
+IS ($status == 0)
+echo neb-stat read lock
+
+neb-stat $myFile | grep "file:" > /dev/null
+IS ($status == 0)
+echo neb-stat file:
+
+OK neb-replicate $myFile
+echo neb-replicate
+
+set Nfile = `neb-stat $myFile | grep "file:" | wc -l`
+IS ($Nfile == 2)
+echo "two copies"
+
+neb-copies $myFile 2
+IS ($status == 0)
+echo neb-copies
+
+set Nc = `neb-copies $myFile`
+IS ("$Nc" == "user.copies:2")
+echo neb-copies
+
+OK neb-cp $myFile $myAltF
+
+set nebname = `neb-ls $myAltF`
+IS ($status == 0)
+echo "found nebfile"
+
+IS ("$nebname" == "$myAltF")
+echo "name of copy"
+
+set result = `neb-cat $myFile`
+IS ("$result" == "lorem ipsum")
+echo "neb-cat"
+
+neb-cull $myFile
+IS ($status)
+echo "neb-cull will not remove instance if only 2 copies"
+
+OK neb-replicate $myFile
+echo "neb-replicate makes a 3rd copy"
+
+set Nfile = `neb-stat $myFile | grep "file:" | wc -l`
+IS ($Nfile == 3)
+echo "three copies"
+
+OK neb-cull $myFile
+echo "neb-cull will remove 3rd copy"
+
+OK neb-rm $myFile
+echo neb-rm
+
+OK neb-rm $myAltF
+echo neb-rm
+
+touch foobar
+OK neb-insert $myFile foobar
+echo neb-insert
+
+OK neb-stat $myFile > /dev/null
+echo neb-stat
+
+neb-stat $myFile | grep "object id" > /dev/null
+IS ($status == 0)
+echo neb-stat object id
+
+OK neb-mv $myFile $myAltF
+echo neb-mv
+
+OK neb-stat $myAltF > /dev/null
+echo neb-stat
+
+neb-stat $myAltF | grep "object id" > /dev/null
+IS ($status == 0)
+echo neb-stat object id
+
Index: trunk/Nebulous/t/50_client_create.t
===================================================================
--- trunk/Nebulous/t/50_client_create.t	(revision 40680)
+++ trunk/Nebulous/t/50_client_create.t	(revision 41172)
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 9;
+plan tests => 13;
 
 use lib qw( ./t ./lib );
@@ -19,4 +19,5 @@
 use URI::Split qw( uri_split );
 
+# this returns the test apache server location
 my $hostport = Apache::Test->config->{ 'hostport' };
 
@@ -30,7 +31,13 @@
 
     my $uri = $neb->create("foo");
+    if (not defined $uri) { die "failure to create basic file (does it already exist?)"; }
+
     my ($scheme, $auth, $path, $query, $frag) = uri_split($uri);
     ok(-e $path, "good filename");
+
     uri_scheme_ok($uri, 'file');
+    
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -48,4 +55,7 @@
     ok(-e $path, "good filename");
     uri_scheme_ok($uri, 'file');
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -63,4 +73,7 @@
     ok(-e $path, "good filename");
     uri_scheme_ok($uri, 'file');
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -74,4 +87,7 @@
     $neb->create("foo");
     is($neb->create("foo"), undef, "object already exists");
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
Index: trunk/Nebulous/t/53_client_cull.t
===================================================================
--- trunk/Nebulous/t/53_client_cull.t	(revision 40680)
+++ trunk/Nebulous/t/53_client_cull.t	(revision 41172)
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 15;
+plan tests => 18;
 
 use lib qw( ./t ./lib );
@@ -21,5 +21,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -37,5 +36,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -43,11 +41,42 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create("foo");
-    $neb->replicate("foo", "node01");
-
-    my $uri = $neb->cull("foo", "node01");
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+    
+    my $uri = $neb->cull("foo", "node02");
 
     ok($uri, "good cull");
     ok(! -e _get_file_path($uri), "file exists");
+}
+
+Test::Nebulous->setup;
+{
+    # key, $volume
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+
+    # cull with soft volume should succeed even if node does not have instance
+    my $uri = $neb->cull("foo", "node03");
+
+    ok($uri, "good cull");
+    ok(! -e _get_file_path($uri), "file exists");
+}
+
+Test::Nebulous->setup;
+{
+    # key, $volume
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+
+    # cull with hard volume should not succeed if node does not have instance
+    my $uri = $neb->cull("foo", "~node03");
+
+    is($uri, undef, "nothing to cull on this node");
 }
 
@@ -112,5 +141,7 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->cull();
+    my $uri = $neb->cull();
+
+    is ($uri, undef, "no params given to cull");
 };
 like( $@, qr/1 - 3 were expected/, "no params" );
@@ -122,5 +153,7 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->cull( 1, 2, 3, 4 );
+    my $uri = $neb->cull( 1, 2, 3, 4 );
+
+    is ($uri, undef, "no params given to cull");
 };
 like( $@, qr/1 - 3 were expected/, "too many params" );
Index: trunk/Nebulous/t/56_client_find_instances.t
===================================================================
--- trunk/Nebulous/t/56_client_find_instances.t	(revision 40680)
+++ trunk/Nebulous/t/56_client_find_instances.t	(revision 41172)
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 29;
+plan tests => 35;
 
 use lib qw( ./t ./lib );
@@ -20,6 +20,80 @@
 my $hostport = Apache::Test->config->{ 'hostport' };
 
-Test::Nebulous->setup;
-
+### XXX EAM : this test suite finds real errors in the implementation.  
+
+## the API is now slightly different (what is the state of the API in the ipp ops code?)
+## find_instances ("key", "volume", find_invalid)
+
+## this returns a result even if we request a hard volume unless the hard volume is not valid
+
+Test::Nebulous->setup;
+Test::Nebulous->cleanup;
+
+### TEST BLOCK for find_invalid
+## Test::Nebulous->setup;
+## {
+##     # volume/key, volume override
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+##     $neb->create( "foo", "~node02" );
+##     $neb->replicate( "foo", "~node01" );
+## 
+##     Test::Nebulous->switch_node_state;
+##     my $tmphost;
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 1); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 0); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02"); # result : no instance
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 1); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 0); # result : no instance
+##     my $locations = $neb->find_instances( "neb:///foo", "node02"); # result : no instances
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 1);
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
+## #   my $locations = $neb->find_instances( "neb:///foo", $tmphost, 0);
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 1);
+## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 0);
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo");
+## #   my $locations = $neb->find_instances( "neb://bebaz/foo");
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo");
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
+## 
+##     foreach my $f (@$locations) {
+## 	print "location: $f\n";
+##     }
+## 
+##     is( scalar @$locations, 1, "found 1" );
+##     like( @$locations[0], qr/file:/, "URIs match" );
+##     ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+## }
+## die "quit";
+
+## Test::Nebulous->setup;
+## eval {
+##     # volume/key, volume override
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+##     $neb->create( "foo", "~node01" );
+## 
+##     my $locations_1 = $neb->find_instances( "neb://~invalid/foo", "~invalid");
+##     is($locations_1, undef, "no instances in invalid node" );
+## 
+##     # EAM: we fail this test because the logic of find_instances has
+##     # changed: as long as there is an instance on a valid node, the
+##     # instance gets returned
+##     my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
+##     is($locations, undef, "no instances on specified volume" );
+## };
+##print "result: $@\n";
+##like( $@, qr/result message/, "my message" );
+
+# find a single instance by raw name (foo)
+Test::Nebulous->setup;
 {
     # key
@@ -36,6 +110,6 @@
 }
 
-Test::Nebulous->setup;
-
+# find a both instances of a duplicated object
+Test::Nebulous->setup;
 {
     # key
@@ -56,5 +130,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -79,4 +152,5 @@
     );
     $neb->create( "foo" );
+    $neb->set_log_level ('DEBUG');
 
     my $locations = $neb->find_instances( "foo", undef );
@@ -103,6 +177,7 @@
 }
 
-Test::Nebulous->setup;
-
+# find instance using hard_volume for wrong node
+# (should still return a valid instance)
+Test::Nebulous->setup;
 {
     # volume/key, volume override
@@ -111,8 +186,52 @@
     );
     $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
 
     my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
 
+    is( scalar @$locations, 1, "found 1" );
+    like( @$locations[0], qr/file:/, "URIs match" );
+    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+#   the API used to return undef if we used a hard_volume but it was not there
+#   is($locations, undef, "no instances on specified volume" );
+}
+
+# find instance using hard_volume for unknown node
+# (should fail)
+Test::Nebulous->setup;
+{
+    # volume/key, volume override
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
+
+    my $locations = $neb->find_instances( "neb://node01/foo", "~nobody" );
     is($locations, undef, "no instances on specified volume" );
+#   is( scalar @$locations, 0, "found 0" );
+#   under Ubuntu (Perl 5.26.1), @$locations evaluates to 0 if $locations is undef,
+#   but not under Gentoo (Perl 5.6.6)
+}
+
+# find instance using hard_volume for unavailable node
+# (should fail)
+Test::Nebulous->setup;
+{
+    # volume/key, volume override
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
+
+    my $locations = $neb->find_instances( "neb://node01/foo", "~node04" );
+
+    is( scalar @$locations, 1, "found 1" );
+    like( @$locations[0], qr/file:/, "URIs match" );
+    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+
+#   the API used to return undef if we used a hard_volume but it was not there
+#   is($locations, undef, "no instances on specified volume" );
 }
 
@@ -153,5 +272,4 @@
 
 Test::Nebulous->setup;
-
 {
     my $neb = Nebulous::Client->new(
@@ -171,5 +289,5 @@
     $neb->find_instances();
 };
-like( $@, qr/1 - 2 were expected/, "no params" );
+like( $@, qr/1 - 3 were expected/, "no params" );
 
 Test::Nebulous->setup;
@@ -180,7 +298,7 @@
     );
 
-    $neb->find_instances( "foo", 'read', 3 );
+    $neb->find_instances( "foo", 'read', 3, 5 );
 };
-like( $@, qr/1 - 2 were expected/, "too many params" );
+like( $@, qr/1 - 3 were expected/, "too many params" );
 
 Test::Nebulous->cleanup;
Index: trunk/Nebulous/t/59_client_delete.t
===================================================================
--- trunk/Nebulous/t/59_client_delete.t	(revision 40680)
+++ trunk/Nebulous/t/59_client_delete.t	(revision 41172)
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 13;
+plan tests => 18;
 
 use lib qw( ./t ./lib );
@@ -20,6 +20,9 @@
 my $hostport = Apache::Test->config->{ 'hostport' };
 
-Test::Nebulous->setup;
-
+# ensure fresh start:
+Test::Nebulous->setup;
+Test::Nebulous->cleanup;
+
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -78,7 +81,6 @@
 }
 
-# force flag
-Test::Nebulous->setup;
-
+# test passing in undefined force flag
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -90,6 +92,6 @@
 }
 
-Test::Nebulous->setup;
-
+# test passing in force flag = 1
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -101,4 +103,70 @@
 }
 
+# test passing in force flag = 0
+Test::Nebulous->setup;
+{
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+
+    is( $neb->delete("foo", 0), 1, "force flag false" );
+}
+
+# test passing in force flag = 0 where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo", 0);
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
+# test passing in force flag = 1 where disk file is removed
+Test::Nebulous->setup;
+{
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    is( $neb->delete("foo", 1), 1, "force flag true deletes file" );
+}
+
+# test passing in undef force flag where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo", undef);
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
+# test passing in undefined force flag where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo");
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
 Test::Nebulous->setup;
 
@@ -109,5 +177,5 @@
     $neb->delete();
 };
-like( $@, qr/1 - 2 were expected/, "no params" );
+like( $@, qr/1 - 3 were expected/, "no params" );
 
 Test::Nebulous->setup;
@@ -127,7 +195,7 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->delete( "foo", 0, 2 );
-};
-like( $@, qr/1 - 2 were expected/, "too many params" );
+    $neb->delete( "foo", 0, 1, 2 );
+};
+like( $@, qr/1 - 3 were expected/, "too many params" );
 
 Test::Nebulous->cleanup;
Index: trunk/Nebulous/t/64_client_find_objects.t
===================================================================
--- trunk/Nebulous/t/64_client_find_objects.t	(revision 40680)
+++ trunk/Nebulous/t/64_client_find_objects.t	(revision 41172)
@@ -21,5 +21,4 @@
 
 Test::Nebulous->setup;
-
 {
     # key
Index: trunk/Nebulous/t/65_client_mounts.t
===================================================================
--- trunk/Nebulous/t/65_client_mounts.t	(revision 40680)
+++ trunk/Nebulous/t/65_client_mounts.t	(revision 41172)
@@ -29,5 +29,5 @@
     my $mounts = $neb->mounts();
 
-    is(scalar @$mounts, 7, "number of rows");
+    is(scalar @$mounts, 8, "number of rows");
 
     my %row;
Index: trunk/Nebulous/t/66_client_xattr.t
===================================================================
--- trunk/Nebulous/t/66_client_xattr.t	(revision 40680)
+++ trunk/Nebulous/t/66_client_xattr.t	(revision 41172)
@@ -21,8 +21,6 @@
 );
 
-# 1 key / xattr
-
-Test::Nebulous->setup;
-
+# setxattr
+Test::Nebulous->setup;
 {
     my $uri = $neb->create('foo');
@@ -46,7 +44,5 @@
 
 # multiple xattrs
-
-Test::Nebulous->setup;
-
+Test::Nebulous->setup;
 {
     my $uri = $neb->create('foo');
@@ -184,5 +180,5 @@
     $neb->getxattr('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
+like($@, qr|xattr user.bar does not exist|,
     "get xattr from non-existant xattr key");
 
@@ -256,5 +252,5 @@
     $neb->removexattr('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
+like($@, qr|xattr user.bar does not exist|,
     "remove xattr from non-existant xattr key");
 
Index: trunk/Nebulous/t/68_client_chmod.t
===================================================================
--- trunk/Nebulous/t/68_client_chmod.t	(revision 40680)
+++ trunk/Nebulous/t/68_client_chmod.t	(revision 41172)
@@ -41,4 +41,7 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path);
 }
 
@@ -53,6 +56,9 @@
     my $mode = $neb->chmod($key, 0440);
 
-    my $st1 = stat(URI->new($uri1)->path);
-    my $st2 = stat(URI->new($uri2)->path);
+    my $path1 = URI->new($uri1)->path;
+    my $path2 = URI->new($uri2)->path;
+
+    my $st1 = stat($path1);
+    my $st2 = stat($path2);
 
     is($st1->mode &07777, 0440, "chmod() first instance");
@@ -60,4 +66,8 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path1);
+    chmod(0600, $path2);
 }
 
Index: trunk/Nebulous/t/70_neb-ls.t
===================================================================
--- trunk/Nebulous/t/70_neb-ls.t	(revision 40680)
+++ trunk/Nebulous/t/70_neb-ls.t	(revision 41172)
@@ -23,4 +23,5 @@
 use Test::Cmd;
 use Nebulous::Client;
+use Nebulous::Util qw( :standard );
 use Test::Nebulous;
 
@@ -28,8 +29,7 @@
 my $neb_url  = "http://$hostport/nebulous";
 
-
 my $cmd = 'bin/neb-ls';
 
-# last ditch effort to make sure dsget is executable
+# last ditch effort to make sure neb-ls is executable
 chmod 0755, 'bin/neb-ls';
 
@@ -37,8 +37,7 @@
 isa_ok($test, 'Test::Cmd');
 
-# NEB_SERVER env var not set
+# test if NEB_SERVER env var not set
 undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
 Test::Nebulous->setup;
-
 {
     $test->run(args => '');
@@ -46,7 +45,6 @@
 }
 
-# NEB_SERVER set
-Test::Nebulous->setup;
-
+# NEB_SERVER set (and used in the following tests)
+Test::Nebulous->setup;
 {
     $ENV{NEB_SERVER} = $neb_url;
@@ -57,13 +55,11 @@
 
 Test::Nebulous->setup;
-
-{
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create('foo');
-
-    $test->run(args => '');
-
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+
+    $test->run(args => '');
     is($? >> 8, 0, "exit code");
     like($test->stdout, qr/^foo$/,      "stdout");
@@ -72,5 +68,4 @@
 
 Test::Nebulous->setup;
-
 {
     my $neb = Nebulous::Client->new(
@@ -83,10 +78,27 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^bar\nfoo$/,  "stdout");
+    like($test->stdout, qr/^bar\s+foo$/,  "stdout");
     like($test->stderr, qr/^$/,         "stderr");
 }
 
 Test::Nebulous->setup;
-
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path( @$locations[0] );
+    my $filestats = `ls -l $diskfile`;
+
+    $test->run(args => '-l');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^$filestats$/,  "stdout");
+    like($test->stderr, qr/^$/,         "stderr");
+}
+
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -96,5 +108,5 @@
     $neb->create('bar');
 
-    $test->run(args => '-l');
+    $test->run(args => '-c');
 
     is($? >> 8, 0, "exit code");
@@ -109,26 +121,10 @@
         proxy => $neb_url,
     );
-    $neb->create('foo');
-    $neb->create('bar');
-
-    $test->run(args => '-1');
-
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^bar\nfoo\n$/,   "stdout");
-    like($test->stderr, qr/^$/,             "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create('a/foo');
-
-    $test->run(args => '');
-
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr||,               "stdout");
+    $neb->create('a/foo');
+
+    $test->run(args => '');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/$|,           "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }
@@ -161,5 +157,5 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr|^a/bar\na/foo$|,  "stdout");
+    like($test->stdout, qr|^a/bar\s+a/foo$|,  "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }
@@ -193,5 +189,5 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr|^a/b/\na/foo$|,        "stdout");
+    like($test->stdout, qr|^a/b/\s+a/foo$|,        "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }
Index: trunk/Nebulous/t/71_client_there_can_be_only_one.t
===================================================================
--- trunk/Nebulous/t/71_client_there_can_be_only_one.t	(revision 40680)
+++ trunk/Nebulous/t/71_client_there_can_be_only_one.t	(revision 41172)
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 8;
+plan tests => 13;
 
 use lib qw( ./t ./lib );
@@ -21,5 +21,37 @@
 
 Test::Nebulous->setup;
+Test::Nebulous->cleanup;
 
+## volume is defined, but it is not valid (should exit)
+## Test::Nebulous->setup;
+## {
+##     my $key = "foo";
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+## 
+##     my $uri = $neb->create($key, "node01");
+## 
+##     $neb->replicate($key, "node02");
+## 
+##     ## set node02 to an invalid state:
+##     Test::Nebulous->switch_node_state;
+## 
+##     # if requested node is invalid, the other instance is kept and the invalid one removed
+##     # but since invalid instances are removed with a prune call, they are not counted
+##     # in the number of items removed (return by the function call)
+##     is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
+##     my $locations = $neb->find_instances( "foo" );
+## 
+##     is( scalar @$locations, 1, "found 1" );
+##     is($locations->[0], $uri, "instance on correct volume" );
+## };
+## is($@, "", "there_can_be_only_one with supplied volume should succeed");
+## die "asdf";
+
+#######
+
+## volume not defined
+Test::Nebulous->setup;
 {
     my $key = "foo";
@@ -37,7 +69,7 @@
 }
 
+## volume is defined, only valid instances
 Test::Nebulous->setup;
-
-{
+eval {
     my $key = "foo";
     my $neb = Nebulous::Client->new(
@@ -45,6 +77,6 @@
     );
     $neb->create($key, "node01");
+
     my $uri = $neb->replicate($key, "node02");
-
     is($neb->there_can_be_only_one($key, "node02"), 1, "there can be only one!");
 
@@ -53,5 +85,30 @@
     is( scalar @$locations, 1, "found 1" );
     is($locations->[0], $uri, "instance on correct volume" );
-}
+};
+is($@, "", "there_can_be_only_one with supplied volume should succeed");
+
+## volume is defined, but it is not valid (should exit)
+Test::Nebulous->setup;
+eval {
+    my $key = "foo";
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    my $uri = $neb->create($key, "node01");
+
+    $neb->replicate($key, "node02");
+
+    ## set node02 to an invalid state:
+    Test::Nebulous->switch_node_state;
+
+    # if requested node is invalid, nothing is removed
+    is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
+
+    my $locations = $neb->find_instances( "foo" );
+
+    is( scalar @$locations, 1, "found 1" );
+    is($locations->[0], $uri, "instance on correct volume" );
+};
+is($@, "", "there_can_be_only_one with supplied volume should succeed");
 
 Test::Nebulous->setup;
Index: trunk/Nebulous/t/72_neb-cull.t
===================================================================
--- trunk/Nebulous/t/72_neb-cull.t	(revision 40680)
+++ trunk/Nebulous/t/72_neb-cull.t	(revision 41172)
@@ -39,6 +39,55 @@
 # NEB_SERVER env var not set
 undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
-Test::Nebulous->setup;
-
+
+## problem test
+## Test::Nebulous->setup;
+## {
+##     $ENV{NEB_SERVER} = $neb_url;
+## 
+##     my $key = 'foo';
+## 
+##     my $neb = Nebulous::Client->new(
+##         proxy => $neb_url,
+##     );
+##     $neb->create($key);
+## 
+##     $test->run(args => "--one_only " . $key);
+## 
+##     ## my $line1 = $test->stdout;
+##     ## print "stdout: $line1\n";
+##     ## 
+##     ## my $line2 = $test->stderr;
+##     ## print "stderr: $line2\n";
+## 
+##     is($neb->stat($key)->[6], 1, "correct # of instances");
+##     is($? >> 8, 255, "exit code");
+##     like($test->stdout, qr/^$/, "stdout");
+##     like($test->stderr, qr/no instances/, "stderr");
+## }
+## die "stop";
+
+## Test::Nebulous->setup;
+## {
+##     my $key = 'foo';
+## 
+##     my $neb = Nebulous::Client->new(
+##         proxy => $neb_url,
+##     );
+##     $neb->create($key);
+##     $neb->replicate($key);
+## 
+##     $test->run(args => "--server $neb_url $key");
+## 
+##     my $line = $test->stderr;
+##     print "stderr: $line\n";
+## 
+##     is($neb->stat($key)->[6], 1, "correct # of instances");
+##     is($? >> 8, 0, "exit code");
+##     like($test->stdout, qr/^$/, "stdout");
+##     like($test->stderr, qr/^$/, "stderr");
+## }
+## die "TEST";
+
+Test::Nebulous->setup;
 {
     $test->run(args => '');
@@ -48,5 +97,4 @@
 # NEB_SERVER set
 Test::Nebulous->setup;
-
 {
     $ENV{NEB_SERVER} = $neb_url;
@@ -57,54 +105,57 @@
 
 Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+
+    $test->run(args => "--server $neb_url $key");
+#   $test->run(args => $key);
+
+    is($neb->stat($key)->[6], 1, "correct # of instances");
+    is($? >> 8, 255, "exit code");
+    like($test->stdout, qr/^$/, "stdout");
+    like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+    $neb->replicate($key);
 
     $test->run(args => $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 255, "exit code");
-    like($test->stdout, qr/^$/, "stdout");
-    like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
+    my $line = $test->stderr;
+
+    # the default min_copies is 2:
+    is($neb->stat($key)->[6], 2, "correct # of instances");
+    is($? >> 8, 255, "exit code");
+    like($test->stdout, qr/^$/, "stdout");
+    like($test->stderr, qr/not enough instances/, "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+    $neb->replicate($key);
     $neb->replicate($key);
 
     $test->run(args => $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^$/, "stdout");
-    like($test->stderr, qr/^$/, "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
-    $neb->replicate($key);
-    $neb->replicate($key);
-
-    $test->run(args => $key);
-
     is($neb->stat($key)->[6], 2, "correct # of instances");
     is($? >> 8, 0, "exit code");
@@ -113,4 +164,5 @@
 }
 
+## problem test
 Test::Nebulous->setup;
 
@@ -202,5 +254,6 @@
 
     is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 0, "exit code");
+
+    is($? >> 8, 255, "exit code");
     like($test->stdout, qr/^$/, "stdout");
     like($test->stderr, qr/not enough instances/, "stderr");
@@ -221,5 +274,5 @@
     $test->run(args => "--min_copies 2 " . $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
+    is($neb->stat($key)->[6], 2, "correct # of instances");
     is($? >> 8, 0, "exit code");
     like($test->stdout, qr/^$/, "stdout");
Index: trunk/Nebulous/t/conf/extra.conf.in
===================================================================
--- trunk/Nebulous/t/conf/extra.conf.in	(revision 40680)
+++ trunk/Nebulous/t/conf/extra.conf.in	(revision 41172)
@@ -3,7 +3,54 @@
 # this file will be Include-d by @ServerRoot@/httpd.conf
 
+# need to add MPM?
+
+# Conflicts: mpm_worker mpm_prefork
+#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
+#LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
+LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
+#LoadModule cgid_module /usr/lib/apache2/modules/mod_cgid.so
+LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
+LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
+
+# event MPM
+# StartServers: initial number of server processes to start
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestWorkers: maximum number of worker threads
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+<IfModule mpm_event_module>
+	StartServers			 2
+	MinSpareThreads		 25
+	MaxSpareThreads		 75
+	ThreadLimit			 64
+	ThreadsPerChild		 25
+	MaxRequestWorkers	  150
+	MaxConnectionsPerChild   0
+</IfModule>
+
+<IfModule mod_alias.c>
+	<IfModule mod_cgi.c>
+#	Define ENABLE_USR_LIB_CGI_BIN
+	</IfModule>
+
+	<IfModule mod_cgid.c>
+#	Define ENABLE_USR_LIB_CGI_BIN
+	</IfModule>
+
+	<IfDefine ENABLE_USR_LIB_CGI_BIN>
+		ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/
+		<Directory "/usr/lib/cgi-bin">
+			AllowOverride None
+			Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+			Require all granted
+		</Directory>
+	</IfDefine>
+</IfModule>
+
+# nebulous location:
 <Location /nebulous>
     SetHandler perl-script
-    PerlResponseHandler Apache2::SOAP
+    PerlResponseHandler Apache::SOAP
     PerlSetVar dispatch_to "PerlHandler Nebulous::Server::SOAP"
     PerlSetVar options "compress_threshold => 10000"
Index: trunk/Nebulous/t/conf/startup.pl.in
===================================================================
--- trunk/Nebulous/t/conf/startup.pl.in	(revision 40680)
+++ trunk/Nebulous/t/conf/startup.pl.in	(revision 41172)
@@ -5,4 +5,24 @@
 use lib qw( @ServerRoot@ );
 
+use ModPerl::Util (); #for CORE::GLOBAL::exit
+
+use Apache2::RequestRec ();
+use Apache2::RequestIO ();
+use Apache2::RequestUtil ();
+
+use Apache2::ServerRec ();
+use Apache2::ServerUtil ();
+use Apache2::Connection ();
+use Apache2::Log ();
+
+use APR::Table ();
+
+use ModPerl::Registry ();
+
+use Apache2::Const -compile => ':common';
+use APR::Const -compile => ':common';
+
+use Apache::SOAP;
+
 use Apache::DBI;
 use DBI;
@@ -10,9 +30,23 @@
 use Nebulous::Server::Apache;
 use Nebulous::Server;
-use Test::Nebulous;
+
+# do not inherit here, let's be explicit
+# use Test::Nebulous;
+
+# EAM : making this match Nebulous-Server/docs/apache2-mod_perl-startup.pl:
+
+# use the real mysql server:
+#my $dsn         = 'DBI:mysql:database=nebulous:host=localhost,mysql_socket=8500';
+#my $dbuser      = 'nebulous';
+#my $dbpasswd    = '';
+
+# use the test mysql server (defined in Test::Nebulous)
+my $dsn         = 'DBI:mysql:database=test:host=localhost:mysql_socket=/var/run/mysqld/mysqld.sock';
+my $dbuser      = 'test';
+my $dbpasswd    = '';
 
 #$Apache::DBI::DEBUG = 1;
-Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
-Apache::DBI->setPingTimeOut($NEB_DB, 10);
+#Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
+Apache::DBI->setPingTimeOut($dsn, 10);
 
 my $config = Nebulous::Server::Config->new(
@@ -22,14 +56,8 @@
 $config->add_db(
     dbindex     => 0,
-    dsn         => $NEB_DB,
-    dbuser      => $NEB_USER,
-    dbpasswd    => $NEB_PASS,
+    dsn         => $dsn,
+    dbuser      => $dbuser,
+    dbpasswd    => $dbpasswd,
 );
-#$config->add_db(
-#    dbindex     => 1,
-#    dsn         => 'DBI:mysql:database=nebulous1:host=localhost',
-#    dbuser      => $dbuser,
-#    dbpasswd    => $dbpasswd,
-#);
 
 Nebulous::Server::SOAP->new_on_init($config);