Changeset 41172

Timestamp:

Nov 27, 2019, 12:04:14 PM (7 years ago)

Author:

eugene

Message:

merging updates to Nebulous from EAM dev branch ipp-20191011

Location:

trunk

Files:

• Nebulous-Server/Build.PL (modified) (2 diffs)
• Nebulous-Server/MANIFEST (modified) (1 diff)
• Nebulous-Server/bin/neb-voladd (modified) (1 diff)
• Nebulous-Server/docs/apache2-mod_perl-startup.pl (modified) (1 diff)
• Nebulous-Server/docs/install.txt (modified) (1 diff)
• Nebulous-Server/lib/Nebulous/Server.pm (modified) (60 diffs)
• Nebulous-Server/lib/Nebulous/Server/SQL.pm (modified) (1 diff)
• Nebulous-Server/lib/Test/Nebulous.pm (modified) (6 diffs)
• Nebulous-Server/scripts/mkdirs.mana (copied) (copied from branches/eam_branches/ipp-20191011/Nebulous-Server/scripts/mkdirs.mana )
• Nebulous-Server/scripts/stest.sh (copied) (copied from branches/eam_branches/ipp-20191011/Nebulous-Server/scripts/stest.sh )
• Nebulous-Server/t/03_server_setup.t (modified) (1 diff)
• Nebulous-Server/t/04_server_create_object.t (modified) (5 diffs)
• Nebulous-Server/t/05_server_replicate_object.t (modified) (6 diffs)
• Nebulous-Server/t/08_server_find_instances.t (modified) (10 diffs)
• Nebulous-Server/t/11_server_is_valid_volume_name.t (modified) (2 diffs)
• Nebulous-Server/t/15_server_xattr.t (modified) (1 diff)
• Nebulous-Server/t/16_mounts.t (modified) (3 diffs)
• Nebulous-Server/t/18_server_chmod_object.t (modified) (3 diffs)
• Nebulous-Server/t/20_server_find_objects_wildcard.t (copied) (copied from branches/eam_branches/ipp-20191011/Nebulous-Server/t/20_server_find_objects_wildcard.t )
• Nebulous/Build.PL (modified) (1 diff)
• Nebulous/bin/neb-cat (modified) (3 diffs)
• Nebulous/bin/neb-cp (modified) (2 diffs)
• Nebulous/bin/neb-cull (modified) (3 diffs)
• Nebulous/bin/neb-insert (modified) (1 diff)
• Nebulous/bin/neb-ls (modified) (5 diffs)
• Nebulous/bin/neb-migrate (modified) (1 diff)
• Nebulous/bin/neb-repair (modified) (7 diffs)
• Nebulous/bin/neb-replicate (modified) (2 diffs)
• Nebulous/bin/neb-shift (modified) (1 diff)
• Nebulous/bin/neb-stat (modified) (8 diffs)
• Nebulous/lib/Nebulous/Client.pm (modified) (22 diffs)
• Nebulous/lib/Nebulous/Util.pm (modified) (2 diffs)
• Nebulous/nebclient (modified) (1 prop)
• Nebulous/nebclient/tests/tap (modified) (1 prop)
• Nebulous/nebclient/tests/tap/INSTALL (deleted)
• Nebulous/nebclient/tests/tap/INSTALL.txt (copied) (copied from branches/eam_branches/ipp-20191011/Nebulous/nebclient/tests/tap/INSTALL.txt )
• Nebulous/nebclient/tests/tests.c (modified) (4 diffs)
• Nebulous/scripts/cmdtests.sh (copied) (copied from branches/eam_branches/ipp-20191011/Nebulous/scripts/cmdtests.sh )
• Nebulous/t/50_client_create.t (modified) (6 diffs)
• Nebulous/t/53_client_cull.t (modified) (6 diffs)
• Nebulous/t/56_client_find_instances.t (modified) (10 diffs)
• Nebulous/t/59_client_delete.t (modified) (7 diffs)
• Nebulous/t/64_client_find_objects.t (modified) (1 diff)
• Nebulous/t/65_client_mounts.t (modified) (1 diff)
• Nebulous/t/66_client_xattr.t (modified) (4 diffs)
• Nebulous/t/68_client_chmod.t (modified) (3 diffs)
• Nebulous/t/70_neb-ls.t (modified) (11 diffs)
• Nebulous/t/71_client_there_can_be_only_one.t (modified) (5 diffs)
• Nebulous/t/72_neb-cull.t (modified) (6 diffs)
• Nebulous/t/conf/extra.conf.in (modified) (1 diff)
• Nebulous/t/conf/startup.pl.in (modified) (3 diffs)

trunk/Nebulous-Server/Build.PL

@@ -43 +43 @@
     },
     script_files        => [qw(
-        bin/neb-admin
-        bin/neb-fsck
         bin/neb-cabadd
         bin/neb-voladd
@@ -54 +52 @@
     )],
 )->create_build_script;
+
+# these scripts are no longer installed (2019.10.31)
+#       bin/neb-admin
+#       bin/neb-fsck

trunk/Nebulous-Server/MANIFEST

@@ -4 +4 @@
 MANIFEST
 README
-bin/neb-admin
-bin/neb-fsck
-bin/neb-initdb
 bin/neb-voladd
 bin/neb-cabadd

trunk/Nebulous-Server/bin/neb-voladd

@@ -43 +43 @@
 
 my $dbh = DBI->connect(
+    # if you want to work with the localhost database but get socket path errors,
+    # you should set the server host as 127.0.0.1 (not localhost)
+
+    # the following tells DBI to read the mysql/my.cnf files
+    #  "DBI:mysql:database=$db:host=$dbhost:mysql_read_default_group=dbd_mysql",
+
     "DBI:mysql:database=$db:host=$dbhost",
     $dbuser,

trunk/Nebulous-Server/docs/apache2-mod_perl-startup.pl

@@ -28 +28 @@
 use Nebulous::Server;
 
-my $dsn         = 'DBI:mysql:database=nebulous:host=ipp019.ifa.hawaii.edu';
+my $dsn         = 'DBI:mysql:database=nebulous:host=nebulous.ipp.ifa.hawaii.edu';
 my $dbuser      = 'nebulous';
-my $dbpasswd    = '@neb@';
+my $dbpasswd    = 'XXXX';
 
 #$Apache::DBI::DEBUG = 1;
 #Apache::DBI->connect_on_init( $dsn, $dbuser, $dbpasswd );
 Apache::DBI->setPingTimeOut($dsn, 10);
-Nebulous::Server::SOAP->new_on_init(
+
+my $config = Nebulous::Server::Config->new(
+    trace       => 'warn',
+                                           );
+$config->add_db(
+    dbindex     => 0,
     dsn         => $dsn,
     dbuser      => $dbuser,
     dbpasswd    => $dbpasswd,
-    log_level   => 'all',
-);
+                );
+
+Nebulous::Server::SOAP->new_on_init($config);
+
 
 1;

trunk/Nebulous-Server/docs/install.txt

@@ -55 +55 @@
     export NEB_PASS=@neb@
 
+    *** this is very dangerous and should only be run in test examples:
     neb-initdb
 
-    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous
-    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous
-    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous
-    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous
-    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous
+    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous --mountpoint /data/ipp000.0/nebulous
+    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous --mountpoint /data/ipp000.1/nebulous
+    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous --mountpoint /data/ipp002.0/nebulous
+    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous --mountpoint /data/ipp003.0/nebulous
+    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous --mountpoint /data/ipp022.0/nebulous
 
     # or in bulk:

trunk/Nebulous-Server/lib/Nebulous/Server.pm

@@ -13 +13 @@
 use base qw( Class::Accessor::Fast );
 
+use Carp;
+
 use Cache::Memcached;
 use DBI;
+
+## for gentoo:
 use Digest::SHA1 qw( sha1_hex );
+
+## for ubuntu:
+# use Digest::SHA qw( sha1_hex );
+
 use Fcntl ':mode';
 use File::Basename qw( basename dirname fileparse );
@@ -77 +85 @@
         Cache::Memcached->new({
             servers => $config->memcached_servers,
-       })
-    );
+                              })
+        );
 #    $self->cache->set("foo", "bar") or die "set failed";
 #    $self->cache->get("foo") or die "get failed";
@@ -215 +223 @@
 }
 
+my $get_storage_volume_calls = 0;
+
 sub create_object
 {
@@ -252 +262 @@
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
-    if (defined $vol_name
-        and not $self->_is_valid_volume_name($key, $key->volume)) {
+    if (defined $vol_name and not $self->_is_valid_volume_name($key, $key->volume)) {
         unless ($key->hard_volume) {
             $log->warn( "$vol_name is not a known volume name" );
@@ -262 +271 @@
     }
         
+    $get_storage_volume_calls = 0;
+
     my ($vol_id, $vol_host, $vol_path, $vol_xattr)
         = $self->_get_storage_volume($key, $vol_name, $key->hard_volume);
@@ -273 +284 @@
                 # create storage_object
                 my $query = $db->prepare_cached( $sql->new_object ); 
-                $query->execute('NULL', $key->path, basename($key->path), $parent_id);
+# bad syntax    $query->execute('NULL', $key->path, basename($key->path), $parent_id);
+                $query->execute(0, $key->path, basename($key->path), $parent_id);
+                $query->finish;
             }
 
@@ -292 +305 @@
                 my $query = $db->prepare_cached( $sql->new_object_attr ); 
                 $query->execute($so_id);
+                $query->finish;
             }
 
@@ -300 +314 @@
                 my $query = $db->prepare_cached( $sql->new_object_instance );
                 $query->execute($vol_id);
+                $query->finish;
             }
 
@@ -327 +342 @@
                 # vol_id, uri, ins_id
                 $query->execute($vol_id, "$uri", $ins_id);
+                $query->finish;
             }
 
@@ -421 +437 @@
                     }
                     $query->finish;
+                    $db->commit;
                 }
 
@@ -441 +458 @@
                     my $query = $db->prepare_cached($sql->new_directory);
                     $query->execute($dir, $parent_id);
+                    $query->finish;
+                    $db->commit;
                 }
 
@@ -452 +471 @@
                     ($parent_id) = $query->fetchrow_array;
                     $query->finish;
+                    $db->commit;
                 }
                 die("failed to get LAST_INSERT_ID()")
                     unless $parent_id;
-
-                $db->commit;
             }
         };
@@ -712 +730 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -726 +737 @@
 
     # if a volume name is explicity specified then we should make the
-    # replication onto that volume (even if there is alread an instance on that
+    # replication onto that volume (even if there is already an instance on that
     # volume) if at all possible and throw an error if we can not.
+
     # if a volume name IS NOT specified then we should make the replication
     # onto any (hopefully the best) volume that DOES NOT already have an
-    # instance on it.  If all avilable volume already have an instance on them
+    # instance on it.  If all available volumes already have an instance on them
     # then we should throw an error 
+
     # volume names implied as part of the key are *IGNORED* as the source and
     # *SHOULD NOT* be used as the destination either
 
     eval {
-        $key = parse_neb_key($key);
+        # EAM 2019.10.31 : if I supply dest_vol_name here, then
+        # $key->volume will point at that volume and the section below
+        # will correctly be testing the validity of dest_vol_name
+        $key = parse_neb_key($key, $dest_vol_name);
     };
     $log->logdie("$@") if $@;
@@ -742 +758 @@
     my $db  = $self->db($key);
 
+    ## Old comment:
     # puke if the source volume is bogus, we may want to actually use this as
     # the instance to be copied later
-    if (defined $key->volume
-        and not $self->_is_valid_volume_name($key, $key->volume)) {
+
+    # EAM 2019.10.31 : in the past, the key-implied volume was
+    # supplied here, which was inconsistent since above it says the
+    # key-implied volume is ignored. By supplying $dest_vol_name to
+    # parse_neb_key above, the key->volume now refers to the
+    # dest_volume
+    if (defined $key->volume and not $self->_is_valid_volume_name($key, $key->volume)) {
         unless ($key->hard_volume) {
             $log->warn($key->volume . " not a known volume name");
@@ -752 +774 @@
         }
     }
-    # puke if the source volume is bogus, we may want to actually use this as
-    # the instance to be copied later
-    if (defined $dest_vol_name
-        and not $self->_is_valid_volume_name($key, $dest_vol_name)) {
-           $log->logdie($key->volume . " is not a valid volume name");
-    }
-        
+
+    # puke if the destination volume is bogus
+    ## XXX if I supply dest_vol_name to parse_neb_key above, this section below is redundant with the above
+    ## if (defined $dest_vol_name
+    ##     and not $self->_is_valid_volume_name($key, $dest_vol_name)) {
+    ##        $log->logdie($key->volume . " is not a valid volume name");
+    ## }
+
+    $get_storage_volume_calls = 0;
+
     my ($vol_id, $vol_host, $vol_path, $vol_xattr);
     if (defined $dest_vol_name) {
+        # if we supplied dest_vol_name, then the call to parse_neb_key above
+        # will have set $key->volume to that value and hard_volume to match
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
-            = $self->_get_storage_volume($key, $dest_vol_name);
+            = $self->_get_storage_volume($key, $key->volume, $key->hard_volume);
     } else {
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
             = $self->_get_replication_volume($key);
     }
+
+    # print "selected target: $vol_id, $vol_host\n";
 
     my $uri;
@@ -784 +813 @@
                 $so_id = $query->fetchrow_hashref->{ 'so_id' };
                 $query->finish;
+                $db->commit;
             }
 
@@ -800 +830 @@
                 # time LAST_INSERT_ID() is invoked
                 $query->finish;
+                $db->commit;
             }
 
@@ -841 +872 @@
     eval {
         my $mode = $self->getxattr_object("$key", 'user.mode');
-        if (defined $mode) {
-            $self->chmod_object("$key", $mode);
+        if (defined $mode && $mode ne "") {
+            $self->chmod_object("$key", $mode);
         }
     };
@@ -852 +883 @@
 
     $log->debug("leaving");
-
     return "$uri";
 }
@@ -1323 +1353 @@
             die( "xattr $key:$name does not exist" );
         }
+
         # if we go more then one row bad something very bad has happened.
         unless ($rows == 1) {
@@ -1335 +1366 @@
         $value = $row->{ 'value' };
     };
+    $db->commit;
+
     if ($@) {
         if ($@ =~ /user\..*? does not exist/) {
             # do not log xattr does not exist messages
+            # NOTE: the client is using the reported message to interpret the errors
             die $@;
         }
@@ -1386 +1420 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->debug("leaving");
@@ -1442 +1477 @@
                 die( "affected row count is $rows instead of 1" );
             }
-
             $db->commit;
             $log->debug("commit");
@@ -1545 +1579 @@
         $work_dir = $pattern;
         $file_pattern = '%';
-    }
-    else {
+    } else {
         my $dir_plain = dirname($pattern);
         if ($dir_plain eq 'neb:') {
@@ -1575 +1608 @@
         my $query = $db->prepare_cached( $sql->find_dir_by_parent_id . " AND dirname LIKE ? ");
         $query->execute( $dir_id, $file_pattern );
+        $db->commit;
 
         while ( my $row = $query->fetchrow_hashref ) {
@@ -1595 +1629 @@
         my $query = $db->prepare_cached( $sql->find_object_by_dir_id . " AND ext_id_basename LIKE ? ");
         $query->execute( $dir_id , $file_pattern);
+        $db->commit;
 
         while ( my $row = $query->fetchrow_hashref ) {
@@ -1624 +1659 @@
     my $sql = $self->sql;
     my $db  = $self->_db_for_index($index);
-
 
     # first check to see if the key is an exact match
@@ -1645 +1679 @@
         $log->logdie("database error: $@");
     }
+    $db->commit;
 
     if (scalar @keys) {
@@ -1678 +1713 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     # find files under dir
@@ -1692 +1728 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->debug( "leaving" );
@@ -1753 +1790 @@
 }
 
+# sub find_instances
 sub find_instances_old
 {
@@ -1769 +1807 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -1787 +1818 @@
     my $sql = $self->sql;
 
-#    unless ($key) {
-#        $log->warn("key was undefined after validate_pos(), trying again...");
-#        return $self->find_instances(@_);
-#    }
+    # default value for find_invalid is false
+    if (not defined $find_invalid) { $find_invalid = 0; }
 
     # vol_name overrides the key implied volume
@@ -1828 +1857 @@
             my $rows;
             # ext_id, available
-            if (defined($find_invalid)) {
+            if (defined($find_invalid) and $find_invalid) {
+                # XXX returns instances which are NOT available
                 $rows = $query->execute($key->path, 0);
             }
@@ -1865 +1895 @@
 }
 
-#sub find_instances_by_proximity
+# sub find_instances_by_proximity
 sub find_instances
 {
@@ -1882 +1912 @@
         {
             type        => SCALAR|UNDEF,
-#            callbacks   => {
-#                # check that the volume name requested is valid
-#                'is valid volume name' => sub {
-#                    return 1 if not defined $_[0];
-#                    $self->_is_valid_volume_name($_[0])
-#                },
-#            },
             optional    => 1,
         },
@@ -1897 +1920 @@
         }, 
     );
-
     my $sql = $self->sql;
 
-#    unless ($key) {
-#        $log->warn("key was undefined after validate_pos(), trying again...");
-#        return $self->find_instances(@_);
-#    }
+    if (not defined $find_invalid) { $find_invalid = 0; }
+
+    if ($find_invalid eq "find them all") {
+        $log->logdie("old neb-cull --one_only, neb-rm --invalid both disabled by new Server.pm");
+    }
 
     # vol_name overrides the key implied volume
@@ -1932 +1955 @@
             }
         }
-
-
     }
 
@@ -1972 +1993 @@
             # ext_id, name, available
             # host_vol_id host_cab_id host_site_id ext_id available
-            my $rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 1);
+
+            my $rows;
+            if (defined($find_invalid) and $find_invalid) {
+                # returns instances which are NOT available (volume not available)
+                $rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 0);
+            } else {
+                $rows = $query->execute($h_vol_id, $h_cab_id, $h_site_id, $key->path, 1);
+            }           
             unless ($rows > 0) {
                 $query->finish;
-                die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name");
+                die("no instances on storage volume or volume is not available for key: $key volume: $vol_name");
             }
         } else {
             $query = $db->prepare_cached( $sql->get_object_instances );
+            # ext_id, available
+
             my $rows;
-            # ext_id, available
-            if (defined($find_invalid)) {
+            if (defined($find_invalid) and $find_invalid) {
+                # returns instances which are NOT available (volume not available)
                 $rows = $query->execute($key->path, 0);
-            }
-            else {
+            } else {
                 $rows = $query->execute($key->path, 1);
             }
@@ -1992 +2021 @@
             }
         }
+        $db->commit;
+        ## if we do not call commit here, the transaction stays
+        ## open blocking some operations below. 
 
         while (my $row = $query->fetchrow_hashref) {
             my $instance = $row->{ 'uri' };
             push @locations, $instance if $instance;
+
+            # Carp::carp ("instance 2: $row->{ 'uri' }, $row->{ 'vol_id' }, $row->{ 'cab_id' }, $row->{ 'vol_idx' }\n");
         }
     };
@@ -2003 +2037 @@
         if (defined $vol_name and not defined $key->hard_volume) {
             $log->debug("retrying with 'any' volume");
-            return $self->find_instances($key->path, 'any');
+            return $self->find_instances($key->path, 'any', $find_invalid);
         }
         $log->logdie("database error: $@");
@@ -2018 +2052 @@
 }
 
+# this method returns instances on the specified volume
+# or a list of all available instances
+# it does NOT choose instances based on proximity
+# and it does NOT return invalid instances
 sub find_instances_for_cull
 {
@@ -2061 +2099 @@
     my $db  = $self->db($key);
 
+    ## XXX this method is missing the alias deference section (see find_instances)
+    ## if aliases are intended to be compute nodes in the same cabinet / site as
+    ## the storage nodes, then this is not needed.
+
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
@@ -2082 +2124 @@
             unless ($rows > 0) {
                 $query->finish;
-                die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name");
+                die("no instances on storage volume or volume is not available for key: $key volume: $vol_name");
             }
         } else {
@@ -2105 +2147 @@
         $db->rollback;
         # handle soft volumes
-        if (defined $vol_name and not defined $key->hard_volume) {
+        # we need to call find_intances_for_cull here so the returned structure 
+        # is the same (find_instances just returns an array of strings)
+        if (defined $vol_name and not defined $key->hard_volume and ($vol_name ne "any")) {
             $log->debug("retrying with 'any' volume");
-            return $self->find_instances($key->path, 'any');
+            return $self->find_instances_for_cull($key->path, 'any');
         }
         $log->logdie("database error: $@");
     }
+    $db->commit;
 
     # XXX remove this?
@@ -2222 +2267 @@
                 }
             }
-
             $db->commit;
             $log->debug("commit");
@@ -2282 +2326 @@
         $stat = $query->fetchrow_arrayref;
         $query->finish;
+        $db->commit;
     };
     $log->logdie("database error: $@") if $@;
@@ -2318 +2363 @@
     };
     $log->logdie("database error: $@") if $@;
+    $db->commit;
 
     $log->logdie("no mounted volumes found") unless (scalar @$stats);
@@ -2415 +2461 @@
     my ($key, $name, $hard_volume) = @_;
     
-#    $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<");
+    # track the number of calls to this function and
+    # give up after 10 attempts
+    $get_storage_volume_calls ++;
+
+#   $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<");
     my $sql = $self->sql;
     my $db  = $self->db($key);
@@ -2427 +2477 @@
             # %free, name, avaiable, allocate
             $rows = $query->execute($max_used_space, $name, 1, 1);
-            # XXX destinguish between non-existant and unavailable
+            # XXX distinguish between non-existant and unavailable
             unless ($rows > 0) {
                 $query->finish;
+                $db->commit;
 
                 # if a volume name was specified, and is soft, and we failed to
@@ -2442 +2493 @@
             if ($rows > 1) {
                 $query->finish;
+                $db->commit;
                 die("affected row count is $rows instead of 1");
             }
@@ -2449 +2501 @@
             $rows = $query->execute($max_used_space, 1, 1, $topfew_count);
 #           $log->warn("Storage_volume: $rows $topfew_count");
-            # there has to be atleast one storage volume
+            # there has to be at least one storage volume
             unless ($rows > 0) {
                 $query->finish;
-                die("no storage volume is available for key: $key volume: $name hard_volume: $hard_volume");
+                $db->commit;
+                # prevent failure in the die due to undefined variables
+                my $hard_volume_str = defined $hard_volume ? $hard_volume : "undefined";
+                my $name_str = defined $name ? $name : "undefined";
+                die("no storage volume is available for key: $key volume: $name_str hard_volume: $hard_volume_str");
             }
         }
@@ -2465 +2521 @@
             $log->error($@);
             $log->debug("retrying...");
-            return $self->_get_storage_volume(@_);
+            if ($get_storage_volume_calls < 10) {
+                return $self->_get_storage_volume(@_);
+            }
+            # else
+            $log->logdie("no available storage volume: $@");
         } 
         # else
@@ -2502 +2562 @@
         unless ($rows > 0) {
             $query->finish;
-            die("Requested key $key does not exist");
+            die("Requested key $key does not exist (or cabinet is undefined)");
         }
         if ($rows == 1) {
@@ -2513 +2573 @@
             }
             $query->finish;
-        }
-        else {
+        } else {
+            # if instances are spread across multiple cabinets, allow the next instance to go anywhere
+            # NOTE: this does not prevent the new instance from going to the same volume as one of the existing instances
             $forbidden_cabinet = 0;
             $forbidden_site    = 0;
             $query->finish;
         }
-            
+        $db->commit;
 
         $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id );
-        # ext_id, %free, avaiable, allocate
+        # ext_id, %free, available, allocate
+
         $rows = $query->execute($key->path, $max_used_space, 1, 1, $forbidden_cabinet, $forbidden_site, $topfew_count);
-        # XXX destinguish between non-existant and unaviable
+        # XXX distinguish between non-existant and unavailable
         unless ($rows > 0) {
             $query->finish;
@@ -2544 +2606 @@
         ($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array;
         $query->finish;
+        $db->commit;
     };
     $log->logdie("database error: $@") if $@;
@@ -2590 +2653 @@
         ($ext_id) = $query->fetchrow_array;
         $query->finish;
+        $db->commit;
+        ## if we do not call commit here, the transaction stays open blocking some operations below
     };
     if ($@) {
@@ -2612 +2677 @@
 }
 
-
+# in the past, _is_valid_volume_name returned 'undef'.  
+# now it either returns 1 (success) or 0 (failure)
 sub _is_valid_volume_name
 {
@@ -2650 +2716 @@
         ($vol_id, $vol_path, $free) = $query->fetchrow_array;
         $query->finish;
+        $db->commit;
     };
     $log->logdie("database error: $@") if $@;

trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm

@@ -574 +574 @@
 DROP TABLE IF EXISTS directory;
 DROP TABLE IF EXISTS deleted;
+DROP TABLE IF EXISTS cabinet;
+DROP TABLE IF EXISTS aliasvol;
+DROP TABLE IF EXISTS lost_instances;
 SET FOREIGN_KEY_CHECKS=1
 END

trunk/Nebulous-Server/lib/Test/Nebulous.pm

@@ -19 +19 @@
 our @EXPORT = qw( $NEB_DB $NEB_USER $NEB_PASS $NEB_MEMCACHED_SERVERS);
 
-our $NEB_DB     = $ENV{'NEB_DB'}   || "DBI:mysql:database=test:host=localhost";
-our $NEB_USER   = $ENV{'NEB_USER'} || "test";
-our $NEB_PASS   = $ENV{'NEB_PASS'} || '';
+# require use of the test database server:
+
+our $NEB_DB     = "DBI:mysql:database=test:host=localhost:mysql_socket=/var/run/mysqld/mysqld.sock";
+our $NEB_USER   = "test";
+our $NEB_PASS   = '';
 our $NEB_MEMCACHED_SERVERS = ['127.0.0.1:11211'];
 
@@ -35 +37 @@
 my $dir6 = "";
 my $dir7 = "";
+my $dir8 = "";
+
+sub show_setup {
+    my $self = shift;
+
+    print "DB: $NEB_DB, USER: $NEB_USER, PASS: $NEB_PASS\n";
+}
+
 
 sub setup {
@@ -49 +59 @@
     $dir6 = tempdir( CLEANUP => 0 );
     $dir7 = tempdir( CLEANUP => 0 );
+    $dir8 = tempdir( CLEANUP => 0 );
 
     foreach my $statement (@{ $sql->get_db_schema }) {
@@ -54 +65 @@
     }
 
-    # node01/node02: allocate = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 1);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1);
+    # generate 2 cabinets so we can replicate to a valid location
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab01", 1, "main") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab02", 1, "main") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab03", 2, "back") });
+    $dbh->do(qq{ INSERT INTO cabinet (name, site_id, location) values ("cab04", 2, "back") });
 
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 2);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2);
+    # node01 : allocate = TRUE (cab01, site 1)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (1, 1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e7, note FROM volume WHERE vol_id = ? }, undef, 1);
 
-    # node03: allocate = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 3);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3);
+    # node02 : allocate = TRUE (cab02, site 1)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (2, 2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 2);
+
+    # node03: allocate = TRUE (cab03, site 2)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (3, 3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 3);
 
     # node04: allocate = FALSE, available = FALSE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (4, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 4);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (4, 1, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 4);
 
     # node05: allocate = FALSE, available = TRUE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (5, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 5);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (5, 1, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 5);
 
     # node06: allocate = TRUE, available = FALSE
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (6, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 6);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (6, 1, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 6);
 
     # node07: full
-    $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (7, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7);
-    $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e10 FROM volume WHERE vol_id = ? }, undef, 7);
-#    $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7);
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (7, 1, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e10, note FROM volume WHERE vol_id = ? }, undef, 7);
+    
+    # node08: allocate = TRUE, (cab04, site 2)
+    $dbh->do(qq{ INSERT INTO volume (vol_id, cab_id, name, host, path, mountpoint, allocate, available) VALUES (8, 4, 'node08', 'node08', ?, '/', TRUE, TRUE) }, undef, $dir8);
+    $dbh->do(qq{ INSERT INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, 10e10, 10e8, note FROM volume WHERE vol_id = ? }, undef, 8);
 
-#    $dbh->do(qq{ call getmountedvol() });
+#   $dbh->do(qq{ call getmountedvol() });
 }
 
 sub cleanup {
     # memcached needs to be emptied between test runs
-        Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS)->flush_all;
+    Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS)->flush_all;
     # if flush_all() turns out to be useless as feared
 #   my $memd = Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS);
@@ -106 +121 @@
 #   $query->finish;
 
+    
     foreach my $statement (@{ $sql->get_db_clear }) {
         $dbh->do( $statement );
@@ -118 +134 @@
     rmtree([$dir6], 0, 1) if -e $dir6;
     rmtree([$dir7], 0, 1) if -e $dir7;
+    rmtree([$dir8], 0, 1) if -e $dir8;
+}
+
+# function to change state of one node for tests:
+sub switch_node_state {
+    my $self = shift;
+
+    $dbh->do(qq{ UPDATE volume set available = FALSE where name = 'node02'});
+    $dbh->do(qq{ UPDATE mountedvol set available = FALSE where name = 'node02'});
+}
+
+sub get_node_dirs {
+    my $self = shift;
+
+    return ($dir1, $dir2, $dir3, $dir4, $dir5, $dir6, $dir7, $dir8);
 }
 

trunk/Nebulous-Server/t/03_server_setup.t

@@ -14 +14 @@
 use Nebulous::Server;
 use Test::Nebulous;
+
+Test::Nebulous->show_setup;
+die "test";
 
 Test::Nebulous->setup;

trunk/Nebulous-Server/t/04_server_create_object.t

@@ -30 +30 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -40 +39 @@
     uri_scheme_ok($uri, 'file');
 
-SKIP: {
-    skip "requires xattr support", 1 unless $test_xattr;
-    is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
-}
-}
-
-Test::Nebulous->setup;
-
+  SKIP: {
+      skip "requires xattr support", 1 unless $test_xattr;
+      is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
+    }
+}
+
+Test::Nebulous->setup;
 {
     # key
@@ -57 +55 @@
     uri_scheme_ok($uri, 'file');
 
-SKIP: {
-    skip "requires xattr support", 1 unless $test_xattr;
-    is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
-}
+  SKIP: {
+      skip "requires xattr support", 1 unless $test_xattr;
+      is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr');
+    }
 }
 
@@ -528 +526 @@
     $neb->create_object("foo", '~node07');
 };
-like($@, qr/node07 is not available/, "request volume this is full");
+like($@, qr/node07 is not available/, "request volume which is full");
 
 Test::Nebulous->setup;
@@ -535 +533 @@
     $neb->create_object("neb://~node07/foo");
 };
-like($@, qr/node07 is not available/, "request volume this is full");
+like($@, qr/node07 is not available/, "request volume which is full");
 
 Test::Nebulous->setup;

trunk/Nebulous-Server/t/05_server_replicate_object.t

@@ -8 +8 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 25;
+use Test::More tests => 43;
 
 use lib qw( ./t ./lib );
@@ -28 +28 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -45 +44 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -62 +60 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume is undef
@@ -79 +76 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -147 +143 @@
 }
 
+# hard_volume tests:
+
+# replication with hard_volume:
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object: hard_volume as node target
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    like($loc2, qr/$nodedirs[1]/, "instance matches expected node");
+}
+
+# replication to unknown volume: should succeed
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "dummy");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    uri_scheme_ok($loc2, 'file');
+}
+
+# replication to unknown volume with hard_volume: should fail
+Test::Nebulous->setup;
+eval {
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~dummy");
+
+    my $locations = $neb->find_instances("foo");
+};
+like($@, qr/is not a valid volume name/, "replicate to unknown node with hard_volume should fail");
+
+# replication to unavailable volume: should succeed
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "node04");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+    uri_scheme_ok($loc2, 'file');
+}
+
+# replication to unavailable volume with hard_volume: should fail
+Test::Nebulous->setup;
+eval {
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node04");
+
+    my $locations = $neb->find_instances("foo");
+};
+like($@, qr/is not available/, "replicate to unknown node with hard_volume should fail");
+
+### END hard_volume tests
+
+### location-aware tests (do we respect the cab_id / vol_id exclusions?)
+
+# here are the volume definitions (Test/Nebulous.pm):
+# name   : vol_id : cab_id : site_id : xattr
+# node01 :      1 :      1 :       1 :     0
+# node02 :      2 :      2 :       1 :     0
+# node03 :      3 :      3 :       2 :     0
+# node04 :      4 :      1 :       1 :     0
+# node05 :      5 :      1 :       1 :     0
+# node06 :      6 :      1 :       1 :     0
+# node07 :      7 :      1 :       1 :     0
+# node08 :      8 :      4 :       2 :     0
+
+# replication without volume 
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object: hard_volume as node target
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    # if the volume is not explicitly specified, then the first copy
+    # should go to a different volume, cabinet, and site
+    # if first is one node01, replication should go to node03 or node08
+    # since those both have different site_id values
+    like($loc1, qr/$nodedirs[0]/, "instance matches expected node");
+
+    my $isnode03 = $loc2 =~ m/$nodedirs[2]/;
+    my $isnode08 = $loc2 =~ m/$nodedirs[7]/;
+    ok($isnode03 || $isnode08, "instance matches expected node");
+}
+
+# replication without volume
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # checking on replicate_object, does this logic make sense?
+    my $uri1 = $neb->create_object("foo", "~node03");
+    my $uri2 = $neb->replicate_object("foo");
+
+    my $locations = $neb->find_instances("foo");
+    my $Nloc = @$locations;
+
+    is($Nloc, 2, "found 2 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+
+    # if the volume is not explicitly specified, then the first copy
+    # should go to a different volume, cabinet, and site
+    # if first is one node03, replication should go to node01 or node02
+    # since those both have different site_id values
+    like($loc1, qr/$nodedirs[2]/, "instance matches expected node");
+
+    my $isnode01 = $loc2 =~ m/$nodedirs[0]/;
+    my $isnode02 = $loc2 =~ m/$nodedirs[1]/;
+    ok($isnode01 || $isnode02, "instance matches expected node");
+}
+
 Test::Nebulous->setup;
 

trunk/Nebulous-Server/t/08_server_find_instances.t

@@ -8 +8 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 19;
+use Test::More tests => 44;
 
 use lib qw( ./t ./lib );
@@ -22 +22 @@
 );
 
-Test::Nebulous->setup;
-
+# XXX previously, the logic was: if we request a hard volume location
+# using ~node, then we would give an error if no data is available except on that node.
+
+# the code has been modified so that the requested storage object is provided if possible, 
+# even if the apparently "hard" target did not exist.
+
+Test::Nebulous->setup;
+Test::Nebulous->cleanup; # make sure the database is reset before running any tests
+
+# for (my $i = 0; $i < $Nloc; $i++) {
+#       my $loc = $locations->[$i];
+#       print "location: $loc\n";
+#}
+
+Test::Nebulous->setup;
 {
     # key
@@ -35 +48 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -49 +61 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -61 +72 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -73 +83 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -85 +94 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -98 +106 @@
 }
 
-# object exists but instance is on a different volume
-Test::Nebulous->setup;
-
+# request from a hard_volume returns an instance which is not on the hard volume if available
+Test::Nebulous->setup;
 eval {
     # key, volume
     my $uri = $neb->create_object('foo', '~node01');
-
     my $locations = $neb->find_instances('foo', '~node02');
-};
-like($@, qr/no instances on storage volume/, 'instances on a different volume');
+
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on different node (as hard_volume) returned");
+
+# request from a hard_volume returns an instance which is not on the hard volume if available
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node01');
+    my $locations = $neb->find_instances('foo', '~node02');
+
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on different node (as hard_volume) returned");
+
+### TESTS FOR INSTANCE ON INVALID NODE
+diag ("TESTS FOR INSTANCE ON INVALID NODE");
+
+# request from a soft_volume for an instance which is on an invalid node (find_invalid = 1) [SOFT, 1]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02', 1);
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on invalid node (as soft_volume) returned with find_invalid = 1");
+
+# request from a soft_volume for an instance which is on an invalid node (find_invalid = 0) [SOFT, 0]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02', 0);
+};
+like($@, qr/database error/, "instance on invalid node (as soft_volume) not returned (find_invalid = 0)");
+
+# request from a hard_volume for an instance which is on an invalid node (find_instance not supplied) [SOFT, X]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', 'node02');
+};
+like($@, qr/no instances available for key/, "instance on invalid node (as soft_volume) not returned (find_invalid not set)");
+
+# request from a hard_volume for an instance which is on an invalid node (find_invalid = 1) [HARD, 1]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02', 1);
+    uri_scheme_ok($locations->[0], 'file');
+    is($uri, $locations->[0], "URIs match");
+};
+is($@, "", "instance on invalid node (as hard_volume) returned with find_invalid = 1");
+
+# request from a hard_volume for an instance which is on an invalid node (find_invalid = 0) [HARD, 0]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02', 0);
+};
+like($@, qr/no instances on storage volume/, "instance on invalid node (as hard_volume) not returned (find_invalid = 0)");
+
+# request from a hard_volume for an instance which is on an invalid node [HARD, X]
+Test::Nebulous->setup;
+eval {
+    # key, volume
+    my $uri = $neb->create_object('foo', '~node02');
+    Test::Nebulous->switch_node_state;
+
+    my $locations = $neb->find_instances('foo', '~node02');
+};
+like($@, qr/no instances on storage volume/, "instance on invalid node (as hard_volume) not returned (find_invalid not set)");
+
+#### TESTING the instance proximity analysis
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # place 4 instances on specific nodes (hard targets works):
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+    my $uri3 = $neb->replicate_object("foo", "~node03");
+    my $uri4 = $neb->replicate_object("foo", "~node08");
+
+    my $locations = $neb->find_instances("foo", "node01");
+    my $Nloc = @$locations;
+
+    is($Nloc, 4, "found 4 instances");
+    
+    my $loc1 = $locations->[0];
+    my $loc2 = $locations->[1];
+    my $loc3 = $locations->[2];
+    my $loc4 = $locations->[3];
+
+    # choosing instance relative to node01 yields:
+    like($loc1, qr/$nodedirs[0]/, "node01 is closest to node01");    
+    like($loc2, qr/$nodedirs[1]/, "node02 is 2nd closest to node01");
+    like($loc3, qr/$nodedirs[2]/, "node03 is 3rd closest to node01");
+    like($loc4, qr/$nodedirs[7]/, "node08 is 4th closest to node01");
+}
+
+#### TESTING the instance proximity analysis
+Test::Nebulous->setup;
+{
+    my @nodedirs = Test::Nebulous->get_node_dirs();
+
+    # place 4 instances on specific nodes (hard targets works):
+    my $uri1 = $neb->create_object("foo", "~node01");
+    my $uri2 = $neb->replicate_object("foo", "~node02");
+    my $uri3 = $neb->replicate_object("foo", "~node03");
+    my $uri4 = $neb->replicate_object("foo", "~node08");
+
+    my $locations = $neb->find_instances("foo", "node03");
+    my $Nloc = @$locations;
+
+    is($Nloc, 4, "found 4 instances");
+    
+    my $loc1 = $locations->[0]; # 
+    my $loc2 = $locations->[1]; # 
+    my $loc3 = $locations->[2]; # 
+    my $loc4 = $locations->[3]; # 
+
+    # choosing instance relative to node01 yields:
+    like($loc1, qr/$nodedirs[2]/, "node03 is closest to node03");
+    like($loc2, qr/$nodedirs[7]/, "node08 is 2nd closest to node03");
+    like($loc3, qr/$nodedirs[1]/, "node02 is 3rd closest to node03");
+    like($loc4, qr/$nodedirs[0]/, "node08 is 4th closest to node03");
+}
+
+########
 
 Test::Nebulous->setup;
@@ -125 +277 @@
 
 Test::Nebulous->setup;
-
-eval {
-    $neb->find_instances();
-};
-like($@, qr/1 - 2 were expected/, "no params");
+eval {
+    my $locations = $neb->find_instances();
+};
+like($@, qr/1 - 3 were expected/, "no params");
+# note the new API expects 1-3 parameters
 
 Test::Nebulous->setup;
@@ -135 +287 @@
 eval {
     $neb->create_object('foo');
-    $neb->find_instances('foo', 'node01', 3);
-};
-like($@, qr/1 - 2 were expected/, "too many params");
+    $neb->find_instances('foo', 'node01', 3, 5);
+};
+like($@, qr/1 - 3 were expected/, "too many params");
+# note the new API expects 1-3 parameters
 
 Test::Nebulous->cleanup;

trunk/Nebulous-Server/t/11_server_is_valid_volume_name.t

@@ -8 +8 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 3;
+use Test::More tests => 5;
 
 use lib qw( ./t ./lib );
@@ -31 +31 @@
 Test::Nebulous->setup;
 
-is($neb->_is_valid_volume_name('foo', 'node99'), undef, "invalid volume name");
+is($neb->_is_valid_volume_name('foo', 'node99'), 0, "invalid volume name");
+
+Test::Nebulous->setup;
+
+# key does not need to exist (is hashed to select db server)
+ok($neb->_is_valid_volume_name('bax', 'node01'), "valid volume name");
+
+Test::Nebulous->setup;
+
+# 'any' should always be a valid name
+ok($neb->_is_valid_volume_name('foo', 'any'), "valid volume name");
 
 Test::Nebulous->cleanup;

trunk/Nebulous-Server/t/15_server_xattr.t

@@ -184 +184 @@
     $neb->getxattr_object('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
-    "get xattr from non-existant xattr key");
+like($@, qr|xattr neb:///foo:user.bar does not exist|, "get xattr from non-existant xattr key");
 
 Test::Nebulous->setup;

trunk/Nebulous-Server/t/16_mounts.t

@@ -28 +28 @@
     my $mounts = $neb->mounts();
 
-    is(scalar @$mounts, 7, "number of rows");
+    is(scalar @$mounts, 8, "number of rows");
 
     my %row;
@@ -35 +35 @@
         = @{$mounts->[0]};
     
-    is($row{total},     100000000000);
-    is($row{used},      100000000);
-    is($row{vol_id},    1);
-    is($row{name},      "node01");
-    is($row{host},      "node01");
-    is($row{allocate},  1);
-    is($row{available}, 1);
-    is($row{xattr},     0);
+    is($row{total},     100000000000, "total bytes");
+    is($row{used},      100000000, "used bytes");
+    is($row{vol_id},    1, "vol id");
+    is($row{name},      "node01", "name");
+    is($row{host},      "node01", "host");
+    is($row{allocate},  1, "is allocated");
+    is($row{available}, 1, "is available");
+    is($row{xattr},     0, "has no xattr");
 
     # 2nd row
@@ -48 +48 @@
         = @{$mounts->[1]};
 
-    is($row{total},     100000000000);
-    is($row{used},      1000000000);
-    is($row{vol_id},    2);
-    is($row{name},      "node02");
-    is($row{host},      "node02");
-    is($row{allocate},  1);
-    is($row{available}, 1);
-    is($row{xattr},     0);
+    is($row{total},     100000000000, "total bytes");
+    is($row{used},      1000000000, "used bytes");
+    is($row{vol_id},    2, "vol id");
+    is($row{name},      "node02", "name");
+    is($row{host},      "node02", "host");
+    is($row{allocate},  1, "is allocated");
+    is($row{available}, 1, "is available");
+    is($row{xattr},     0, "has no xattr");
 }
 

trunk/Nebulous-Server/t/18_server_chmod_object.t

@@ -38 +38 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path);
 }
 
@@ -50 +53 @@
     my $mode = $neb->chmod_object($key, 0440);
 
-    my $st1 = stat(URI->new($uri1)->path);
-    my $st2 = stat(URI->new($uri2)->path);
+    my $path1 = URI->new($uri1)->path;
+    my $path2 = URI->new($uri2)->path;
+
+    my $st1 = stat($path1);
+    my $st2 = stat($path2);
 
     is($st1->mode &07777, 0440, "chmod() first instance");
@@ -57 +63 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod (0600, $path1);
+    chmod (0600, $path2);
 }
 

trunk/Nebulous/Build.PL

@@ -96 +96 @@
         'Test::Nebulous'            => 0,
         'Test::Cmd'                 => '1.05',
-        'Apache2::SOAP'             => 0,
+# gentoo: 'Apache2::SOAP'             => 0,
+        'Apache::SOAP'             => 0, # needed instead for ubuntu 18.04
         'Apache::DBI'               => '1.05',
         'Apache::Test'              => '1.27',  # bundles Apache::TestMB

trunk/Nebulous/bin/neb-cat

@@ -18 +18 @@
 use constant CHUNK_SIZE => 1024;
 
-my ($server);
+my ($server, $help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -24 +24 @@
 GetOptions(
     'server|s=s'    => \$server,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
 
-pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 )
-        unless defined $server;
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
+
+pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 ) unless defined $server;
 
 my $neb = Nebulous::Client->new(
@@ -95 +98 @@
 =over 4
 
-=item * --server|-s <URL>
+=item --server|-s <URL>
 
 URL of the Nebulous server to connect to.

trunk/Nebulous/bin/neb-cp

@@ -16 +16 @@
 use Pod::Usage qw( pod2usage );
 
-my ($volume, $server);
+my ($volume, $server, $help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -23 +23 @@
     'volume=s'      => \$volume,
     'server|s=s'    => \$server,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 my $src = shift;

trunk/Nebulous/bin/neb-cull

@@ -17 +17 @@
 
 my ($min_copies, $server, $volume, $one_only);
-my ($soft_volume);
+my ($soft_volume, $help);
 $server = $ENV{'NEB_SERVER'} unless $server;
 
@@ -26 +26 @@
     'one_only|o'    => \$one_only,
     'soft_volume'   => \$soft_volume,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 my $key = shift;
@@ -59 +63 @@
 # print "@cull_args\n";
 
-
 if ($one_only) {
     my $status = $neb->there_can_be_only_one(@cull_args);
+    if (not defined $status) {
+        die "not enough instances of Nebulous key: $key";
+    }
     if ($status == 0) {
         die "not enough instances of Nebulous key: $key";

trunk/Nebulous/bin/neb-insert

@@ -95 +95 @@
 =head1 SYNOPSIS
 
-    neb-insert --server <neb-server> <neb-key> <local-file> [--copies <n>] [--destroy]
+    neb-insert [--server <neb-server>] <neb-key> <local-file> [--copies <n>] [--destroy]
 
 #=head1 DESCRIPTION

trunk/Nebulous/bin/neb-ls

@@ -19 +19 @@
     $ls,
     $column,
+    $help
 );
 
 $server = $ENV{'NEB_SERVER'} unless $server;
-
 
 GetOptions(
@@ -29 +29 @@
     'l'             => \$ls,
     'column|c'      => \$column,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
+
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
 
 # twiddle column so that it does it by default.
@@ -35 +39 @@
 $column = not $column;
 
+if ($ls) { $path = 1; }
+
 my $pattern = shift;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --server", -exitval => 2 )
-    unless $server;
+pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
 
 my $neb = Nebulous::Client->new(
@@ -96 +101 @@
 =head1 SYNOPSIS
 
-    neb-ls [--server <URL>] [-c] [-p] <pattern>
+    neb-ls [--server <URL>] [-c] [-p] [-l] <pattern>
 
 =head1 DESCRIPTION
 
 This program list Nebulous keys matched by C<<pattern>>.  Call it with no
-arguments is equivalent to searching with the pattern C<neb://>.  SQL like
+arguments is equivalent to searching with the pattern C<neb://>.  SQL-like
 wildcards are supported to select out certain files.
 
@@ -108 +113 @@
 =over 4
 
-=item * --path|-p
+=item --path|-p
 
 Find the disk files corresponding to the keys found.
 
-=item * --column|-c
+=item --column|-c
 
 Disable column formatting, and output results one to a line.
 
+=item --l|-l
+
+Give disk file information using ls (implies -p).
+
 Optional
 
-=item * --server|-s <URL>
+=item --server|-s <URL>
 
 URL of the Nebulous server to connect to.

trunk/Nebulous/bin/neb-migrate

@@ -23 +23 @@
 
 $server = $ENV{'NEB_SERVER'} unless $server;
+
+die "this tool is probably too inefficient unless used very carefully";
 
 GetOptions(

trunk/Nebulous/bin/neb-repair

@@ -45 +45 @@
 );
 
-die "can't connected to Nebulous Server: $server"
+die "can't connect to Nebulous Server: $server"
     unless defined $neb;
 
 my $stat = $neb->stat($key);
 die "nebulous key: $key not found" unless $stat;   
+
 my $instances = $neb->find_instances($key, 'any');
 die "no instances found" unless $instances;   
@@ -71 +72 @@
 
 for (my $i = 0; $i <= $#files; $i++) {
+    # count existing files and get the MD5 sums
     if (-e $files[$i]) {
         $existance[$i] = 1;
@@ -89 +91 @@
         push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i];
         $last_f = $files[$i];
-    }
-    else {
+    } else {
         $existance[$i] = 0;
         $md5sums[$i] = 'NON-EXISTANT';
@@ -98 +99 @@
 }
 
+# check number of copies and number of unique MD5 sum values
 my $N_mds = scalar(keys(%md5sum_uniq));
 my $N_files = scalar(@files);
+
+# seems like we should not call neb-replicate until *after* we have
+# removed 0-byte copies
 if ($N_files < $user_copies) {
     print "Insufficient copies ($N_files) of $key, replicating!\n";
     system("neb-replicate $key");
 }
+
 if ($N_mds == 1) {
     print "Nebulous key $key is consistent.\n";
-}
-else {
+} else {
     my $valid_sum = '';
     my $valid_file = '';
     my $m;
     foreach $m (keys %md5sum_valid) {
-        if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) {
+        if (($m ne 'd41d8cd98f00b204e9800998ecf8427e') && ($m ne 'NON-EXISTANT')) {
             if ($valid_sum ne '') {
                 die "Too many acceptable files for key $key\n";
@@ -122 +127 @@
     if ($valid_sum eq '') {
         die "No acceptable file for key $key\n";
-    }
-    else {
+    } else {
         print "Repairing instances\n";
         foreach $m (keys %md5sum_valid) {
@@ -130 +134 @@
                     print "\tcp $valid_file $f\n";
                     system("cp $valid_file $f");
+
+                    # if we failed to make a copy, the target ($f) could be inaccessible
                     if ($? >> 8 != 0) {
                         if ($existance[0] == 1) {
-                            # We can fix this with culls and replications.
+
+                            # The target is a valid file 0-byte file
+                            # but we cannot replace it with the
+                            # correct file.  We can fix this with
+                            # culls and replications.
+
                             print "Attempting to fix this.\n";
                             print("neb-replicate $key\n");
                             system("neb-replicate $key");
                             if ($? >> 8 == 0) {
+                                # if the neb-replicate succeeded, remove the bad copy with neb-cull
                                 my $bad_volume = (split /\//, $f)[2];
                                 print("neb-cull --volume $bad_volume $key\n");
                                 system("neb-cull --volume $bad_volume $key");
-                            }
-                            else {
+                            } else {
                                 die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n";
                             }
-                        }
-                        else {
+                        } else {
                             die "Failed to copy file $valid_file to $f!\n";
                         }
@@ -187 +197 @@
 =head1 DESCRIPTION
 
-This program repairs instances and corrects instance counts for a nebulous key.  
+This program repairs instances and corrects instance counts for a nebulous key:
+
+If some of the existing instances have the MD5 sum corresponding to a
+0-byte file, but the rest have the same, non-zero byte md5sum, then
+replace the 0-byte files with the non-zero byte version.
+
+At the end, if the number of instances is less than the value of
+user.copies, then additional copies are generated.
 
 =head1 OPTIONS

trunk/Nebulous/bin/neb-replicate

@@ -42 +42 @@
 );
 
+# Nebulous::Client programs can use this to set the log level: 
+# $neb->set_log_level ('DEBUG');
+
 die "can't connected to Nebulous Server: $server"
     unless defined $neb;
@@ -50 +53 @@
     for (my $i = 0; $i < $copies; $i++) {
         if (defined $volume) {
+            # this is coded so that all copies end up on the specified volume which seems wrong
             $neb->replicate($key, $volume)
                 or die "failed to replicate Nebulous key: $key";

trunk/Nebulous/bin/neb-shift

@@ -17 +17 @@
 
 my ($volume, $server);
+
+die "this tool may fail since find_instances will return an instance which is not on the hard volume";
 
 $server = $ENV{'NEB_SERVER'} unless $server;

trunk/Nebulous/bin/neb-stat

@@ -19 +19 @@
 use URI;
 
-my ($validate,$server,$nouris);
+my ($validate,$server,$nouris,$help);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
 
 GetOptions(
-    'server|s=s'     => \$server,
-    'validate|v'         => \$validate,
-    'files'          => \$nouris,
+    'server|s=s'    => \$server,
+    'validate|v'    => \$validate,
+    'files|f'       => \$nouris,
+    'help|h'        => \$help,
 ) || pod2usage( 2 );
 
+# explicitly defined -h / --help:
+if ($help) { pod2usage( -verbose => 1 ); }
+
 my $key = shift;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --server", -exitval => 2 )
-    unless $server;
-pod2usage( -msg => "missing key", exitval => 2 )
-    unless defined $key;
+pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
+pod2usage( -msg => "missing key", exitval => 2 ) unless defined $key;
 
 my $neb = Nebulous::Client->new(
@@ -52 +54 @@
 eval {
     $user_copies = $neb->getxattr($key, "user.copies");
+    die ("user.copies not found") unless $user_copies;
 };
-unless(defined($user_copies)) {
+unless(defined($user_copies) and $user_copies ne "") {
     $user_copies = 1;
 }
@@ -74 +77 @@
             $md5sum_uniq{$md5sums[$i]} = 1;
             close($fh);
-        }
-        else {
+        } else {
             $existance[$i] = 0;
             $md5sums[$i] = 'NON-EXISTANT';
             $md5sum_uniq{$md5sums[$i]} = 1;
         }
+
         if (defined($nouris)) {
             $validation[$i] = sprintf("% 3d %32s %s",
@@ -85 +88 @@
                                       $md5sums[$i],
                                       $files[$i]);
-        }
-        else {
+        } else {
             $validation[$i] = sprintf("% 3d %32s %s",
                                       $existance[$i],
@@ -112 +114 @@
         "instance location:\n", " " x 4,
         join("\n" . " " x 4, @validation), "\n";
+} else {
+    
+    if (defined($nouris)) {
+        my @files = map {URI->new($_)->file if $_} @$instances;
+
+        print
+            "object id:             ", @$stat[0], "\n",
+            "key:                   ", @$stat[1], "\n",
+            "read lock:             ", @$stat[2], "\n",
+            "write lock:            ", @$stat[3], "\n",
+            "epoch:                 ", @$stat[4], "\n",
+            "mtime:                 ", @$stat[5], "\n",
+            "available instances:   ", @$stat[6], "\n",
+            "requested instances:   ", $user_copies, "\n",
+            "total instances:       ", @$stat[7], "\n",
+            
+            "instance location:\n", " " x 4,
+            join("\n" . " " x 4, @files), "\n";
+    } else {
+        print
+            "object id:             ", @$stat[0], "\n",
+            "key:                   ", @$stat[1], "\n",
+            "read lock:             ", @$stat[2], "\n",
+            "write lock:            ", @$stat[3], "\n",
+            "epoch:                 ", @$stat[4], "\n",
+            "mtime:                 ", @$stat[5], "\n",
+            "available instances:   ", @$stat[6], "\n",
+            "requested instances:   ", $user_copies, "\n",
+            "total instances:       ", @$stat[7], "\n",
+            
+            "instance location:\n", " " x 4,
+            join("\n" . " " x 4, @$instances), "\n";
+    }
+
 }
-else {
-    print
-        "object id:             ", @$stat[0], "\n",
-        "key:                   ", @$stat[1], "\n",
-        "read lock:             ", @$stat[2], "\n",
-        "write lock:            ", @$stat[3], "\n",
-        "epoch:                 ", @$stat[4], "\n",
-        "mtime:                 ", @$stat[5], "\n",
-        "available instances:   ", @$stat[6], "\n",
-        "requested instances:   ", $user_copies, "\n",
-        "total instances:       ", @$stat[7], "\n",
-        
-        "instance location:\n", " " x 4,
-        join("\n" . " " x 4, @$instances), "\n";
-}
 
 
@@ -150 +171 @@
 =over 4
 
-=item * --server|-s <URL>
+=item  --server|-s <URL>
 
 URL of the Nebulous server to connect to.
@@ -156 +177 @@
 Optional if the appropriate environment variable is set.
 
-=item * --validate|-v
+=item --validate|-v
 
 Optional - prints out additional information including md5sum.
 
-=item * --files
+=item --files
 
 Optional - prints out disk files instead of uris during validate stage.
@@ -174 +195 @@
 =over 4
 
-=item * C<NEB_SERVER>
+=item C<NEB_SERVER>
 
 Equivalent to --server|-s

trunk/Nebulous/lib/Nebulous/Client.pm

@@ -209 +209 @@
         },
     );
-
+    
     $log->debug( "entered - @_" );
 
@@ -310 +310 @@
     my $stats = $self->stat($key);
     my $instances = $stats->[6];
+
     if ((not defined $instances) or ($instances == 0)) {
         $self->set_err("can not cull - no instances");
@@ -315 +316 @@
         return;
     }
-    if ($instances < $min_copies) {
+    if ($instances <= $min_copies) {
         $self->set_err("can not cull - not enough instances");
         $log->debug("leaving");
-
         return;
     }
@@ -334 +334 @@
 
     unless (defined $locations) {
-        $self->set_err( "no instances" );
         $log->debug( "leaving" );
-
         return;
     }
@@ -351 +349 @@
         }
     }
+
     # If we don't have two copies on one volume, see if we have two copies on a single 
     # cabinet. If so, delete the first of those copies.
@@ -362 +361 @@
         }
     }
+
     # Fail-safe. We didn't have any duplicates (the instances are "well-mixed"), so 
     # delete the first.
@@ -403 +403 @@
     eval {
         # first - strip off any inaccesible instances
+        # these are not counted in the 'removed' number
         if (not defined $self->prune($key)) {
             # use the prune() method to also determine if $key is valid
             die "invalid key: $key";
         }
-
-        # check to see if there is an instance on $vol_name that should be the
-        # sole survivor
+        
+        # check to see if there is an instance on $vol_name that
+        # should be the sole survivor.  This is a bit dangerous : what
+        # if the only instance is on an invalid volume?  what if an
+        # invalid volume is specified?
+
         if (defined $vol_name) {
             $locations = $self->find_instances($key, $vol_name);
+            if (not defined $locations) {
+                die "no instances on requested node";
+            }
+            if (scalar @$locations == 0) {
+                die "no instances on requested node";
+            }
         }
 
@@ -418 +428 @@
 
         if (defined $locations) {
-            my $instances = $self->find_instances($key, undef, 'find them all');
-            if (scalar @$instances == 1) {
-                # only one instance nothing to do
-                return 0;
-            }
-            if (scalar @$instances == 0) {
+            print "dev: looking for undef volume instances\n";
+            # first delete the valid instances (except the one we own)
+            my $instances_valid = $self->find_instances($key, undef, 0); # find all valid instances
+            if (not defined $instances_valid) {
+                # there should have been at least one valid instance?
                 die "no instances";
             }
-            foreach my $victim (@$instances) {
-                next if $victim eq $locations->[0];
-                $self->delete_instance($key, $victim);
-                $removed++;
+            if (scalar @$instances_valid == 0) {
+                # there should have been at least one valid instance?
+                die "no instances";
             }
+
+            # we have more than one instance, delete the others
+            if (scalar @$instances_valid > 1) {
+                foreach my $victim (@$instances_valid) {
+                    next if $victim eq $locations->[0];
+                    $self->delete_instance($key, $victim);
+                    $removed ++;
+                }
+            }
+            # the prune above should have deleted any invalid instances, so we only
+            # need to worry about the valid instances;
         } else {
             # nuke whatever
@@ -448 +467 @@
             }
         }
-
     };
     if ($@) {
@@ -663 +681 @@
     if ($response->fault) {
         $self->set_err($response->faultstring);
+
+        ## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+            die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+        ## invalid object key
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+            die "invalid xattr key $name : fails 'xattr is in user. namespace' test\n";
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -690 +721 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
+        ## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+            die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+        ## invalid xattr key (not in user. namespace)
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+            die "xattr $name is invalid : xattr is in user. namespace\n";
+        }
+
+        ## invalid xattr key
+        if ($response->faultstring =~ /user\..*? does not exist/) {
+            $log->debug( "leaving" );
+            die "xattr key $name not found : xattr $name does not exist\n";
+        }
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -747 +796 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
+        ## invalid object key
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+            die "invalid key $key, fails 'is valid object key' test\n";
+        }
+
+        ## invalid xattr key (not in user. namespace)
+        if ($response->faultstring =~ /xattr is in user. namespace/) {
+            $log->debug( "leaving" );
+            die "xattr $name is invalid : xattr is in user. namespace\n";
+        }
+
+        ## invalid xattr key
+        if ($response->faultstring =~ /user\..*? does not exist/) {
+            $log->debug( "leaving" );
+            die "xattr key $name not found : xattr $name does not exist\n";
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -865 +933 @@
     unless(defined($params[0])) {
         $params[0] = hostname() . ".0";
-#       print STDERR "Setting host to $params[0]\n";
+        # print STDERR "Setting host to $params[0]\n";
     }
 
@@ -871 +939 @@
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
+
         # check to see if this failure is because $key doesn't exist
         if ($response->faultstring =~ /is valid object key/) {
@@ -878 +947 @@
         # key is valid but no instances are on the specified volume
         if ($response->faultstring =~ /no instances on storage volume/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        # check to see if this failure is volume is unknown
+        if ($response->faultstring =~ /is not a valid volume name/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        # key is valid but no instances are on the specified volume
+        if ($response->faultstring =~ /no instances available for key/) {
             $log->debug( "leaving" );
             return;
@@ -949 +1028 @@
         $self->set_err($response->faultstring);
         # check to see if this failure is because $key doesn't exist
+        my $string = $response->faultstring;
         if ($response->faultstring =~ /is valid object key/) {
+            $self->set_err("invalid object key");
             $log->debug( "leaving" );
             return;
@@ -955 +1036 @@
         # key is valid but no instances are on the specified volume
         if ($response->faultstring =~ /no instances on storage volume/) {
+            $self->set_err("no instances on storage volume or volume is not available");
             $log->debug( "leaving" );
             return;
@@ -972 +1054 @@
 
 
+## this is used by PS:IPP:Config nebulous functions
 sub find
 {
@@ -1014 +1097 @@
     return $path;
 }
-
 
 sub open
@@ -1124 +1206 @@
     $log->debug( "entered - @_" );
 
-
     my $locations;
     if ($invalid) {
-        $locations = $self->find_instances( $key, 'any', 'find them all');
+        # delete the invalid instances:
+        $locations = $self->find_instances( $key, 'any', 1);
     }
     else {
@@ -1315 +1397 @@
         }
 
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->logdie("parameter #1 to delete_instance did not pass the 'is valid object key' callback");
+        }
+
         $log->logdie("unhandled fault - ", $self->err);
     }
@@ -1477 +1563 @@
 }
 
+sub set_log_level
+{
+    my $self = shift;
+
+    my $log_level = shift;
+
+    $log->level($log_level);
+
+    return $self;
+}
+
 
 1;

trunk/Nebulous/lib/Nebulous/Util.pm

@@ -78 +78 @@
     # The instance file attempting to be opened should always exist as it was
     # created by the Nebulous server.
+
     my $fh;
     for (my $i = 0; $i < 60; $i++) {
@@ -160 +161 @@
         die "can not process $arg because it is not in name:value form"
             unless defined $name and defined $value;
-        die "xattr name: $name is not in to the form user.name"
+        die "xattr name: $name is not in the form user.name"
             unless $name =~ /^user\./;
         $neb->setxattr($key, $name, $value, "replace")

trunk/Nebulous/nebclient

@@ -21 +21 @@
 stamp-h1
 test-driver
+INSTALL
+COPYING

@@ -21 +21 @@
 stamp-h1
 test-driver
+INSTALL
+COPYING

trunk/Nebulous/nebclient/tests/tap

@@ -19 +19 @@
 src/.deps
 src/Makefile
+INSTALL
+COPYING

@@ -19 +19 @@
 src/.deps
 src/Makefile
+INSTALL
+COPYING

trunk/Nebulous/nebclient/tests/tests.c

@@ -16 +16 @@
     if (foo) nebFree(foo)
 
+int show_instances (nebServer *server, char *key) {
+
+  nebObjectInstances *locations = NULL;
+
+  locations = nebFindInstances(server, key, NULL);
+
+  for (int i = 0; i < locations->n; i++) {
+    fprintf (stderr, "# loc %d : %s\n", i, locations->URI[i]);
+  }
+  nebObjectInstancesFree(locations);
+  return 1;
+}
+
+int count_instances (nebServer *server, char *key) {
+
+  nebObjectInstances *locations = NULL;
+  locations = nebFindInstances(server, key, NULL);
+  int Ninstances = locations ? locations->n : 0;
+  nebObjectInstancesFree(locations);
+  return Ninstances;
+}
+
 int main (int argc, char **argv) {
     nebServer       *server = NULL;
     char            *key = "foobarbaz";
 
-    plan_tests(31);
+    plan_tests(37);
 
     if (getenv("NEB_SERVER")) {
@@ -37 +59 @@
         filename = nebCreate(server, key, NULL, &URI);
         diag("filename is %s", filename);
+
+        if (!filename) {
+          fprintf (stdout, "not ok: no files in test server, exiting\n");
+          exit (2);
+        }
+
+        show_instances (server, key);
+        ok (count_instances (server, key) == 1, "correct instance count"); 
+
         neb_ok(server, filename, "create object");
         ok(URI, "URI not NULL");
@@ -53 +84 @@
         neb_ok(server, fh > -1, "create new object filehandle");
         ok(URI, "URI not NULL");
+        ok (count_instances (server, key) == 1, "correct instance count"); 
 
         tst_nebFree(URI);
         close(fh);
     }
+
+    show_instances (server, key);
 
     {
@@ -80 +114 @@
     }
 
-
     neb_ok(server, nebReplicate(server, key, NULL, NULL), "replicate object");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 2, "correct instance count"); 
 
     neb_ok(server, nebOpen(server, key, NEB_WRITE) < 0, "write to object with multiple instances");
+    show_instances (server, key);
 
     neb_ok(server, nebCull(server, key), "cull object");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 1, "correct instance count"); 
+
+    neb_ok(server, nebCull(server, key) == 0, "cannot cull object with one instance");
+    show_instances (server, key);
+    ok (count_instances (server, key) == 1, "correct instance count"); 
 
     neb_ok(server, nebStat(server, key), "stat object");
+    show_instances (server, key);
 
     neb_ok(server, nebLock(server, key, NEB_WRITE), "lock object write");

trunk/Nebulous/t/50_client_create.t

@@ -10 +10 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 9;
+plan tests => 13;
 
 use lib qw( ./t ./lib );
@@ -19 +19 @@
 use URI::Split qw( uri_split );
 
+# this returns the test apache server location
 my $hostport = Apache::Test->config->{ 'hostport' };
 
@@ -30 +31 @@
 
     my $uri = $neb->create("foo");
+    if (not defined $uri) { die "failure to create basic file (does it already exist?)"; }
+
     my ($scheme, $auth, $path, $query, $frag) = uri_split($uri);
     ok(-e $path, "good filename");
+
     uri_scheme_ok($uri, 'file');
+    
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -48 +55 @@
     ok(-e $path, "good filename");
     uri_scheme_ok($uri, 'file');
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -63 +73 @@
     ok(-e $path, "good filename");
     uri_scheme_ok($uri, 'file');
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 
@@ -74 +87 @@
     $neb->create("foo");
     is($neb->create("foo"), undef, "object already exists");
+
+    # need to remove the test entry we made above (create fails if file exists)
+    ok( $neb->delete( "foo" ), "delete object" );
 }
 

trunk/Nebulous/t/53_client_cull.t

@@ -10 +10 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 15;
+plan tests => 18;
 
 use lib qw( ./t ./lib );
@@ -21 +21 @@
 
 Test::Nebulous->setup;
-
 {
     # key
@@ -37 +36 @@
 
 Test::Nebulous->setup;
-
 {
     # key, $volume
@@ -43 +41 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create("foo");
-    $neb->replicate("foo", "node01");
-
-    my $uri = $neb->cull("foo", "node01");
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+    
+    my $uri = $neb->cull("foo", "node02");
 
     ok($uri, "good cull");
     ok(! -e _get_file_path($uri), "file exists");
+}
+
+Test::Nebulous->setup;
+{
+    # key, $volume
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+
+    # cull with soft volume should succeed even if node does not have instance
+    my $uri = $neb->cull("foo", "node03");
+
+    ok($uri, "good cull");
+    ok(! -e _get_file_path($uri), "file exists");
+}
+
+Test::Nebulous->setup;
+{
+    # key, $volume
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create("foo", "node01");
+    $neb->replicate("foo", "node02");
+
+    # cull with hard volume should not succeed if node does not have instance
+    my $uri = $neb->cull("foo", "~node03");
+
+    is($uri, undef, "nothing to cull on this node");
 }
 
@@ -112 +141 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->cull();
+    my $uri = $neb->cull();
+
+    is ($uri, undef, "no params given to cull");
 };
 like( $@, qr/1 - 3 were expected/, "no params" );
@@ -122 +153 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->cull( 1, 2, 3, 4 );
+    my $uri = $neb->cull( 1, 2, 3, 4 );
+
+    is ($uri, undef, "no params given to cull");
 };
 like( $@, qr/1 - 3 were expected/, "too many params" );

trunk/Nebulous/t/56_client_find_instances.t

@@ -10 +10 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 29;
+plan tests => 35;
 
 use lib qw( ./t ./lib );
@@ -20 +20 @@
 my $hostport = Apache::Test->config->{ 'hostport' };
 
-Test::Nebulous->setup;
-
+### XXX EAM : this test suite finds real errors in the implementation.  
+
+## the API is now slightly different (what is the state of the API in the ipp ops code?)
+## find_instances ("key", "volume", find_invalid)
+
+## this returns a result even if we request a hard volume unless the hard volume is not valid
+
+Test::Nebulous->setup;
+Test::Nebulous->cleanup;
+
+### TEST BLOCK for find_invalid
+## Test::Nebulous->setup;
+## {
+##     # volume/key, volume override
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+##     $neb->create( "foo", "~node02" );
+##     $neb->replicate( "foo", "~node01" );
+## 
+##     Test::Nebulous->switch_node_state;
+##     my $tmphost;
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 1); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 0); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "~node02"); # result : no instance
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 1); # result : no instance
+## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 0); # result : no instance
+##     my $locations = $neb->find_instances( "neb:///foo", "node02"); # result : no instances
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 1);
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
+## #   my $locations = $neb->find_instances( "neb:///foo", $tmphost, 0);
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 1);
+## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 0);
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo");
+## #   my $locations = $neb->find_instances( "neb://bebaz/foo");
+## 
+## #   my $locations = $neb->find_instances( "neb:///foo");
+## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
+## 
+##     foreach my $f (@$locations) {
+##      print "location: $f\n";
+##     }
+## 
+##     is( scalar @$locations, 1, "found 1" );
+##     like( @$locations[0], qr/file:/, "URIs match" );
+##     ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+## }
+## die "quit";
+
+## Test::Nebulous->setup;
+## eval {
+##     # volume/key, volume override
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+##     $neb->create( "foo", "~node01" );
+## 
+##     my $locations_1 = $neb->find_instances( "neb://~invalid/foo", "~invalid");
+##     is($locations_1, undef, "no instances in invalid node" );
+## 
+##     # EAM: we fail this test because the logic of find_instances has
+##     # changed: as long as there is an instance on a valid node, the
+##     # instance gets returned
+##     my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
+##     is($locations, undef, "no instances on specified volume" );
+## };
+##print "result: $@\n";
+##like( $@, qr/result message/, "my message" );
+
+# find a single instance by raw name (foo)
+Test::Nebulous->setup;
 {
     # key
@@ -36 +110 @@
 }
 
-Test::Nebulous->setup;
-
+# find a both instances of a duplicated object
+Test::Nebulous->setup;
 {
     # key
@@ -56 +130 @@
 
 Test::Nebulous->setup;
-
 {
     # key, volume
@@ -79 +152 @@
     );
     $neb->create( "foo" );
+    $neb->set_log_level ('DEBUG');
 
     my $locations = $neb->find_instances( "foo", undef );
@@ -103 +177 @@
 }
 
-Test::Nebulous->setup;
-
+# find instance using hard_volume for wrong node
+# (should still return a valid instance)
+Test::Nebulous->setup;
 {
     # volume/key, volume override
@@ -111 +186 @@
     );
     $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
 
     my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
 
+    is( scalar @$locations, 1, "found 1" );
+    like( @$locations[0], qr/file:/, "URIs match" );
+    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+#   the API used to return undef if we used a hard_volume but it was not there
+#   is($locations, undef, "no instances on specified volume" );
+}
+
+# find instance using hard_volume for unknown node
+# (should fail)
+Test::Nebulous->setup;
+{
+    # volume/key, volume override
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
+
+    my $locations = $neb->find_instances( "neb://node01/foo", "~nobody" );
     is($locations, undef, "no instances on specified volume" );
+#   is( scalar @$locations, 0, "found 0" );
+#   under Ubuntu (Perl 5.26.1), @$locations evaluates to 0 if $locations is undef,
+#   but not under Gentoo (Perl 5.6.6)
+}
+
+# find instance using hard_volume for unavailable node
+# (should fail)
+Test::Nebulous->setup;
+{
+    # volume/key, volume override
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo", "~node01" );
+    $neb->set_log_level ('DEBUG');
+
+    my $locations = $neb->find_instances( "neb://node01/foo", "~node04" );
+
+    is( scalar @$locations, 1, "found 1" );
+    like( @$locations[0], qr/file:/, "URIs match" );
+    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
+
+#   the API used to return undef if we used a hard_volume but it was not there
+#   is($locations, undef, "no instances on specified volume" );
 }
 
@@ -153 +272 @@
 
 Test::Nebulous->setup;
-
 {
     my $neb = Nebulous::Client->new(
@@ -171 +289 @@
     $neb->find_instances();
 };
-like( $@, qr/1 - 2 were expected/, "no params" );
+like( $@, qr/1 - 3 were expected/, "no params" );
 
 Test::Nebulous->setup;
@@ -180 +298 @@
     );
 
-    $neb->find_instances( "foo", 'read', 3 );
+    $neb->find_instances( "foo", 'read', 3, 5 );
 };
-like( $@, qr/1 - 2 were expected/, "too many params" );
+like( $@, qr/1 - 3 were expected/, "too many params" );
 
 Test::Nebulous->cleanup;

trunk/Nebulous/t/59_client_delete.t

@@ -10 +10 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 13;
+plan tests => 18;
 
 use lib qw( ./t ./lib );
@@ -20 +20 @@
 my $hostport = Apache::Test->config->{ 'hostport' };
 
-Test::Nebulous->setup;
-
+# ensure fresh start:
+Test::Nebulous->setup;
+Test::Nebulous->cleanup;
+
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -78 +81 @@
 }
 
-# force flag
-Test::Nebulous->setup;
-
+# test passing in undefined force flag
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -90 +92 @@
 }
 
-Test::Nebulous->setup;
-
+# test passing in force flag = 1
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -101 +103 @@
 }
 
+# test passing in force flag = 0
+Test::Nebulous->setup;
+{
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+
+    is( $neb->delete("foo", 0), 1, "force flag false" );
+}
+
+# test passing in force flag = 0 where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo", 0);
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
+# test passing in force flag = 1 where disk file is removed
+Test::Nebulous->setup;
+{
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    is( $neb->delete("foo", 1), 1, "force flag true deletes file" );
+}
+
+# test passing in undef force flag where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo", undef);
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
+# test passing in undefined force flag where disk file is removed
+Test::Nebulous->setup;
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->create( "foo" );
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path(@$locations[0]);
+    unlink $diskfile;
+    $neb->delete("foo");
+};
+like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
+
 Test::Nebulous->setup;
 
@@ -109 +177 @@
     $neb->delete();
 };
-like( $@, qr/1 - 2 were expected/, "no params" );
+like( $@, qr/1 - 3 were expected/, "no params" );
 
 Test::Nebulous->setup;
@@ -127 +195 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->delete( "foo", 0, 2 );
-};
-like( $@, qr/1 - 2 were expected/, "too many params" );
+    $neb->delete( "foo", 0, 1, 2 );
+};
+like( $@, qr/1 - 3 were expected/, "too many params" );
 
 Test::Nebulous->cleanup;

trunk/Nebulous/t/64_client_find_objects.t

@@ -21 +21 @@
 
 Test::Nebulous->setup;
-
 {
     # key

trunk/Nebulous/t/65_client_mounts.t

@@ -29 +29 @@
     my $mounts = $neb->mounts();
 
-    is(scalar @$mounts, 7, "number of rows");
+    is(scalar @$mounts, 8, "number of rows");
 
     my %row;

trunk/Nebulous/t/66_client_xattr.t

@@ -21 +21 @@
 );
 
-# 1 key / xattr
-
-Test::Nebulous->setup;
-
+# setxattr
+Test::Nebulous->setup;
 {
     my $uri = $neb->create('foo');
@@ -46 +44 @@
 
 # multiple xattrs
-
-Test::Nebulous->setup;
-
+Test::Nebulous->setup;
 {
     my $uri = $neb->create('foo');
@@ -184 +180 @@
     $neb->getxattr('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
+like($@, qr|xattr user.bar does not exist|,
     "get xattr from non-existant xattr key");
 
@@ -256 +252 @@
     $neb->removexattr('foo', 'user.bar');
 };
-like($@, qr|xattr neb:///foo:user.bar does not exist|,
+like($@, qr|xattr user.bar does not exist|,
     "remove xattr from non-existant xattr key");
 

trunk/Nebulous/t/68_client_chmod.t

@@ -41 +41 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path);
 }
 
@@ -53 +56 @@
     my $mode = $neb->chmod($key, 0440);
 
-    my $st1 = stat(URI->new($uri1)->path);
-    my $st2 = stat(URI->new($uri2)->path);
+    my $path1 = URI->new($uri1)->path;
+    my $path2 = URI->new($uri2)->path;
+
+    my $st1 = stat($path1);
+    my $st2 = stat($path2);
 
     is($st1->mode &07777, 0440, "chmod() first instance");
@@ -60 +66 @@
     is($mode, 0440, "returned mode");
     is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
+
+    # reset to 600 so the user can delete the file and /tmp entry
+    chmod(0600, $path1);
+    chmod(0600, $path2);
 }
 

trunk/Nebulous/t/70_neb-ls.t

@@ -23 +23 @@
 use Test::Cmd;
 use Nebulous::Client;
+use Nebulous::Util qw( :standard );
 use Test::Nebulous;
 
@@ -28 +29 @@
 my $neb_url  = "http://$hostport/nebulous";
 
-
 my $cmd = 'bin/neb-ls';
 
-# last ditch effort to make sure dsget is executable
+# last ditch effort to make sure neb-ls is executable
 chmod 0755, 'bin/neb-ls';
 
@@ -37 +37 @@
 isa_ok($test, 'Test::Cmd');
 
-# NEB_SERVER env var not set
+# test if NEB_SERVER env var not set
 undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
 Test::Nebulous->setup;
-
 {
     $test->run(args => '');
@@ -46 +45 @@
 }
 
-# NEB_SERVER set
-Test::Nebulous->setup;
-
+# NEB_SERVER set (and used in the following tests)
+Test::Nebulous->setup;
 {
     $ENV{NEB_SERVER} = $neb_url;
@@ -57 +55 @@
 
 Test::Nebulous->setup;
-
-{
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create('foo');
-
-    $test->run(args => '');
-
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+
+    $test->run(args => '');
     is($? >> 8, 0, "exit code");
     like($test->stdout, qr/^foo$/,      "stdout");
@@ -72 +68 @@
 
 Test::Nebulous->setup;
-
 {
     my $neb = Nebulous::Client->new(
@@ -83 +78 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^bar\nfoo$/,  "stdout");
+    like($test->stdout, qr/^bar\s+foo$/,  "stdout");
     like($test->stderr, qr/^$/,         "stderr");
 }
 
 Test::Nebulous->setup;
-
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+
+    my $locations = $neb->find_instances( "foo" );
+    my $diskfile = _get_file_path( @$locations[0] );
+    my $filestats = `ls -l $diskfile`;
+
+    $test->run(args => '-l');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^$filestats$/,  "stdout");
+    like($test->stderr, qr/^$/,         "stderr");
+}
+
+Test::Nebulous->setup;
 {
     my $neb = Nebulous::Client->new(
@@ -96 +108 @@
     $neb->create('bar');
 
-    $test->run(args => '-l');
+    $test->run(args => '-c');
 
     is($? >> 8, 0, "exit code");
@@ -109 +121 @@
         proxy => $neb_url,
     );
-    $neb->create('foo');
-    $neb->create('bar');
-
-    $test->run(args => '-1');
-
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^bar\nfoo\n$/,   "stdout");
-    like($test->stderr, qr/^$/,             "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create('a/foo');
-
-    $test->run(args => '');
-
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr||,               "stdout");
+    $neb->create('a/foo');
+
+    $test->run(args => '');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/$|,           "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }
@@ -161 +157 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr|^a/bar\na/foo$|,  "stdout");
+    like($test->stdout, qr|^a/bar\s+a/foo$|,  "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }
@@ -193 +189 @@
 
     is($? >> 8, 0, "exit code");
-    like($test->stdout, qr|^a/b/\na/foo$|,        "stdout");
+    like($test->stdout, qr|^a/b/\s+a/foo$|,        "stdout");
     like($test->stderr, qr/^$/,             "stderr");
 }

trunk/Nebulous/t/71_client_there_can_be_only_one.t

@@ -10 +10 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 8;
+plan tests => 13;
 
 use lib qw( ./t ./lib );
@@ -21 +21 @@
 
 Test::Nebulous->setup;
+Test::Nebulous->cleanup;
 
+## volume is defined, but it is not valid (should exit)
+## Test::Nebulous->setup;
+## {
+##     my $key = "foo";
+##     my $neb = Nebulous::Client->new(
+##         proxy => "http://$hostport/nebulous",
+##     );
+## 
+##     my $uri = $neb->create($key, "node01");
+## 
+##     $neb->replicate($key, "node02");
+## 
+##     ## set node02 to an invalid state:
+##     Test::Nebulous->switch_node_state;
+## 
+##     # if requested node is invalid, the other instance is kept and the invalid one removed
+##     # but since invalid instances are removed with a prune call, they are not counted
+##     # in the number of items removed (return by the function call)
+##     is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
+##     my $locations = $neb->find_instances( "foo" );
+## 
+##     is( scalar @$locations, 1, "found 1" );
+##     is($locations->[0], $uri, "instance on correct volume" );
+## };
+## is($@, "", "there_can_be_only_one with supplied volume should succeed");
+## die "asdf";
+
+#######
+
+## volume not defined
+Test::Nebulous->setup;
 {
     my $key = "foo";
@@ -37 +69 @@
 }
 
+## volume is defined, only valid instances
 Test::Nebulous->setup;
-
-{
+eval {
     my $key = "foo";
     my $neb = Nebulous::Client->new(
@@ -45 +77 @@
     );
     $neb->create($key, "node01");
+
     my $uri = $neb->replicate($key, "node02");
-
     is($neb->there_can_be_only_one($key, "node02"), 1, "there can be only one!");
 
@@ -53 +85 @@
     is( scalar @$locations, 1, "found 1" );
     is($locations->[0], $uri, "instance on correct volume" );
-}
+};
+is($@, "", "there_can_be_only_one with supplied volume should succeed");
+
+## volume is defined, but it is not valid (should exit)
+Test::Nebulous->setup;
+eval {
+    my $key = "foo";
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    my $uri = $neb->create($key, "node01");
+
+    $neb->replicate($key, "node02");
+
+    ## set node02 to an invalid state:
+    Test::Nebulous->switch_node_state;
+
+    # if requested node is invalid, nothing is removed
+    is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
+
+    my $locations = $neb->find_instances( "foo" );
+
+    is( scalar @$locations, 1, "found 1" );
+    is($locations->[0], $uri, "instance on correct volume" );
+};
+is($@, "", "there_can_be_only_one with supplied volume should succeed");
 
 Test::Nebulous->setup;

trunk/Nebulous/t/72_neb-cull.t

@@ -39 +39 @@
 # NEB_SERVER env var not set
 undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
-Test::Nebulous->setup;
-
+
+## problem test
+## Test::Nebulous->setup;
+## {
+##     $ENV{NEB_SERVER} = $neb_url;
+## 
+##     my $key = 'foo';
+## 
+##     my $neb = Nebulous::Client->new(
+##         proxy => $neb_url,
+##     );
+##     $neb->create($key);
+## 
+##     $test->run(args => "--one_only " . $key);
+## 
+##     ## my $line1 = $test->stdout;
+##     ## print "stdout: $line1\n";
+##     ## 
+##     ## my $line2 = $test->stderr;
+##     ## print "stderr: $line2\n";
+## 
+##     is($neb->stat($key)->[6], 1, "correct # of instances");
+##     is($? >> 8, 255, "exit code");
+##     like($test->stdout, qr/^$/, "stdout");
+##     like($test->stderr, qr/no instances/, "stderr");
+## }
+## die "stop";
+
+## Test::Nebulous->setup;
+## {
+##     my $key = 'foo';
+## 
+##     my $neb = Nebulous::Client->new(
+##         proxy => $neb_url,
+##     );
+##     $neb->create($key);
+##     $neb->replicate($key);
+## 
+##     $test->run(args => "--server $neb_url $key");
+## 
+##     my $line = $test->stderr;
+##     print "stderr: $line\n";
+## 
+##     is($neb->stat($key)->[6], 1, "correct # of instances");
+##     is($? >> 8, 0, "exit code");
+##     like($test->stdout, qr/^$/, "stdout");
+##     like($test->stderr, qr/^$/, "stderr");
+## }
+## die "TEST";
+
+Test::Nebulous->setup;
 {
     $test->run(args => '');
@@ -48 +97 @@
 # NEB_SERVER set
 Test::Nebulous->setup;
-
 {
     $ENV{NEB_SERVER} = $neb_url;
@@ -57 +105 @@
 
 Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+
+    $test->run(args => "--server $neb_url $key");
+#   $test->run(args => $key);
+
+    is($neb->stat($key)->[6], 1, "correct # of instances");
+    is($? >> 8, 255, "exit code");
+    like($test->stdout, qr/^$/, "stdout");
+    like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+    $neb->replicate($key);
 
     $test->run(args => $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 255, "exit code");
-    like($test->stdout, qr/^$/, "stdout");
-    like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
+    my $line = $test->stderr;
+
+    # the default min_copies is 2:
+    is($neb->stat($key)->[6], 2, "correct # of instances");
+    is($? >> 8, 255, "exit code");
+    like($test->stdout, qr/^$/, "stdout");
+    like($test->stderr, qr/not enough instances/, "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = 'foo';
+
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create($key);
+    $neb->replicate($key);
     $neb->replicate($key);
 
     $test->run(args => $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 0, "exit code");
-    like($test->stdout, qr/^$/, "stdout");
-    like($test->stderr, qr/^$/, "stderr");
-}
-
-Test::Nebulous->setup;
-
-{
-    my $key = 'foo';
-
-    my $neb = Nebulous::Client->new(
-        proxy => $neb_url,
-    );
-    $neb->create($key);
-    $neb->replicate($key);
-    $neb->replicate($key);
-
-    $test->run(args => $key);
-
     is($neb->stat($key)->[6], 2, "correct # of instances");
     is($? >> 8, 0, "exit code");
@@ -113 +164 @@
 }
 
+## problem test
 Test::Nebulous->setup;
 
@@ -202 +254 @@
 
     is($neb->stat($key)->[6], 1, "correct # of instances");
-    is($? >> 8, 0, "exit code");
+
+    is($? >> 8, 255, "exit code");
     like($test->stdout, qr/^$/, "stdout");
     like($test->stderr, qr/not enough instances/, "stderr");
@@ -221 +274 @@
     $test->run(args => "--min_copies 2 " . $key);
 
-    is($neb->stat($key)->[6], 1, "correct # of instances");
+    is($neb->stat($key)->[6], 2, "correct # of instances");
     is($? >> 8, 0, "exit code");
     like($test->stdout, qr/^$/, "stdout");

trunk/Nebulous/t/conf/extra.conf.in

@@ -3 +3 @@
 # this file will be Include-d by @ServerRoot@/httpd.conf
 
+# need to add MPM?
+
+# Conflicts: mpm_worker mpm_prefork
+#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
+#LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
+LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
+#LoadModule cgid_module /usr/lib/apache2/modules/mod_cgid.so
+LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
+LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
+
+# event MPM
+# StartServers: initial number of server processes to start
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestWorkers: maximum number of worker threads
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+<IfModule mpm_event_module>
+        StartServers                     2
+        MinSpareThreads          25
+        MaxSpareThreads          75
+        ThreadLimit                      64
+        ThreadsPerChild          25
+        MaxRequestWorkers         150
+        MaxConnectionsPerChild   0
+</IfModule>
+
+<IfModule mod_alias.c>
+        <IfModule mod_cgi.c>
+#       Define ENABLE_USR_LIB_CGI_BIN
+        </IfModule>
+
+        <IfModule mod_cgid.c>
+#       Define ENABLE_USR_LIB_CGI_BIN
+        </IfModule>
+
+        <IfDefine ENABLE_USR_LIB_CGI_BIN>
+                ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/
+                <Directory "/usr/lib/cgi-bin">
+                        AllowOverride None
+                        Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+                        Require all granted
+                </Directory>
+        </IfDefine>
+</IfModule>
+
+# nebulous location:
 <Location /nebulous>
     SetHandler perl-script
-    PerlResponseHandler Apache2::SOAP
+    PerlResponseHandler Apache::SOAP
     PerlSetVar dispatch_to "PerlHandler Nebulous::Server::SOAP"
     PerlSetVar options "compress_threshold => 10000"

trunk/Nebulous/t/conf/startup.pl.in

@@ -5 +5 @@
 use lib qw( @ServerRoot@ );
 
+use ModPerl::Util (); #for CORE::GLOBAL::exit
+
+use Apache2::RequestRec ();
+use Apache2::RequestIO ();
+use Apache2::RequestUtil ();
+
+use Apache2::ServerRec ();
+use Apache2::ServerUtil ();
+use Apache2::Connection ();
+use Apache2::Log ();
+
+use APR::Table ();
+
+use ModPerl::Registry ();
+
+use Apache2::Const -compile => ':common';
+use APR::Const -compile => ':common';
+
+use Apache::SOAP;
+
 use Apache::DBI;
 use DBI;
@@ -10 +30 @@
 use Nebulous::Server::Apache;
 use Nebulous::Server;
-use Test::Nebulous;
+
+# do not inherit here, let's be explicit
+# use Test::Nebulous;
+
+# EAM : making this match Nebulous-Server/docs/apache2-mod_perl-startup.pl:
+
+# use the real mysql server:
+#my $dsn         = 'DBI:mysql:database=nebulous:host=localhost,mysql_socket=8500';
+#my $dbuser      = 'nebulous';
+#my $dbpasswd    = '';
+
+# use the test mysql server (defined in Test::Nebulous)
+my $dsn         = 'DBI:mysql:database=test:host=localhost:mysql_socket=/var/run/mysqld/mysqld.sock';
+my $dbuser      = 'test';
+my $dbpasswd    = '';
 
 #$Apache::DBI::DEBUG = 1;
-Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
-Apache::DBI->setPingTimeOut($NEB_DB, 10);
+#Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
+Apache::DBI->setPingTimeOut($dsn, 10);
 
 my $config = Nebulous::Server::Config->new(
@@ -22 +56 @@
 $config->add_db(
     dbindex     => 0,
-    dsn         => $NEB_DB,
-    dbuser      => $NEB_USER,
-    dbpasswd    => $NEB_PASS,
+    dsn         => $dsn,
+    dbuser      => $dbuser,
+    dbpasswd    => $dbpasswd,
 );
-#$config->add_db(
-#    dbindex     => 1,
-#    dsn         => 'DBI:mysql:database=nebulous1:host=localhost',
-#    dbuser      => $dbuser,
-#    dbpasswd    => $dbpasswd,
-#);
 
 Nebulous::Server::SOAP->new_on_init($config);

@@ -21 +21 @@
 stamp-h1
 test-driver
+INSTALL
+COPYING

@@ -19 +19 @@
 src/.deps
 src/Makefile
+INSTALL
+COPYING