Index: /branches/neb_distrib_20081210/Nebulous-Server/Build.PL
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/Build.PL	(revision 23707)
+++ /branches/neb_distrib_20081210/Nebulous-Server/Build.PL	(revision 23708)
@@ -15,17 +15,19 @@
         'DBI'                   => '1.53',
         'Digest::SHA1'          => 0,
+        'File::Basename'        => 0,
         'File::ExtAttr'         => '1.03',
         'File::Mountpoint'      => '0.01',
         'File::Path'            => '1.08',
         'File::Spec'            => 0,
-        'Filesys::Df'           => '0.92',
         'File::Spec::Functions' => 0,
         'File::Temp'            => 0,
+        'Filesys::Df'           => '0.92',
+        'Log::Dispatch::Email::MailSend' => 0,
         'Log::Log4perl'         => '0.48',
         'Net::Server::Daemonize'=> '0.05',
         'Params::Validate'      => '0.73',
         'SOAP::Lite'            => '0.69',
+        'SQL::Interp'           => '1.06',
         'URI'                   => '1.30',
-        'SQL::Interp'           => '1.06',
     },
     build_requires      => {
Index: /branches/neb_distrib_20081210/Nebulous-Server/Changes
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/Changes	(revision 23707)
+++ /branches/neb_distrib_20081210/Nebulous-Server/Changes	(revision 23708)
@@ -3,5 +3,10 @@
 0.17
     - retry database transactions when a deadlock is detected
-
+    - add log4perl logging to nebdiskd
+    - base the on disk directory hashing of files only on the dirname()
+      component of keys
+    - add email logging of events to nebdiskd
+    - attempt to optimize _is_valid_object_key() by eliminating an unused join
+      
 0.16
     - add so_id/name idxs to storage_object_xattr table
Index: /branches/neb_distrib_20081210/Nebulous-Server/bin/nebdiskd
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/bin/nebdiskd	(revision 23707)
+++ /branches/neb_distrib_20081210/Nebulous-Server/bin/nebdiskd	(revision 23708)
@@ -16,4 +16,5 @@
 use File::Spec;
 use Filesys::Df;
+use Log::Log4perl;
 use Nebulous::Server::SQL;
 use Net::Server::Daemonize qw( daemonize unlink_pid_file );
@@ -85,4 +86,35 @@
     unless $db && $dbuser && $dbpass;
 
+# start up logging
+my $conf = '
+    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE, Mailer
+
+    log4perl.appender.Screen        = Log::Log4perl::Appender::Screen
+    log4perl.appender.Screen.stderr = 1
+    log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.Screen.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+    log4perl.appender.SERVERLOGFILE           = Log::Log4perl::Appender::File
+    log4perl.appender.SERVERLOGFILE.filename  = /tmp/nebdiskd.log
+    log4perl.appender.SERVERLOGFILE.mode      = append
+    log4perl.appender.SERVERLOGFILE.layout    = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.SERVERLOGFILE.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+    log4perl.filter.MatchWarn  = Log::Log4perl::Filter::LevelMatch
+    log4perl.filter.MatchWarn.LevelToMatch  = WARN
+    log4perl.filter.MatchWarn.AcceptOnMatch = off
+
+    log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
+    log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
+    log4perl.appender.Mailer.subject = nebdiskd alert
+    log4perl.appender.AppError.Filter= MatchWarn
+    log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+';
+Log::Log4perl::init(\$conf);
+my $log = Log::Log4perl::get_logger("nebdiskd");
+$log->level('WARN');
+$log->level('DEBUG') if $debug;
+
 daemonize(
     $user,     # User
@@ -104,8 +136,8 @@
                 poll_interval   => $poll_interval,
                 debug           => $debug,
-        ) or die "poll_mounts() should not have returned";
+        );
     };
-    if ($!) {
-        warn $@;
+    if ($@) {
+        $log->warn($@);
     }
 
@@ -113,5 +145,5 @@
 }
 
-die "poll loop exited -- THIS SHOULD NOT HAPPEN";
+$log->logdie("poll loop exited -- THIS SHOULD NOT HAPPEN");
 
 
@@ -141,14 +173,14 @@
         # determine valid mountpoints
         foreach my $mnt (@$mounts) {
-            print "checking $mnt\n" if $debug;
+            $log->debug("checking $mnt");
             # this /SHOULD/ fail if the mount point is handled by the
             # automounter and it fails to mount
             eval {
                 unless (is_mountpoint($mnt)) {
-                    die "$mnt is not a valid mountpoint\n";
+                    $log->warn("$mnt is not a valid mountpoint");
                 }
             };
             if ($@) {
-                print $@ if $debug;
+                $log->warn($@);
                 $d_query->execute($mnt);
                 next;
@@ -160,10 +192,10 @@
             my $dev_info = df($mnt, 1024);
             unless (defined $dev_info) {
-                print "can't find device info for $mnt\n" if $debug;
+                $log->error("can't find device info for $mnt");
                 next;
             }
 
             $r_query->execute($mnt, @$dev_info{qw( blocks used )});
-            print "adding $mnt to db\n" if $debug;
+            $log->debug("adding $mnt to db");
 
         }
@@ -172,10 +204,10 @@
 
         $dbh->commit;
-        print "commited to database\n" if $debug;
+        $log->debug("commited to database");
     };
     if ($@) {
         $dbh->rollback;
-        print "rolledback transaction\n" if $debug;
-        warn $@;
+        $log->debug("rolledback transaction");
+        $log->logdie($@);
     }
 }
@@ -188,6 +220,6 @@
 
     if (!-f $rcfile) {
-        open(my $fh, '>', $rcfile) or die "can't open file: $!";
-        close($fh) or die "can't close file:$!";
+        open(my $fh, '>', $rcfile) or $log->logdie("can't open file: $!");
+        close($fh) or $log->logdie("can't close file:$!");
     }
 
@@ -226,6 +258,6 @@
     };
     if ($@) { 
-        $db->rollback;
-        die $@;
+        $dbh->rollback;
+        $log->logdie($@);
     }
 
@@ -242,7 +274,7 @@
     foreach my $path (@$mounts) {
         if (stat File::Spec->canonpath($path)) {
-            print "stated $path\n" if $debug;
+            $log->debug("stated $path");
         } else {
-            warn "can not stat path: $path";
+            $log->warn("can not stat path: $path");
         }
     }
@@ -263,9 +295,9 @@
   ### get the currently listed pid
   if( ! open(_PID,$pid_file) ){
-    die "Couldn't open existant pid_file \"$pid_file\" [$!]\n";
+    $log->logdie("Couldn't open existant pid_file \"$pid_file\" [$!]");
   }
   my $_current_pid = <_PID>;
   close _PID;
-  my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : die "Couldn't find pid in existing pid_file";
+  my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : $log->logdie("Couldn't find pid in existing pid_file");
 
   my $exists = undef;
@@ -289,10 +321,10 @@
 
         if( $current_pid == $$ ){
-            warn "Pid_file created by this same process. Doing nothing.\n";
+            $log->warn("Pid_file created by this same process. Doing nothing.");
             return 1;
         }else{
             kill 'TERM', $current_pid
-                or die "Failed to signal process ($current_pid)";
-            unlink $pid_file || die "Couldn't remove pid_file \"$pid_file\" [$!]\n";
+                or $log->logdie("Failed to signal process ($current_pid)");
+            unlink $pid_file || $log->logdie("Couldn't remove pid_file \"$pid_file\" [$!]");
         }
     }
Index: /branches/neb_distrib_20081210/Nebulous-Server/docs/install.txt
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/docs/install.txt	(revision 23707)
+++ /branches/neb_distrib_20081210/Nebulous-Server/docs/install.txt	(revision 23708)
@@ -56,10 +56,20 @@
 
     neb-initdb
-    neb-addvol --name ipp000.0 --uri file:///data/ipp000.0/nebulous
-    neb-addvol --name ipp000.1 --uri file:///data/ipp000.1/nebulous
-    neb-addvol --name ipp002.0 --uri file:///data/ipp002.0/nebulous
-    neb-addvol --name ipp003.0 --uri file:///data/ipp003.0/nebulous
-
-#    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-addvol --name $i --uri file:/$i/nebulous" }'
+
+    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous
+    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous
+    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous
+    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous
+    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous
+
+    # or in bulk:
+    
+    for i in ipp005 ipp006 ipp007 ipp009 ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp020 ipp021 ipp022 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 
+    do 
+        /usr/bin/neb-voladd --vname ${i}.0 --vhost $i --uri file:///data/${i}.0/nebulous_beta
+    done
+
+#    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-voladd
+#    --vname $i --vhost $i --uri file:/$i/nebulous" }'
     .
     .
Index: /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 23707)
+++ /branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 23708)
@@ -47,8 +47,4 @@
         (so_id, vol_id, uri)
         VALUES (?, ?, 'error')
-    },
-    get_all_instances   => qq{
-        SELECT * FROM INSTANCE
-        WHERE so_id = ?
     },
     get_object          => qq{
@@ -63,4 +59,11 @@
         JOIN storage_object_attr
         USING (so_id)
+        WHERE ext_id = ?
+    },
+    check_object_name => qq{
+        SELECT
+            so_id,
+            ext_id
+        FROM storage_object
         WHERE ext_id = ?
     },
@@ -351,4 +354,29 @@
         limit 5;
     },
+    find_objects_with_extra_instances => qq{
+        SELECT
+            storage_object.so_id,
+            ext_id,
+            count(ins_id) as instances,
+            volume.name as volume_name,
+            volume.host as volume_host,
+            count(mymountedvol.vol_id) as available_instances,
+            count(mymountedvol.vol_id) > 0 as recoverable,
+            storage_object_xattr.value as copies
+        FROM storage_object
+        JOIN instance
+            USING(so_id)
+        JOIN volume
+            USING(vol_id)
+        LEFT JOIN storage_object_xattr
+            ON storage_object.so_id = storage_object_xattr.so_id
+        JOIN mymountedvol
+            USING(vol_id)
+        WHERE
+            mymountedvol.available = 1
+            AND storage_object_xattr.name = 'user.copies'
+        GROUP BY so_id
+        HAVING available_instances > copies
+    },
     get_mounted_volumes => qq{
         SELECT * FROM mountedvol ORDER BY host, name
@@ -408,5 +436,4 @@
     type enum('REG_FILE'),
     PRIMARY KEY(so_id),
-    KEY(ext_id(64)),
     KEY(type)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -443,5 +470,5 @@
     type ENUM( 'read', 'write' ) NOT NULL,
     epoch TIMESTAMP,
-    KEY(so_ID)
+    KEY(so_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -457,5 +484,4 @@
     xattr BOOLEAN DEFAULT FALSE,
     PRIMARY KEY(vol_id),
-    KEY(name(16)),
     KEY(host(16)),
     KEY(path(255)),
@@ -472,5 +498,5 @@
     vol_id INT NOT NULL,
     FOREIGN KEY(vol_id) REFERENCES volume(vol_id),
-    uri VARCHAR(255) NOT NULL UNIQUE,
+    uri VARCHAR(255) NOT NULL,
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     mtime TIMESTAMP,
@@ -478,5 +504,5 @@
     KEY(so_id),
     KEY(vol_id),
-    KEY(uri(64))
+    KEY(uri(40))
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -517,7 +543,12 @@
     available BOOLEAN DEFAULT FALSE,
     xattr BOOLEAN DEFAULT FALSE,
+    PRIMARY KEY(mountpoint),
     KEY(vol_id),
+    KEY(name),
+    KEY(host),
+    KEY(path),
     KEY(allocate),
-    KEY(available)
+    KEY(available),
+    KEY(xattr)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;