Index: /trunk/Nebulous-Server/Changes
===================================================================
--- /trunk/Nebulous-Server/Changes	(revision 24636)
+++ /trunk/Nebulous-Server/Changes	(revision 24637)
@@ -31,4 +31,6 @@
     - change Nebulous::Server->find_objects() to return dirs and to sort it's
       output 
+    - add the ability to delete a storage object when it has instances that are
+      offline
       
 0.16
Index: /trunk/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- /trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 24636)
+++ /trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 24637)
@@ -56,5 +56,5 @@
     my ($config) = @_;
 
-    # log4perl is not avaliable until we call init()
+    # log4perl is not available until we call init()
     Nebulous::Server::Log->init($config);
     my $log = Log::Log4perl::get_logger( "Nebulous::Server" );
@@ -1622,5 +1622,6 @@
 TRANS: while (1) {
         eval {
-            my $instances;
+            my $total;
+            my $available;
             my $so_id;
             my $ins_id;
@@ -1636,5 +1637,6 @@
                 my $record = $query->fetchrow_hashref;
                 $so_id      = $record->{ 'so_id' };
-                $instances  = $record->{ 'count(ins_id)' };
+                $total      = $record->{ 'total' };
+                $available  = $record->{ 'available' };
                 $query->finish;
             }
@@ -1664,13 +1666,29 @@
                     die( "affected row count is $rows instead of 1" );
                 }
-            }
-
-            # if we just deleted the last instance associated with a storage object
-            # remove it too
-            if ( $instances == 1 ) {
+                
+            }
+
+            # if we just deleted the last 'available' instance associated with
+            # a storage object remove it too
+            if ( $available == 1 ) {
                 # remove key from cache
                 $self->cache->delete($key->path) if defined $self->cache;
 
-                # we just removed the last instance
+                # record the path of the innaccesible files for deferred
+                # deletion
+                {
+                    my $query = $db->prepare_cached( $sql->copy_instances_to_deleted );
+                    $query->execute( $so_id );
+                }
+                # remove all instances... not strictly nessicary as the delete
+                # from storage_object should cascade but the fkey was specified
+                # without the cascade 
+                {
+                    my $query = $db->prepare_cached( $sql->delete_instance_by_so_id );
+                    $query->execute( $so_id );
+                }
+                
+                # delete the storage object, remaining instances should be
+                # removed via cascading delete
                 my $query = $db->prepare_cached( $sql->delete_object );
                 my $rows = $query->execute( $so_id );
Index: /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 24636)
+++ /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 24637)
@@ -180,4 +180,8 @@
         WHERE ins_id = ?
     },
+    delete_instance_by_so_id => qq{
+        DELETE FROM instance
+        WHERE so_id = ?
+    },
     get_object_from_uri   => qq{
         SELECT so_id
@@ -198,11 +202,23 @@
     },
     get_instance_count_by_ext_id   => qq{
-        SELECT count(ins_id), so_id
+        SELECT 
+            so_id,
+            count(ins_id) as total,
+            sum(available) as available
         FROM instance
         JOIN storage_object
             USING(so_id)
+        LEFT JOIN mountedvol
+            USING(vol_id)
         WHERE
             ext_id = ?
         GROUP BY so_id
+    },
+    copy_instances_to_deleted => qq{
+        INSERT INTO deleted 
+        SELECT NULL, vol_id, uri
+        FROM instance
+        WHERE
+            so_id = ?
     },
     get_object_instances    => qq{
@@ -432,7 +448,8 @@
 DROP TABLE IF EXISTS log;
 DROP TABLE IF EXISTS directory;
-DROP PROCEDURE IF EXISTS getmountedvol;
+DROP TABLE IF EXISTS deleted;
 SET FOREIGN_KEY_CHECKS=1
 END
+#DROP PROCEDURE IF EXISTS getmountedvol;
     $sql{get_db_clear} = \@clear;
 }
@@ -451,4 +468,71 @@
 
 1;
+
+# ###
+# 
+# CREATE PROCEDURE getmountedvol() DETERMINISTIC
+# BEGIN
+#     DECLARE done BOOLEAN DEFAULT FALSE;
+#     DECLARE vol_idvar INT;
+#     DECLARE namevar VARCHAR(255);
+#     DECLARE hostvar VARCHAR(255);
+#     DECLARE pathvar VARCHAR(255);
+#     DECLARE allocatevar BOOLEAN;
+#     DECLARE availablevar BOOLEAN;
+#     DECLARE xattrvar BOOLEAN;
+#     DECLARE trans_level VARCHAR(255);
+#     DECLARE key_checks BOOLEAN;
+#     DECLARE cur1 CURSOR FOR SELECT vol_id, name, host, path, allocate, available, xattr FROM myvolume;
+#     DECLARE CONTINUE HANDLER FOR SQLSTATE '02000' SET done = TRUE;
+# 
+#     -- store the okey checking state
+# --    SELECT @@FOREIGN_KEY_CHECKS INTO key_checks; 
+#     -- disable foregin check checks to prevent deadlocks on the mountedvol table
+# --    SET FOREIGN_KEY_CHECKS=0;
+# 
+#     -- make sure the temp table does not already exist... this can happy if the
+#     -- stored proc fails for some reason
+#     DROP TABLE IF EXISTS myvolume;
+#     CREATE TEMPORARY TABLE myvolume LIKE volume;
+#     INSERT INTO myvolume SELECT * FROM volume;
+# 
+#     -- store the current transaction level
+# --    SELECT @@session.tx_isolation INTO trans_level; 
+#     -- set trans level to repeatable-read so the volume table does not change
+#     -- out from under our cursor
+# --    SET @@session.tx_isolation = 'REPEATABLE-READ';
+# 
+#     -- iterate over the volume table finding the coresponding entry in the
+#     -- mount table and inserting union of the volume & mount row into the
+#     -- mountedvol table
+#     OPEN cur1;
+# 
+#     myloop: LOOP
+#         FETCH cur1 INTO vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar;
+#         IF `done` THEN LEAVE myloop; END IF;
+#         REPLACE INTO mountedvol
+#             SELECT mountpoint, total, used, vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar
+#             FROM
+#                 (SELECT *, INSTR(pathvar, mountpoint) = 1 as substring
+#                 FROM mount
+#                 HAVING substring = 1
+#                 ORDER BY substring DESC, LENGTH(mountpoint) DESC
+#                 LIMIT 1) as bar;
+#     END LOOP myloop;
+#     
+#     CLOSE cur1;
+# 
+#     -- restore the original transaction level
+# --    SET @@session.tx_isolation = trans_level;
+# 
+#     -- restore the original key checking state
+# --    SET @@FOREIGN_KEY_CHECKS = key_checks; 
+# 
+#     DROP TABLE IF EXISTS myvolume;
+# 
+# --    SET FOREIGN_KEY_CHECKS=1;
+# 
+#     COMMIT;
+# END 
 
 __DATA__
@@ -568,5 +652,5 @@
     ins_id BIGINT NOT NULL AUTO_INCREMENT,
     so_id BIGINT NOT NULL,
-    FOREIGN KEY(so_id) REFERENCES storage_object(so_id),
+    FOREIGN KEY(so_id) REFERENCES storage_object(so_id) ON DELETE CASCADE,
     vol_id INT NOT NULL,
     FOREIGN KEY(vol_id) REFERENCES volume(vol_id),
@@ -593,66 +677,10 @@
 ###
 
-CREATE PROCEDURE getmountedvol() DETERMINISTIC
-BEGIN
-    DECLARE done BOOLEAN DEFAULT FALSE;
-    DECLARE vol_idvar INT;
-    DECLARE namevar VARCHAR(255);
-    DECLARE hostvar VARCHAR(255);
-    DECLARE pathvar VARCHAR(255);
-    DECLARE allocatevar BOOLEAN;
-    DECLARE availablevar BOOLEAN;
-    DECLARE xattrvar BOOLEAN;
-    DECLARE trans_level VARCHAR(255);
-    DECLARE key_checks BOOLEAN;
-    DECLARE cur1 CURSOR FOR SELECT vol_id, name, host, path, allocate, available, xattr FROM myvolume;
-    DECLARE CONTINUE HANDLER FOR SQLSTATE '02000' SET done = TRUE;
-
-    -- store the okey checking state
---    SELECT @@FOREIGN_KEY_CHECKS INTO key_checks; 
-    -- disable foregin check checks to prevent deadlocks on the mountedvol table
---    SET FOREIGN_KEY_CHECKS=0;
-
-    -- make sure the temp table does not already exist... this can happy if the
-    -- stored proc fails for some reason
-    DROP TABLE IF EXISTS myvolume;
-    CREATE TEMPORARY TABLE myvolume LIKE volume;
-    INSERT INTO myvolume SELECT * FROM volume;
-
-    -- store the current transaction level
---    SELECT @@session.tx_isolation INTO trans_level; 
-    -- set trans level to repeatable-read so the volume table does not change
-    -- out from under our cursor
---    SET @@session.tx_isolation = 'REPEATABLE-READ';
-
-    -- iterate over the volume table finding the coresponding entry in the
-    -- mount table and inserting union of the volume & mount row into the
-    -- mountedvol table
-    OPEN cur1;
-
-    myloop: LOOP
-        FETCH cur1 INTO vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar;
-        IF `done` THEN LEAVE myloop; END IF;
-        REPLACE INTO mountedvol
-            SELECT mountpoint, total, used, vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar
-            FROM
-                (SELECT *, INSTR(pathvar, mountpoint) = 1 as substring
-                FROM mount
-                HAVING substring = 1
-                ORDER BY substring DESC, LENGTH(mountpoint) DESC
-                LIMIT 1) as bar;
-    END LOOP myloop;
-    
-    CLOSE cur1;
-
-    -- restore the original transaction level
---    SET @@session.tx_isolation = trans_level;
-
-    -- restore the original key checking state
---    SET @@FOREIGN_KEY_CHECKS = key_checks; 
-
-    DROP TABLE IF EXISTS myvolume;
-
---    SET FOREIGN_KEY_CHECKS=1;
-
-    COMMIT;
-END 
+CREATE TABLE deleted (
+    timestamp TIMESTAMP,
+    vol_id INT NOT NULL,
+    uri VARCHAR(255) NOT NULL,
+    PRIMARY KEY(timestamp),
+    KEY(vol_id),
+    KEY(uri)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
Index: /trunk/Nebulous-Server/t/09_server_delete_instance.t
===================================================================
--- /trunk/Nebulous-Server/t/09_server_delete_instance.t	(revision 24636)
+++ /trunk/Nebulous-Server/t/09_server_delete_instance.t	(revision 24637)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 8;
+use Test::More tests => 11;
 
 use lib qw( ./t ./lib );
@@ -20,4 +20,6 @@
     dbpasswd    => $NEB_PASS,
 );
+
+use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS;
 
 Test::Nebulous->setup;
@@ -44,4 +46,27 @@
 
     ok($neb->delete_instance($key, $uri2), "delete instance");
+
+    eval {
+        $neb->find_instances($key);
+    };
+    like($@, qr/is valid object key/, "storage object was deleted");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = "foo";
+    my $uri1 = $neb->create_object($key, 'node01');
+    my $uri2 = $neb->replicate_object($key, 'node2');
+
+    # make one of the instances unavailable
+    my $dbh = test_dbh();
+    $dbh->do("UPDATE mountedvol SET available = 0 WHERE name = 'node01'");
+
+    ok($neb->delete_instance($key, $uri2), "delete instance");
+
+    expected_dataset_ok(
+        deleted => [vol_id => 1, uri => $uri1],
+    );
 
     eval {