Index: trunk/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 19956)
+++ trunk/Nebulous-Server/lib/Nebulous/Server.pm	(revision 20046)
@@ -1,5 +1,5 @@
 # Copyright (c) 2004-2008  Joshua Hoblitt
 #
-# $Id: Server.pm,v 1.88 2008-10-07 20:30:45 jhoblitt Exp $
+# $Id: Server.pm,v 1.89 2008-10-10 21:42:59 jhoblitt Exp $
 
 package Nebulous::Server;
@@ -28,5 +28,6 @@
 __PACKAGE__->mk_accessors(qw( log sql config ));
 
-use constant SUBPATH_DEPTH => 2;
+use constant SUBPATH_DEPTH  => 2;
+use constant NFS_RETRIES    => 100;
 
 sub new {
@@ -1367,13 +1368,13 @@
         my $storage_filename = $self->_generate_storage_filename($key, $ins_id);
         unless (-d $storage_path) {
-            mkpath($storage_path, 0, 0775)
+            _retry(sub { mkpath(@_) }, $storage_path, 0, 0775)
                 or die "can't create storage path: $storage_path";
         }
         # check to make sure at least the parent directory has the proper
         # permissions
-        my $mode = (stat($storage_path))[2] & 07777;
+        my $mode = [_retry(sub { stat($storage_path) } )]->[2] & 07777;
         unless ($mode == 0775) {
             $log->error("$storage_path has the wrong permissions of: %04o", $mode);
-            chmod(0775, $storage_path) or die "can't chmod $storage_path: $!";
+            _retry(sub { chmod(0775, $storage_path) }) or die "can't chmod $storage_path: $!";
         }
 
@@ -1408,9 +1409,9 @@
     my $fh;
     die "can not open $path: $!"
-        unless (open($fh, '>', $path));
+        unless (_retry(sub { open($fh, '>', $path) }));
 
     # chmod before fsync() to make sure the changed perms hit the disk too
     die "can not chmod $path: $!"
-        unless (chmod 0664, $path);
+        unless (_retry(sub { chmod(0664, $path) }));
 
     die "can not flush $path: $!"
@@ -1421,5 +1422,5 @@
 
     die "can not close $path: $!"
-        unless (close($fh));
+        unless (_retry(sub { close($fh) }));
 
     return $path;
@@ -1455,4 +1456,30 @@
 }
 
+sub _retry
+{
+    my $func = shift;
+
+    my @ret;
+    for (my $i = 0; $i < NFS_RETRIES; $i++) {
+        eval {
+            @ret = $func->(@_);
+        };
+        if ($@) {
+            die $@;
+            sleep 1;
+            next;
+        }
+
+        last;
+    }
+
+    # if the loop ended and $@ is set, rethrow the error
+    if ($@) {
+        die $@;
+    }
+
+    return @ret;
+}
+
 
 sub DESTROY