Index: trunk/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- trunk/pstamp/scripts/pstamp_job_run.pl	(revision 30799)
+++ trunk/pstamp/scripts/pstamp_job_run.pl	(revision 30850)
@@ -80,4 +80,5 @@
 my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
 my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
+my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -131,5 +132,5 @@
 
     # check that actual input files exist
-    check_files($PSTAMP_GONE, @file_list);
+    check_files($PSTAMP_NOT_AVAILABLE, @file_list);
 
     # find our output directory
@@ -463,9 +464,16 @@
     foreach my $f (@_) {
         if (!$ipprc->file_exists($f)) {
-            if ($error_code) {
+            my $gone;
+            if (storage_object_exists($f, \$gone)) {
+                if ($error_code and $gone) {
+                    my_die( "file $f is GONE:", $job_id, $PSTAMP_GONE, 'stop');
+                } else {
+                    print STDERR "file $f is not available\n";
+                    $return_code = 0;
+                }
+            } else {
+                # This shouldn't happen. The job shouldn't have been queued unless the file
+                # exist. I guess if cleanup got triggered after the job was queued
                 my_die( "file $f does not exist:", $job_id, $error_code, 'stop');
-            } else {
-                print STDERR "file $f does not exist\n";
-                $return_code = 0;
             }
         }
@@ -506,4 +514,77 @@
 
     return 0;
+}
+
+my $neb;
+sub storage_object_exists
+{
+    my $file = shift;
+    my $ref_all_gone = shift;
+
+    if (!$neb) {
+        my $scheme = file_scheme($file);
+        if ($scheme and $scheme eq 'neb') {
+            $neb = $ipprc->nebulous();
+        } else {
+            return 0;
+        }
+    }
+
+    my $exists = $neb->storage_object_exists($file);
+    if (!$exists) {
+        return 0;
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+
+    if (scalar @lines == 0) {
+        # no output the file is really and truely gone
+        # XXX: this is now caught above
+        print STDERR "storage object for $file does not exist\n";
+        return 0;
+    }
+
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'available') {
+            $numNotGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone volume set all_gone to 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        $$ref_all_gone = 1;
+    } else {
+        $$ref_all_gone = 0;
+    }
+
+    # storage object exists so return true
+    return 1;
 }