Index: /trunk/Ohana/src/opihi/include/pcontrol.h
===================================================================
--- /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 20046)
+++ /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 20047)
@@ -83,4 +83,11 @@
 } Fifo;
 
+typedef struct {
+  IOBuffer     buffer;
+  int          completed;
+  int          size;
+  int          requested;
+} JobOutput;
+
 /* data to define a job */
 typedef struct {
@@ -91,11 +98,9 @@
   int          exit_status;
   int          Reset;
-  int          stdout_size;
-  int          stderr_size;
   JobMode      mode;
   JobStat      state;
   JobStat      stack;
-  IOBuffer     stdout_buff;
-  IOBuffer     stderr_buff;
+  JobOutput    stdout;
+  JobOutput    stderr;
   Ptime        start;
   Ptime        stop;
@@ -199,5 +204,5 @@
 int CheckIdleHost (Host *host);
 int CheckDoneJob (Job *job, Host *host);
-int GetJobOutput (char *command, Host *host, IOBuffer *buffer, int Nbytes);
+int GetJobOutput (char *command, Host *host, JobOutput *output);
 int rconnect (char *command, char *hostname, char *shell, int *stdio);
 
@@ -247,4 +252,6 @@
 
 /*** JobOps.c ***/
+int InitJobOutput (JobOutput *output);
+int ResetJobOutput (JobOutput *output);
 void   InitJobStacks ();
 void   FreeJobStacks ();
Index: /trunk/Ohana/src/opihi/pclient/job.c
===================================================================
--- /trunk/Ohana/src/opihi/pclient/job.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pclient/job.c	(revision 20047)
@@ -8,5 +8,5 @@
   if (argc < 2) {
     gprint (GP_ERR, "USAGE: job (arg0) (arg1) ... (argN)\n");
-    gprint (GP_LOG, "STATUS %d\n", -2);
+    gprint (GP_LOG, "PCLIENT_PID: %d\n", -2);
     return (FALSE);
   }
@@ -14,5 +14,5 @@
   if (ChildStatus != PCLIENT_NONE) {
     gprint (GP_ERR, "need to clear existing child\n");
-    gprint (GP_LOG, "STATUS %d\n", -3);
+    gprint (GP_LOG, "PCLIENT_PID: %d\n", -3);
     return (FALSE);
   }
@@ -73,5 +73,5 @@
   ChildPID = pid;
 
-  gprint (GP_LOG, "STATUS %d\n", ChildPID);
+  gprint (GP_LOG, "PCLIENT_PID: %d\n", ChildPID);
   return (TRUE);
 
@@ -85,5 +85,5 @@
   if (child_stderr_fd[1] != 0) close (child_stderr_fd[1]);
 
-  gprint (GP_LOG, "STATUS %d\n", -1);
+  gprint (GP_LOG, "PCLIENT_PID: %d\n", -1);
   return (FALSE);
 }
@@ -91,8 +91,8 @@
 /* possible responses:
 
-STATUS -1 - pipe error
-STATUS -2 - syntax error
-STATUS -3 - existing child
-STATUS >0 - success (PID)
+PCLIENT_PID: -1 - pipe error
+PCLIENT_PID: -2 - syntax error
+PCLIENT_PID: -3 - existing child
+PCLIENT_PID: >0 - success (PID)
 
 */
Index: /trunk/Ohana/src/opihi/pclient/reset.c
===================================================================
--- /trunk/Ohana/src/opihi/pclient/reset.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pclient/reset.c	(revision 20047)
@@ -8,5 +8,5 @@
   if (argc != 1) {
     gprint (GP_ERR, "USAGE: reset\n");
-    gprint (GP_LOG, "STATUS -1\n");
+    gprint (GP_LOG, "RESET_RESULT: -1\n");
     return (FALSE);
   }
@@ -14,5 +14,5 @@
   if (ChildStatus == PCLIENT_NONE) {
     gprint (GP_ERR, "no child process, cannot reset\n");
-    gprint (GP_LOG, "STATUS 2\n");
+    gprint (GP_LOG, "RESET_RESULT: 2\n");
     return (TRUE);
   }
@@ -43,5 +43,5 @@
     /* total failure, don't reset */
     gprint (GP_ERR, "child process %d is hung, cannot reset\n", ChildPID);
-    gprint (GP_LOG, "STATUS 0\n");
+    gprint (GP_LOG, "RESET_RESULT: 0\n");
     return (FALSE);
   }
@@ -62,5 +62,5 @@
   ChildExitStatus = 0;
 
-  gprint (GP_LOG, "STATUS 1\n");
+  gprint (GP_LOG, "RESET_RESULT: 1\n");
   return (TRUE);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 20047)
@@ -81,5 +81,11 @@
   if (!strcmp(string, "EXIT")) outstate = PCONTROL_JOB_EXIT;
   if (!strcmp(string, "CRASH")) outstate = PCONTROL_JOB_CRASH;
-  ASSERT (outstate != PCONTROL_JOB_BUSY, "invalid status response (CheckBusyJobResponse)");
+  if (outstate == PCONTROL_JOB_BUSY) {
+    if (DEBUG || VerboseMode()) gprint (GP_ERR, "invalid status response (CheckBusyJobResponse), try again\n");
+    PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
+    PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+    return (TRUE);
+  }
+
 
   /* parse the exit status and sizes of output buffers */
@@ -87,11 +93,11 @@
   sscanf (p, "%*s %d", &job[0].exit_status);
   p = memstr (buffer[0].buffer, "STDOUT", buffer[0].Nbuffer);
-  sscanf (p, "%*s %d", &job[0].stdout_size);
+  sscanf (p, "%*s %d", &job[0].stdout.size);
   p = memstr (buffer[0].buffer, "STDERR", buffer[0].Nbuffer);
-  sscanf (p, "%*s %d", &job[0].stderr_size);
+  sscanf (p, "%*s %d", &job[0].stderr.size);
 
   // XXX runaway job if output too large?
-  if (job[0].stdout_size > 0x100000) abort();
-  if (job[0].stderr_size > 0x100000) abort();
+  if (job[0].stdout.size > 0x1000000) abort();
+  if (job[0].stderr.size > 0x1000000) abort();
 
   // job has exited : move to DONE stack 
Index: /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 20047)
@@ -42,7 +42,7 @@
 
   /** successful command, examine result **/
-  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
+  p = memstr (buffer[0].buffer, "RESET_RESULT:", buffer[0].Nbuffer);
   if (p == NULL) {
-      if (DEBUG || VerboseMode()) gprint (GP_ERR, "missing STATUS in response; try again\n");
+      if (DEBUG || VerboseMode()) gprint (GP_ERR, "missing RESET_RESULT: in response; try again\n");
       PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
       return (FALSE);
Index: /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 20047)
@@ -1,7 +1,8 @@
 # include "pcontrol.h"
+# define DEBUG 0
 
 int CheckDoneJob (Job *job, Host *host) {
   
-  int success;
+  int status1, status2;
 
   ASSERT (job, "job not set");
@@ -11,11 +12,27 @@
   ASSERT (job == (Job *) host[0].job, "invalid job");
 
-  success = TRUE;
-  success &= GetJobOutput ("stdout", host, &job[0].stdout_buff, job[0].stdout_size);
-  success &= GetJobOutput ("stderr", host, &job[0].stderr_buff, job[0].stderr_size);
+  status1 = GetJobOutput ("stdout", host, &job[0].stdout);
+  status2 = GetJobOutput ("stderr", host, &job[0].stderr);
 
-  if (!success) {
-    // XXX some kind of error?
-    // XXX try again later?
+  if ((status1 == PCLIENT_DOWN) || (status2 == PCLIENT_DOWN)) {
+    // unlink host & job
+    if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+    job[0].host = NULL;
+    host[0].job = NULL;
+	
+    PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+
+    // clear the response data
+    host[0].response_state = PCONTROL_RESP_NONE;
+    host[0].response = NULL;
+
+    // host has shutdown; harvest the defunct process
+    HarvestHost (host[0].pid);
+    PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+    return (FALSE);
+  }
+
+  // try again if we are still waiting
+  if ((status1 == PCLIENT_HUNG) || (status2 == PCLIENT_HUNG)) {
     PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
     PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
@@ -32,4 +49,2 @@
   return (TRUE);
 }
-
-/** need to add timeout check here **/
Index: /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 20047)
@@ -1,65 +1,72 @@
 # include "pcontrol.h"
-# define PCLIENT_TIMEOUT 500
+# define PCLIENT_TIMEOUT 100
+# define DEBUG 0
 
-/* we read Nbytes from the host, then watch for the prompt */ 
-int GetJobOutput (char *cmd, Host *host, IOBuffer *buffer, int Nbytes) {
+// we are trying to read a total of Nbytes from the host.  This function may be called
+// repeatedly until the buffer has the complete set of data.  We need to read output[0].size
+// bytes, then look for the PCLIENT_PROMPT in the output stream
+
+int GetJobOutput (char *command, Host *host, JobOutput *output) {
   
-  int i, status, Nstart;
   char *line;
+  int i, status;
   struct timespec request, remain;
 
-  ASSERT (cmd, "cmd missing");
+  ASSERT (command, "command missing");
   ASSERT (host, "host missing");
-  ASSERT (buffer, "buffer missing");
+  ASSERT (output, "output missing");
 
-  /* flush any earlier messages */
-  ReadtoIOBuffer (buffer, host[0].stdout_fd);
-  FlushIOBuffer (buffer);
-  Nstart = buffer[0].Nbuffer;
-
-  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  /* avoid blocking on waitpid, test every 100 usec, up to 10 msec */
   request.tv_sec = 0;
   request.tv_nsec = 100000;
 
-  /* send cmd (stdout / stderr) */
-  status = write_fmt (host[0].stdin_fd, "%s\n", cmd);
+  if (!output[0].requested) {
+      /* send command (stdout / stderr) */
+      status = write_fmt (host[0].stdin_fd, "%s\n", command);
 
-  /* is pipe still open? */
-  if ((status == -1) && (errno == EPIPE)) return (PCLIENT_DOWN);
+      /* is pipe still open? */
+      if ((status == -1) && (errno == EPIPE)) return PCLIENT_DOWN;
+      output[0].requested = TRUE;
+  }
 
-  /* read at least Nbytes, then watch for PCLIENT_PROMPT */
+  if (output[0].completed) return PCLIENT_GOOD;
+
+  // attempt to read the output->size bytes from the host 
+  if (output[0].buffer.Nbuffer < output[0].size) {
+      status = -1;
+      for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (output[0].buffer.Nbuffer < output[0].size); i++) {
+	  status = ReadtoIOBuffer (&output[0].buffer, host[0].stdout_fd);
+	  if (status == -1) nanosleep (&request, &remain);
+      }
+      if (VerboseMode()) gprint (GP_ERR, "%s\n Read %d of %d bytes so far\n", output[0].buffer.buffer, output[0].buffer.Nbuffer, output[0].size);
+      if (status == 0) {
+	  if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+	  return PCLIENT_DOWN;
+      }
+      if (output[0].buffer.Nbuffer < output[0].size) {
+	  if (VerboseMode()) gprint (GP_ERR, "host %s still has data, keep trying\n", host[0].hostname);
+	  return PCLIENT_HUNG;
+      }
+  }
+
+  // keep trying to read until we get the prompt
   line = NULL;
   status = -1;
   for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) {
-    status = ReadtoIOBuffer (buffer, host[0].stdout_fd);
-    if ((buffer[0].Nbuffer - Nstart) >= Nbytes) {
-      line = memstr (buffer[0].buffer, PCLIENT_PROMPT, buffer[0].Nbuffer);
-    }
+    status = ReadtoIOBuffer (&output[0].buffer, host[0].stdout_fd);
+    line = memstr (output[0].buffer.buffer, PCLIENT_PROMPT, output[0].buffer.Nbuffer);
     if (status == -1) nanosleep (&request, &remain);
   }
-  if (status ==  0) return (PCLIENT_DOWN);
-  if (status == -1) return (PCLIENT_HUNG);
-
-  /* check on success of pclient command */
-  switch (status) {
-    case -1:
-      if (VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
-      return (FALSE);
-
-    case 0:
-      if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
-      return (FALSE);
-
-    default:
-      if (VerboseMode()) gprint (GP_ERR, "message received (GetJobOutput : %s)\n", cmd);  
-      /* drop extra bytes from pclient (not pclient:job) */
-      buffer[0].Nbuffer = Nstart + Nbytes;
-      if (buffer[0].Nalloc > buffer[0].Nbuffer) {
-	bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
-      }
-      return (TRUE);
+  if (VerboseMode()) gprint (GP_ERR, "%s\n Read %d of %d bytes so far\n", output[0].buffer.buffer, output[0].buffer.Nbuffer, output[0].size);
+  if (status == 0) {
+    if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+    return PCLIENT_DOWN;
+  }
+  if (line == NULL) {
+    if (VerboseMode()) gprint (GP_ERR, "host %s not yet at prompt, keep trying\n", host[0].hostname);
+    return PCLIENT_HUNG;
   }
 
-  gprint (GP_ERR, "programming error: should not reach here (GetJobOutput)\n");
-  pcontrol_exit (50);
+  output[0].completed = TRUE;
+  return PCLIENT_GOOD;
 }
Index: /trunk/Ohana/src/opihi/pcontrol/JobOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 20047)
@@ -175,4 +175,24 @@
 }
 
+int InitJobOutput (JobOutput *output) {
+
+  output[0].size = 0;
+  output[0].requested = FALSE;
+  output[0].completed = FALSE;
+  InitIOBuffer (&output[0].buffer, 0x1000);
+
+  return TRUE;
+}
+
+int ResetJobOutput (JobOutput *output) {
+
+  output[0].size = 0;
+  output[0].requested = FALSE;
+  output[0].completed = FALSE;
+  FlushIOBuffer (&output[0].buffer);
+
+  return TRUE;
+}
+
 IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv) {
 
@@ -189,6 +209,7 @@
   job[0].exit_status = 0;
   job[0].Reset    = FALSE;
-  job[0].stdout_size = 0;
-  job[0].stderr_size = 0;
+
+  InitJobOutput (&job[0].stdout);
+  InitJobOutput (&job[0].stderr);
 
   job[0].mode     = mode;
@@ -196,8 +217,4 @@
   job[0].state = 0;
   job[0].stack = 0;
-
-  /* do this step on start? */
-  InitIOBuffer (&job[0].stdout_buff, 0x1000);
-  InitIOBuffer (&job[0].stderr_buff, 0x1000);
 
   job[0].dtime = 0.0;
@@ -233,6 +250,6 @@
   FREE (job[0].argv);
 
-  FreeIOBuffer (&job[0].stdout_buff);
-  FreeIOBuffer (&job[0].stderr_buff);
+  FreeIOBuffer (&job[0].stdout.buffer);
+  FreeIOBuffer (&job[0].stderr.buffer);
 
   FREE (job);
Index: /trunk/Ohana/src/opihi/pcontrol/StartJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 20047)
@@ -12,4 +12,7 @@
   ASSERT (host == (Host *) job[0].host, "invalid host");
   ASSERT (job  == (Job *) host[0].job, "invalid job");
+
+  ResetJobOutput (&job[0].stdout);
+  ResetJobOutput (&job[0].stderr);
 
   /* construct command line : job arg0 arg1 ... argN\n */
@@ -80,5 +83,5 @@
 
   /* check on result of pclient command */
-  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
+  p = memstr (buffer[0].buffer, "PCLIENT_PID:", buffer[0].Nbuffer);
   if (p == NULL) {
       // failed to get a valid response.  kill the job and try again, 
Index: /trunk/Ohana/src/opihi/pcontrol/check.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/check.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/check.c	(revision 20047)
@@ -31,6 +31,6 @@
     gprint (GP_LOG, "STATUS %s\n", GetJobStackName(job[0].stack));
     gprint (GP_LOG, "EXITST %d\n", job[0].exit_status);
-    gprint (GP_LOG, "STDOUT %d\n", job[0].stdout_size);
-    gprint (GP_LOG, "STDERR %d\n", job[0].stderr_size);
+    gprint (GP_LOG, "STDOUT %d\n", job[0].stdout.size);
+    gprint (GP_LOG, "STDERR %d\n", job[0].stderr.size);
     gprint (GP_LOG, "DTIME %lf\n", job[0].dtime);
     if (job[0].realhost) {
Index: /trunk/Ohana/src/opihi/pcontrol/stdout.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/stdout.c	(revision 20046)
+++ /trunk/Ohana/src/opihi/pcontrol/stdout.c	(revision 20047)
@@ -43,5 +43,5 @@
 
 found_stdout:
-  buffer = &job[0].stdout_buff;
+  buffer = &job[0].stdout.buffer;
   if (varName == NULL) {
     fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout);
@@ -97,5 +97,5 @@
 
 found_stderr:
-  buffer = &job[0].stderr_buff;
+  buffer = &job[0].stderr.buffer;
   if (varName == NULL) {
     fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout);