Index: /trunk/Ohana/src/opihi/include/pcontrol.h
===================================================================
--- /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 17474)
+++ /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 17475)
@@ -28,8 +28,20 @@
   PCONTROL_HOST_IDLE,
   PCONTROL_HOST_BUSY,  
+  PCONTROL_HOST_RESP,
   PCONTROL_HOST_DOWN,
   PCONTROL_HOST_DONE,
   PCONTROL_HOST_OFF,
 } HostStat;
+
+/** host response options **/
+typedef enum {
+  PCONTROL_RESP_NONE,
+  PCONTROL_RESP_START_JOB,
+  PCONTROL_RESP_CHECK_BUSY_JOB,  
+  PCONTROL_RESP_CHECK_DONE_HOST,  
+  PCONTROL_RESP_CHECK_HOST,
+  PCONTROL_RESP_KILL_JOB,
+  PCONTROL_RESP_STOP_HOST,
+} HostResp;
 
 typedef enum {
@@ -97,4 +109,7 @@
   Ptime       nexttry;
   IDtype      HostID;
+  IOBuffer    comms_buffer;
+  char       *response;
+  HostResp    response_state;
   struct Job *job;
 } Host;
@@ -156,16 +171,30 @@
 
 /*** own files ***/
+int StartJob (Job *job, Host *host);
+int StartJobResponse (Host *host);
+
 int CheckHost (Host *host);
+int CheckHostResponse (Host *host);
+
+int CheckDoneHost (Host *host);
+int CheckDoneHostResponse (Host *host);
+
+int CheckBusyJob (Job *job, Host *host);
+int CheckBusyJobResponse (Host *host);
+
+int KillJob (Job *job, Host *host);
+int KillJobResponse (Host *host);
+
 int StartHost (Host *host);
 int CheckIdleHost (Host *host);
-int CheckDoneHost (Host *host);
-int CheckBusyJob (Job *job, Host *host);
 int CheckDoneJob (Job *job, Host *host);
-int KillJob (Job *job, Host *host);
-int StartJob (Job *job, Host *host);
-int ResetJob (Job *job);
 int GetJobOutput (char *command, Host *host, IOBuffer *buffer, int Nbytes);
-int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer);
 int rconnect (char *command, char *hostname, char *shell, int *stdio);
+
+int PclientCommand (Host *host, char *command, char *response, HostResp response_state);
+int PclientResponse (Host *host, char *response, IOBuffer *buffer);
+
+int CheckRespHosts (float MaxDelay);
+int CheckRespHost (Host *host);
 
 /*** misc files ***/
@@ -203,4 +232,5 @@
 int    DownHosts ();
 int    StopHost (Host *host);
+int    StopHostResponse (Host *host);
 int    HarvestHost (int pid);
 
Index: /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 17475)
@@ -4,9 +4,5 @@
 int CheckBusyJob (Job *job, Host *host) {
 
-  int      status;
-  int      outstate;
-  char    *p;
-  char     string[64];
-  IOBuffer buffer;
+  int status;
 
   /* we are checking a job which is currently busy.  it has been pulled from the
@@ -16,51 +12,53 @@
   ASSERT (job, "job not set");
   ASSERT (host, "host not set");
-
   ASSERT (host == (Host *) job[0].host, "invalid host");
   ASSERT (job  == (Job *) host[0].job, "invalid job");
 
-  InitIOBuffer (&buffer, 0x100);
-
-  status = PclientCommand (host, "status", PCLIENT_PROMPT, &buffer);
+  status = PclientCommand (host, "status", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_BUSY_JOB);
 
   /* check on success of pclient command */
   switch (status) {
     case PCLIENT_DOWN:
-      HarvestHost (host[0].pid);
+      // free the realhost name
+      if (job[0].realhost) free (job[0].realhost);
+      job[0].realhost = NULL;
+
       // unlink host & job
+      if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
       job[0].host = NULL;
       host[0].job = NULL;
-      if (job[0].realhost) free (job[0].realhost);
-      job[0].realhost = NULL;
+      HarvestHost (host[0].pid);
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
-
-    case PCLIENT_HUNG:
-      // don't do anything drastic, just keep trying
-      if (DEBUG || VerboseMode()) gprint (GP_ERR, "client is busy, not responding");
-      PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
-      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
-      return (TRUE);
 
     case PCLIENT_GOOD:
       if (DEBUG || VerboseMode()) gprint (GP_ERR, "message received (CheckBusyJob)");
-      break;
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      return (TRUE);
 
     default:
       ABORT ("unknown status for pclient command");  
   }
+}
+
+int CheckBusyJobResponse (Host *host) {
+
+  int      outstate;
+  char    *p;
+  char     string[64];
+  IOBuffer *buffer;
+  Job *job;
+
+  /* job must have assigned host */
+  ASSERT (host, "missing host");
+  ASSERT (host[0].job, "missing job");
+  buffer = &host[0].comms_buffer;
+  job = (Job *) host[0].job;
 
   /** host is up, need to parse message **/
-  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
-  // this condition means the message is garbage.  toss it and try again
-  if (p == NULL) {
-    PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
-    PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-    FreeIOBuffer (&buffer);
-    return (FALSE);
-  }
+  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
+  ASSERT (p != NULL, "missing STATUS in pclient message");
 
   sscanf (p, "%*s %s", string);
@@ -71,5 +69,4 @@
     PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
     PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-    FreeIOBuffer (&buffer);
     return (TRUE);
   }
@@ -79,12 +76,12 @@
   if (!strcmp(string, "EXIT")) outstate = PCONTROL_JOB_EXIT;
   if (!strcmp(string, "CRASH")) outstate = PCONTROL_JOB_CRASH;
-  ASSERT (outstate != PCONTROL_JOB_BUSY, "should not reach here (CheckJob)");
+  ASSERT (outstate != PCONTROL_JOB_BUSY, "invalid status response (CheckBusyJobResponse)");
 
   /* parse the exit status and sizes of output buffers */
-  p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
+  p = memstr (buffer[0].buffer, "EXITST", buffer[0].Nbuffer);
   sscanf (p, "%*s %d", &job[0].exit_status);
-  p = memstr (buffer.buffer, "STDOUT", buffer.Nbuffer);
+  p = memstr (buffer[0].buffer, "STDOUT", buffer[0].Nbuffer);
   sscanf (p, "%*s %d", &job[0].stdout_size);
-  p = memstr (buffer.buffer, "STDERR", buffer.Nbuffer);
+  p = memstr (buffer[0].buffer, "STDERR", buffer[0].Nbuffer);
   sscanf (p, "%*s %d", &job[0].stderr_size);
 
@@ -98,7 +95,6 @@
   PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
   PutJobSetState (job, PCONTROL_JOB_DONE, STACK_BOTTOM, outstate);
-  gettimeofday (&job[0].stop, (void *) NULL);
+  gettimeofday (&job[0].stop, NULL);
   job[0].dtime = DTIME(job[0].stop, job[0].start);
-  FreeIOBuffer (&buffer);
   return (TRUE);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 17475)
@@ -5,12 +5,8 @@
   
   int       status;
-  char     *p;
-  IOBuffer  buffer;
 
   ASSERT (host, "host not set");
 
-  InitIOBuffer (&buffer, 0x100);
-  
-  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+  status = PclientCommand (host, "reset", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_DONE_HOST);
 
   /* check on success of pclient command */
@@ -18,28 +14,34 @@
     case PCLIENT_DOWN:
       if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
-      /* DONE host does not have an incomplete job */
       HarvestHost (host[0].pid);
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
+      /* DONE host does not have an incomplete job */
       // XXX do we need to close the connection?
 
-    case PCLIENT_HUNG:
-      // don't do anything drastic, just try again later
-      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
-      if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
-      FreeIOBuffer (&buffer);
-      return (FALSE);
-
     case PCLIENT_GOOD:
-      if (VerboseMode()) gprint (GP_ERR, "message received (CheckDoneHost)\n");  
-      break;
+      if (VerboseMode()) gprint (GP_ERR, "checking done host %s\n", host[0].hostname);  
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      return (TRUE);
 
     default:
       ABORT ("unknown status for pclient command");  
   }
+  ABORT ("should not reach here (CheckDoneHost)"); 
+}
+
+int CheckDoneHostResponse (Host *host) {
+
+  int status;
+  char *p;
+  IOBuffer *buffer;
+
+  /* job must have assigned host */
+  ASSERT (host, "missing host");
+  ASSERT (host[0].job, "missing job");
+  buffer = &host[0].comms_buffer;
 
   /** successful command, examine result **/
-  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
   ASSERT (p != NULL, "missing STATUS in pclient message (CheckDoneHost)");
 
@@ -52,5 +54,4 @@
       if (DEBUG || VerboseMode()) gprint (GP_ERR, "reset failed\n");
       PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
       
@@ -59,5 +60,4 @@
       if (DEBUG || VerboseMode()) gprint (GP_ERR, "successful reset\n");
       PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
 
@@ -67,5 +67,2 @@
   ABORT ("should not reach here (CheckDoneHost)");
 }
-
-/** probably need to flush the buffer before the command **/
-/** need to add timeout check here **/
Index: /trunk/Ohana/src/opihi/pcontrol/CheckHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 17475)
@@ -1,8 +1,9 @@
 # include "pcontrol.h"
 
+// if the host has a job, we skip it (down or crash state will be caught elsewhere)
+// in fact, just touch the IDLE hosts, not the BUSY hosts?
 int CheckHost (Host *host) {
   
   int status;
-  IOBuffer buffer;
 
   ASSERT (host, "host not set");
@@ -18,30 +19,35 @@
   }
 
-  InitIOBuffer (&buffer, 0x100);
+  // the argument to echo (OK) is the expected response below in CheckHostResponse
+  status = PclientCommand (host, "echo OK", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_HOST);
 
-  status = PclientCommand (host, "echo OK", PCLIENT_PROMPT, &buffer);
   switch (status) {
-    case 0:
+    case PCLIENT_DOWN:
       if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
       HarvestHost (host[0].pid);
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
       
-    case -1:
-      if (VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
-      /*** do we mark this in some way (HUNG) ? ***/
-      PutHost (host, host[0].stack, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
-      return (FALSE);
+    case PCLIENT_GOOD:
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      return (TRUE);
 
     default:
-      PutHost (host, host[0].stack, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
-      return (TRUE);
+      ABORT ("unknown status for pclient command");  
   }
-  ABORT ("should not reach here (Check Host)"); 
+  ABORT ("should not reach here (CheckHost)"); 
 }
 
-// if the host has a job, we skip it (down or crash state will be caught elsewhere)
-// in fact, just touch the IDLE hosts, not the BUSY hosts?
+int CheckHostResponse (Host *host) {
+  
+  IOBuffer *buffer;
+
+  /* we only check IDLE hosts without jobs */
+  ASSERT (host, "missing host");
+  buffer = &host[0].comms_buffer;
+
+  // XXX check on the value of the response? (OK)
+
+  PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+  return (TRUE);
+}
Index: /trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c	(revision 17475)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c	(revision 17475)
@@ -0,0 +1,83 @@
+# include "pcontrol.h"
+# define DEBUG 0
+
+// this function operates on hosts waiting for a response. we simply check if the message is
+// complete, and if so, send it to the correct parsing function
+int CheckRespHost (Host *host) {
+  
+  int status;
+  Job  *job;
+
+  ASSERT (host, "host not set");
+  job = (Job *) host[0].job;
+
+  status = PclientResponse (host, host[0].response, &host[0].comms_buffer);
+
+  /* check on output from pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+
+      // not all hosts here have a job; if it does, return it to PENDING
+      if (job) {
+	// unlink host & job
+	if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+	job[0].host = NULL;
+	host[0].job = NULL;
+	PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      }
+
+      // host has shutdown; harvest the defunct process
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      // not done yet; try again later
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      if (job) {
+	PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      }
+      if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      if (VerboseMode()) gprint (GP_ERR, "message received (CheckRespHost)\n");  
+      break;
+
+    default:
+      ABORT ("unknown status for pclient command");  
+  }
+
+  switch (host[0].response_state) {
+    case PCONTROL_RESP_START_JOB:
+      status = StartJobResponse (host);
+      return (status);
+
+    case PCONTROL_RESP_CHECK_HOST:
+      status = CheckHostResponse (host);
+      return (status);
+
+    case PCONTROL_RESP_CHECK_DONE_HOST:
+      status = CheckDoneHostResponse (host);
+      return (status);
+
+    case PCONTROL_RESP_CHECK_BUSY_JOB:
+      status = CheckBusyJobResponse (host);
+      return (status);
+
+    case PCONTROL_RESP_KILL_JOB:
+      status = KillJobResponse (host);
+      return (status);
+
+    case PCONTROL_RESP_STOP_HOST:
+      status = StopHostResponse (host);
+      return (status);
+
+    default:
+      ABORT ("undefined response state");
+  }
+
+  ABORT ("should not be able to get here");
+}      
+
Index: /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 17475)
@@ -99,4 +99,6 @@
 
     if (RunLevel != PCONTROL_RUN_NONE) {
+      Nhostchecks += CheckRespHosts(0.020); /* check for incoming messages */
+      TestCheckPoint ();
       Nhostchecks += CheckDoneHosts(0.020); /* reset the host */
       TestCheckPoint ();
@@ -258,4 +260,30 @@
 }
 
+int CheckRespHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  /* Loop through objects on the stack, no more than once. see note above */
+  stack = GetHostStack (PCONTROL_HOST_RESP);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = PullStackByLocation (stack, STACK_TOP);
+    if (host == NULL) break;
+    CheckRespHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
+  return (i);
+}
+
 int CheckDoneHosts (float MaxDelay) {
 
Index: /trunk/Ohana/src/opihi/pcontrol/HostOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/HostOps.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/HostOps.c	(revision 17475)
@@ -1,13 +1,15 @@
 # include "pcontrol.h"
 
-Stack *HostPool_Idle;
-Stack *HostPool_Busy;
-Stack *HostPool_Done;
-Stack *HostPool_Down;
-Stack *HostPool_Off;
+Stack *HostPool_Idle; // these hosts are waiting for something to do
+Stack *HostPool_Busy; // these hosts are working
+Stack *HostPool_Resp; // these hosts are trying to respond
+Stack *HostPool_Done; // these hosts have finished a job
+Stack *HostPool_Down; // these hosts are not responding
+Stack *HostPool_Off;  // these hosts are off
 
 void InitHostStacks () {
   HostPool_Idle = InitStack ();
   HostPool_Busy = InitStack ();
+  HostPool_Resp = InitStack ();
   HostPool_Done = InitStack ();
   HostPool_Down = InitStack ();
@@ -26,4 +28,5 @@
   FreeHostStack (HostPool_Idle);
   FreeHostStack (HostPool_Busy);
+  FreeHostStack (HostPool_Resp);
   FreeHostStack (HostPool_Done);
   FreeHostStack (HostPool_Down);
@@ -35,4 +38,5 @@
     case PCONTROL_HOST_IDLE: return ("IDLE");
     case PCONTROL_HOST_DOWN: return ("DOWN");
+    case PCONTROL_HOST_RESP: return ("RESP");
     case PCONTROL_HOST_DONE: return ("DONE");
     case PCONTROL_HOST_BUSY: return ("BUSY");
@@ -48,4 +52,5 @@
     case PCONTROL_HOST_IDLE: return (HostPool_Idle);
     case PCONTROL_HOST_DOWN: return (HostPool_Down);
+    case PCONTROL_HOST_RESP: return (HostPool_Resp);
     case PCONTROL_HOST_DONE: return (HostPool_Done);
     case PCONTROL_HOST_BUSY: return (HostPool_Busy);
@@ -60,4 +65,5 @@
   if (!strcasecmp (name, "idle")) return (HostPool_Idle);
   if (!strcasecmp (name, "down")) return (HostPool_Down);
+  if (!strcasecmp (name, "resp")) return (HostPool_Resp);
   if (!strcasecmp (name, "done")) return (HostPool_Done);
   if (!strcasecmp (name, "busy")) return (HostPool_Busy);
@@ -94,4 +100,8 @@
   if (host != NULL) return (host);
 
+  *StackID = PCONTROL_HOST_RESP;
+  host = PullHostFromStackByID (*StackID, HostID);
+  if (host != NULL) return (host);
+
   *StackID = PCONTROL_HOST_DONE;
   host = PullHostFromStackByID (*StackID, HostID);
@@ -120,4 +130,8 @@
 
   *StackID = PCONTROL_HOST_DOWN;
+  host = PullHostFromStackByName (*StackID, name);
+  if (host != NULL) return (host);
+
+  *StackID = PCONTROL_HOST_RESP;
   host = PullHostFromStackByName (*StackID, name);
   if (host != NULL) return (host);
@@ -180,4 +194,8 @@
   host[0].nexttry.tv_usec = 0;
 
+  InitIOBuffer (&host[0].comms_buffer, 0x100);
+  host[0].response_state = PCONTROL_RESP_NONE;
+  host[0].response = NULL;
+
   host[0].markoff  = FALSE;
   host[0].job      = NULL;
@@ -187,4 +205,5 @@
 
 void DelHost (Host *host) {
+  FreeIOBuffer (&host[0].comms_buffer);
   FREE (host[0].hostname);
   FREE (host[0].job);
Index: /trunk/Ohana/src/opihi/pcontrol/KillJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/KillJob.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/KillJob.c	(revision 17475)
@@ -3,52 +3,55 @@
 int KillJob (Job *job, Host *host) {
   
-  IOBuffer buffer;
   int status;
-  char *p;
 
   ASSERT (host != NULL, "host missing");
   ASSERT (job != NULL, "job missing");
-
   ASSERT (host == (Host *) job[0].host, "invalid host");
   ASSERT (job  == (Job *) host[0].job, "invalid job");
 
-  InitIOBuffer (&buffer, 0x100);
-
-  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+  status = PclientCommand (host, "reset", PCLIENT_PROMPT, PCONTROL_RESP_KILL_JOB);
 
   /* check on success of pclient command */
   switch (status) {
     case PCLIENT_DOWN:
-      HarvestHost (host[0].pid);
       // unlink host & job
+      if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
       job[0].host = NULL;
       host[0].job = NULL;
+      HarvestHost (host[0].pid);
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_CRASH, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
 
-    case PCLIENT_HUNG:
-      // don't do anything drastic, just keep trying
-      // XXX move to which stack??
-      gprint (GP_ERR, "client is busy, not responding (KillJob)");
-      FreeIOBuffer (&buffer);
+    case PCLIENT_GOOD:
+      if (VerboseMode()) gprint (GP_ERR, "kill job on host %s\n", host[0].hostname);  
+      FlushIOBuffer (&host[0].comms_buffer);
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
       return (TRUE);
-
-    case PCLIENT_GOOD:
-      if (VerboseMode()) gprint (GP_ERR, "message received (KillJob)\n");  
-      break;
 
     default:
       ABORT ("unknown status for pclient command");  
   }
+}
 
-  /** host is up, need to parse message **/
-  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+int KillJobResponse (Host *host) {
+  
+  int status;
+  char *p;
+  IOBuffer *buffer;
+  Job *job;
+
+  ASSERT (host != NULL, "host missing");
+  ASSERT (host[0].job, "missing job");
+  buffer = &host[0].comms_buffer;
+  job = (Job *) host[0].job;
+
+  /** check on response to pclient command **/
+  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
   ASSERT (p != NULL, "missing STATUS in pclient message");
-  if (VerboseMode()) gprint (GP_ERR, "client message: %s\n", buffer.buffer);
+  if (VerboseMode()) gprint (GP_ERR, "client message: %s\n", buffer[0].buffer);
 
   sscanf (p, "%*s %d", &status);
-  FreeIOBuffer (&buffer);
   gprint (GP_ERR, "client status: %d\n", status);
 
@@ -62,5 +65,5 @@
       return (FALSE);
     case 1:
-      gprint (GP_ERR, "killing job %s on %s\n", job[0].argv[0], host[0].hostname);
+      gprint (GP_ERR, "killed job %s on %s\n", job[0].argv[0], host[0].hostname);
       // unlink host & job
       job[0].host = NULL;
Index: /trunk/Ohana/src/opihi/pcontrol/Makefile
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 17475)
@@ -13,5 +13,5 @@
 LIBS1         = -lkapa -lFITS -lohana
 LIBS2         = -lbasiccmd -lshell -ldata 
-FULL_CFLAGS   = $(BASE_CFLAGS)
+FULL_CFLAGS   = $(BASE_CFLAGS) -Wall -Werror
 FULL_CPPFLAGS = $(BASE_CPPFLAGS)
 FULL_LDFLAGS  = $(LIBS1) $(LIBS2) $(BASE_LDFLAGS)
@@ -27,4 +27,5 @@
 $(SRC)/CheckBusyJob.$(ARCH).o \
 $(SRC)/CheckDoneHost.$(ARCH).o \
+$(SRC)/CheckRespHost.$(ARCH).o \
 $(SRC)/CheckDoneJob.$(ARCH).o \
 $(SRC)/CheckHost.$(ARCH).o \
@@ -38,5 +39,4 @@
 $(SRC)/StackOps.$(ARCH).o \
 $(SRC)/PclientCommand.$(ARCH).o \
-$(SRC)/ResetJob.$(ARCH).o \
 $(SRC)/StartHost.$(ARCH).o \
 $(SRC)/StopHosts.$(ARCH).o \
Index: /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 17475)
@@ -1,26 +1,19 @@
 # include "pcontrol.h"
-# define PCLIENT_TIMEOUT 20000
+# define PCLIENT_TIMEOUT 100
 
-int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer) {
+// send a command and check for errors; ignore output
+int PclientCommand (Host *host, char *command, char *response, HostResp response_state) {
 
-  int i;
   int status;
-  char *line;
-  struct timespec request, remain;
+  IOBuffer buffer;
 
   ASSERT (host != NULL, "host missing");
-  ASSERT (buffer != NULL, "buffer missing");
   ASSERT (command != NULL, "command missing");
-  ASSERT (response != NULL, "response missing");
-
-  /* avoid blocking on read, test every 100 usec, up to 2.0 sec */
-  request.tv_sec = 0;
-  request.tv_nsec = 100000;
 
   // flush the stdout and stderr buffers here
-  ReadtoIOBuffer (buffer, host[0].stdout_fd);
-  FlushIOBuffer (buffer);
-  ReadtoIOBuffer (buffer, host[0].stderr_fd);
-  FlushIOBuffer (buffer);
+  ReadtoIOBuffer (&buffer, host[0].stdout_fd);
+  FlushIOBuffer (&buffer);
+  ReadtoIOBuffer (&buffer, host[0].stderr_fd);
+  FlushIOBuffer (&buffer);
 
   /* send command to client (adding on \n) */
@@ -33,8 +26,33 @@
   }
   
+  // prepare host to accept response
+  host[0].response_state = response_state;
+  host[0].response = response;
+  FlushIOBuffer (&host[0].comms_buffer);
+
+  return (PCLIENT_GOOD);
+}
+  
+// check for response; message must end with specified string.
+// accumulate the response in the buffer
+int PclientResponse (Host *host, char *response, IOBuffer *buffer) {
+
+  int i;
+  int status;
+  char *line;
+  struct timespec request, remain;
+
+  ASSERT (response != NULL, "response missing");
+  ASSERT (buffer != NULL, "buffer missing");
+
+  /* avoid blocking very long on read, test every 100 usec, up to 0.1 sec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
   /* watch for response - wait up to 1 second */
   line = NULL;
   status = -1;
 
+  // how long does each cycle really take?
   for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) {
     status = ReadtoIOBuffer (buffer, host[0].stdout_fd);
@@ -46,8 +64,13 @@
     return (PCLIENT_DOWN);
   }
+  if (line == NULL) return (PCLIENT_HUNG);
   if (status == -1) return (PCLIENT_HUNG);
-  if (line == NULL) return (PCLIENT_HUNG);
 
   // fprintf (stderr, "buffer.buffer: %s\n", buffer[0].buffer);
+
+  // we have detected a valid response, clear the response data
+  host[0].response_state = PCONTROL_RESP_NONE;
+  host[0].response = NULL;
+
   return (PCLIENT_GOOD);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/ResetJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 17475)
@@ -1,8 +1,9 @@
 # include "pcontrol.h"
+
+// XXX deprecated
 
 int ResetJob (Job *job) {
   
   int       status;
-  IOBuffer  buffer;
   Host     *host;
 
@@ -13,41 +14,42 @@
   ASSERT (job != NULL, "host missing");
 
-  InitIOBuffer (&buffer, 0x100);
-  
   /* we have tried to reset the job; may not get status */
   job[0].Reset = TRUE;
 
-  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+  status = PclientCommand (host, "reset");
 
   /* check on success of pclient command */
   switch (status) {
     case PCLIENT_DOWN:
-      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST? ***/
-      gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+      if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
       HarvestHost (host[0].pid);
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
-      return (FALSE);
-
-    case PCLIENT_HUNG:
-      /*** should we consider a HUNG host DOWN? ***/
-      gprint (GP_ERR, "host %s is not responding (ResetJob)\n", host[0].hostname);
-      FreeIOBuffer (&buffer);
       return (FALSE);
 
     case PCLIENT_GOOD:
-      gprint (GP_ERR, "message received (ResetJob)\n");  
-      FreeIOBuffer (&buffer);
+      host[0].response_state = PCONTROL_RESP_RESET_JOB;
+      host[0].response = PCLIENT_PROMPT;
+      FlushIOBuffer (&host[0].comms_buffer, 0x100);
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
       return (TRUE);
 
     default:
-      gprint (GP_ERR, "unknown status for pclient command: programming error\n");  
-      pcontrol_exit (55);
+      ABORT ("unknown status for pclient command");  
   }
+  ABORT ("should not reach here (ResetJob)"); 
+}
 
-  gprint (GP_ERR, "programming error in ResetJob (should not reach here)\n");
-  FreeIOBuffer (&buffer);
-  pcontrol_exit (56);
-  return (FALSE);
+int ResetJobResponse (Host *host) {
+  
+  int       status;
+  IOBuffer *buffer;
+
+  /* job must have assigned host */
+  ASSERT (host, "missing host");
+  ASSERT (host[0].job, "missing job");
+  buffer = host[0].comms_buffer;
+
+  gprint (GP_ERR, "message received (ResetJob)\n");  
+  return (TRUE);
 }
 
Index: /trunk/Ohana/src/opihi/pcontrol/StartJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 17475)
@@ -1,11 +1,9 @@
 # include "pcontrol.h"
 
+// job and host are bound together (why pass in both?)
 int StartJob (Job *job, Host *host) {
 
   int  i, Nline, status;
-  char *line, *p;
-  IOBuffer buffer;
-
-  InitIOBuffer (&buffer, 0x100);
+  char *line;
 
   /* job must have assigned host */
@@ -28,5 +26,5 @@
   }
 
-  status = PclientCommand (host, line, PCLIENT_PROMPT, &buffer);
+  status = PclientCommand (host, line, PCLIENT_PROMPT, PCONTROL_RESP_START_JOB);
   free (line);
 
@@ -34,35 +32,44 @@
   switch (status) {
     case PCLIENT_DOWN:
+      // unlink host & job
       if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
-      goto failure;
-
-    case PCLIENT_HUNG:
-      // we need the job start to return a valid Job ID, 
-      // give up on jobs which don't get started.
-      // XXX we are sensitive here to the time it takes pclient
-      // to fork the job.  if this is slow, the client may appear to hang.
-      gprint (GP_ERR, "host %s is not responding (StartJob)\n", host[0].hostname);
-      if (VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
-
-      // unlink host & job
       job[0].host = NULL;
       host[0].job = NULL;
-      if (job[0].realhost) free (job[0].realhost);
-      job[0].realhost = NULL;
-      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
       return (FALSE);
 
     case PCLIENT_GOOD:
-      if (VerboseMode()) gprint (GP_ERR, "message received (StartJob)\n");  
-      break;
+      job[0].realhost = strcreate (host[0].hostname);
+      job[0].pid = -1;
+      gettimeofday (&job[0].start, (void *) NULL);
+
+      if (VerboseMode()) gprint (GP_ERR, "started job on host %s\n", host[0].hostname);  
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      return (TRUE);
 
     default:
       ABORT ("unknown status for pclient command");  
   }
+}
+
+// message has been received from the host, interpret results
+int StartJobResponse (Host *host) {
+  
+  int status;
+  char *p;
+  IOBuffer *buffer;
+  Job *job;
+
+  /* job must have assigned host */
+  ASSERT (host, "missing host");
+  ASSERT (host[0].job, "missing job");
+  buffer = &host[0].comms_buffer;
+  job = (Job *) host[0].job;
 
   /* check on result of pclient command */
-  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer);
   ASSERT (p != NULL, "missing STATUS in pclient message");
 
@@ -71,5 +78,11 @@
     case -1:
       if (VerboseMode()) gprint (GP_ERR, "error in pclient child\n");
-      goto failure;
+      // unlink host & job
+      job[0].host = NULL;
+      host[0].job = NULL;
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      return (FALSE);
 
     case -2:
@@ -80,23 +93,13 @@
 
     default:
-      job[0].realhost = strcreate (host[0].hostname);
+      if (VerboseMode()) gprint (GP_ERR, "message received (StartJobResponse)\n");  
       job[0].pid = status;
       PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-      FreeIOBuffer (&buffer);
-      gettimeofday (&job[0].start, (void *) NULL);
+      gettimeofday (&job[0].start, NULL);
       return (TRUE);
   }
+
   /* we should never reach here */
   ABORT ("should not reach here (StartJob)");
-
-failure:
-  // unlink host & job
-  job[0].host = NULL;
-  host[0].job = NULL;
-  HarvestHost (host[0].pid);
-  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
-  PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
-  FreeIOBuffer (&buffer);
-  return (FALSE);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/StopHosts.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 17474)
+++ /trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 17475)
@@ -72,26 +72,27 @@
 
   int       status;
-  IOBuffer  buffer;
 
-  InitIOBuffer (&buffer, 0x100);
-  status = PclientCommand (host, "exit", "Goodbye", &buffer);
-  FreeIOBuffer (&buffer);
+  status = PclientCommand (host, "exit", "Goodbye", PCONTROL_RESP_STOP_HOST);
 
   /* check on success of pclient command */
   switch (status) {
     case PCLIENT_DOWN:
-      break;
-
-    case PCLIENT_HUNG:
-      gprint (GP_ERR, "host %s is not responding\n", host[0].hostname);
+      // XXX this is the desired result in any case, so ignore it
       break;
 
     case PCLIENT_GOOD:
-      break;
+      if (VerboseMode()) gprint (GP_ERR, "stop host %s\n", host[0].hostname);  
+      FlushIOBuffer (&host[0].comms_buffer);
+      PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM);
+      return (TRUE);
 
     default:
-      gprint (GP_ERR, "unknown status for pclient command: programming error\n");  
-      pcontrol_exit (57);
+      ABORT ("unknown status for pclient command");  
   }
+  ABORT ("should not reach here");  
+}
+
+int StopHostResponse (Host *host) {
+
   HarvestHost (host[0].pid);
   return (TRUE);