Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckBusyJob.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 5922)
@@ -0,0 +1,86 @@
+# include "pcontrol.h"
+
+int CheckBusyJob (Job *job) {
+
+  int      status;
+  int      outstate;
+  char    *p;
+  char     string[64];
+  IOBuffer buffer;
+  Host    *host;
+
+  /** must have a valid host : if not? **/
+  host = (Host *) job[0].host;
+
+  InitIOBuffer (&buffer, 0x100);
+
+  status = PclientCommand (host, "status", PCLIENT_PROMPT, &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      HarvestHost (host[0].pid);
+      UnlinkJobAndHost (job);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      job[0].state = PCONTROL_JOB_HUNG;
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+
+    case PCLIENT_GOOD:
+      if (VerboseMode()) fprintf (stderr, "message received (CheckBusyJob)\n");
+      break;
+
+    default:
+      fprintf (stderr, "programming error: unknown status for pclient command\n");  
+      exit (1);
+  }
+
+  /** host is up, need to parse message **/
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "programming error: missing STATUS in pclient message\n");
+    exit (1);
+  }
+  sscanf (p, "%*s %s", string);
+  if (!strcmp(string, "NONE")) {
+    fprintf (stderr, "programming error: no current job\n");
+    exit (1);
+  }
+  /** no status change, return to BUSY stack **/
+  if (!strcmp(string, "BUSY")) {
+    PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+    FreeIOBuffer (&buffer);
+    return (TRUE);
+  }
+
+  /* exit status better be either EXIT or CRASH */
+  outstate = PCONTROL_JOB_BUSY;
+  if (!strcmp(string, "EXIT")) outstate = PCONTROL_JOB_EXIT;
+  if (!strcmp(string, "CRASH")) outstate = PCONTROL_JOB_CRASH;
+  if (outstate == PCONTROL_JOB_BUSY) {
+    fprintf (stderr, "programming error : should not reach here (CheckJob)\n");
+    exit (1);
+  }
+
+  /* parse the exit status and sizes of output buffers */
+  p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
+  sscanf (p, "%*s %d", &job[0].exit_status);
+  p = memstr (buffer.buffer, "STDOUT", buffer.Nbuffer);
+  sscanf (p, "%*s %d", &job[0].stdout_size);
+  p = memstr (buffer.buffer, "STDERR", buffer.Nbuffer);
+  sscanf (p, "%*s %d", &job[0].stderr_size);
+
+  /** job has exited : move to DONE stack (host still BUSY) **/
+  PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+  job[0].state = outstate;
+  FreeIOBuffer (&buffer);
+  return (TRUE);
+}
+
+/** need to add timeout check here **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneHost.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 5922)
@@ -0,0 +1,73 @@
+# include "pcontrol.h"
+
+int CheckDoneHost (Host *host) {
+  
+  int       status;
+  char     *p;
+  IOBuffer  buffer;
+
+  InitIOBuffer (&buffer, 0x100);
+  
+  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      if (VerboseMode()) fprintf (stderr, "host %s is down\n", host[0].hostname);
+      /* DONE host does not have an incomplete job */
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+      /** do we need to close the connection? **/
+
+    case PCLIENT_HUNG:
+      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+      if (VerboseMode()) fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      if (VerboseMode()) fprintf (stderr, "message received (CheckDoneHost)\n");  
+      break;
+
+    default:
+      if (VerboseMode()) fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  /** successful command, examine result **/
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "programming error: missing STATUS in pclient message (CheckDoneHost)\n");
+    exit (1);
+  }
+  sscanf (p, "%*s %d", &status);
+  switch (status) {
+    case -1:
+      fprintf (stderr, "programming error: reset syntax error\n");
+      exit (1);
+      
+    case 0:
+      if (VerboseMode()) fprintf (stderr, "reset failed\n");
+      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+      
+    case 1:
+    case 2:
+      if (VerboseMode()) fprintf (stderr, "successful reset\n");
+      PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    default:
+      fprintf (stderr, "programming error: should not reach here (CheckDoneHost)\n");
+      exit (1);
+  }
+  fprintf (stderr, "programming error: should not reach here either (CheckDoneHost)\n");
+  exit (1);
+}
+
+/** probably need to flush the buffer before the command **/
+/** need to add timeout check here **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneJob.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 5922)
@@ -0,0 +1,26 @@
+# include "pcontrol.h"
+
+int CheckDoneJob (Job *job) {
+  
+  Host *host;
+
+  if (!GetJobOutput ("stdout", (Host *) job[0].host, &job[0].stdout, job[0].stdout_size)) {
+    /* strip off first and last lines */
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    return (FALSE);
+  }
+
+  if (!GetJobOutput ("stderr", (Host *) job[0].host, &job[0].stderr, job[0].stderr_size)) {
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    return (FALSE);
+  }
+
+  host = UnlinkJobAndHost (job);
+  PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+  PutJob (job, job[0].state, STACK_BOTTOM);
+  /* job's state is either EXIT or CRASH (verify?) */
+
+  return (TRUE);
+}
+
+/** need to add timeout check here **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckHost.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 5922)
@@ -0,0 +1,40 @@
+# include "pcontrol.h"
+
+int CheckHost (Host *host) {
+  
+  int status;
+  IOBuffer buffer;
+  Job *job;
+
+  InitIOBuffer (&buffer, 0x100);
+
+  status = PclientCommand (host, "echo OK", PCLIENT_PROMPT, &buffer);
+  switch (status) {
+    case 0:
+      if (VerboseMode()) fprintf (stderr, "host %s is down\n", host[0].hostname);
+      /* if host has a job, job is dead, return to Pending */
+      job = (Job *) host[0].job;
+      if (job != NULL) {
+	UnlinkJobAndHost (job);
+	PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      }
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+      
+    case -1:
+      if (VerboseMode()) fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      /*** do we mark this in some way (HUNG) ? ***/
+      PutHost (host, host[0].stack, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    default:
+      PutHost (host, host[0].stack, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+  }
+  fprintf (stderr, "programming error: should not reach here (Check Host)\n"); 
+  return (FALSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckIdleHost.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 5922)
@@ -0,0 +1,54 @@
+# include "pcontrol.h"
+
+int CheckIdleHost (Host *host) {
+
+  int i;
+  Stack *stack;
+  Job *job;
+
+  stack = GetJobStack (PCONTROL_JOB_PENDING);
+  
+  /* look for first NEEDHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].mode != PCONTROL_JOB_NEEDHOST) continue;
+    if (job[0].hostname == NULL) {
+      fprintf (stderr, "programming error: NEEDHOST hostname missing\n");
+      exit (2);
+    }
+    if (strcasecmp (job[0].hostname, host[0].hostname)) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+
+  /* no NEEDHOST entry, look for first WANTHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    /*** this currently never runs WANTHOST jobs on any other machines ***/
+    if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
+    if (job[0].hostname == NULL) {
+      fprintf (stderr, "programming error: WANTHOST hostname missing\n");
+      exit (2);
+    }
+    if (strcasecmp (job[0].hostname, host[0].hostname)) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+
+  /* no WANTHOST entry, look for first ANYHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].mode != PCONTROL_JOB_ANYHOST) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+  /* no jobs for host, but back on IDLE stack */
+  PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+  return (TRUE);
+}
+
+/** note : host and job popped off IDLE and PENDING stacks, 
+    unless no job is available **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 5922)
@@ -0,0 +1,222 @@
+# include "pcontrol.h"
+
+static struct timeval lastlive = {0, 0};
+
+int CheckSystem () {
+
+  struct timeval now;
+  float dtime;
+
+  /* we want to give each block a maximum allowed time */
+  CheckIdleHosts(0.020); /* submit a new job */
+
+  CheckBusyJobs(0.020);  /* get job status */
+  CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
+
+  CheckDoneHosts(0.020); /* reset the host */
+  CheckDownHosts(0.100); /* launch the host */
+
+  /* always allow at least one test */
+  /* most tests require about 2ms per host.  
+     CheckDoneJobs must depend on the size of the output buffer */
+
+  gettimeofday (&now, (void *) NULL);
+  dtime = DTIME (now, lastlive);
+  if (dtime > 1.0) {
+    CheckLiveHosts(0.040);
+    lastlive = now;
+  } 
+
+  if (0) { 
+    Stack *stack;
+    int Nidle, Ndown, Nbusy;
+    stack = GetHostStack (PCONTROL_HOST_IDLE);
+    Nidle = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_DOWN);
+    Ndown = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_BUSY);
+    Nbusy = stack[0].Nobject;
+    fprintf (stderr, "busy, idle, down: %2d %2d %2d\n", Nbusy, Nidle, Ndown);
+  }
+
+  return (TRUE);
+}
+
+int CheckBusyJobs (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Job   *job;
+  float dtime;
+
+  stack = GetJobStack (PCONTROL_JOB_BUSY);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    job = GetStack (stack, STACK_TOP);
+    CheckBusyJob (job);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneJobs (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Job   *job;
+  float dtime;
+
+  stack = GetJobStack (PCONTROL_JOB_DONE);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    job = GetStack (stack, STACK_TOP);
+    CheckDoneJob (job);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  stack = GetHostStack (PCONTROL_HOST_DONE);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckDoneHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckDownHosts (float MaxDelay) {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  struct timeval start, stop;
+  float dtime;
+
+  stack = GetHostStack (PCONTROL_HOST_DOWN);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    dtime = DTIME (host[0].nexttry, start);
+    if (dtime > 0) {
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+    } else {
+      StartHost (host);
+    }
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckIdleHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  /* check if there are any pending jobs, otherwise skip step */
+  stack = GetJobStack (PCONTROL_JOB_PENDING);
+  if (!stack[0].Nobject) return (TRUE);
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckIdleHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+/* this is just a heartbeat check */
+int CheckLiveHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  gettimeofday (&start, (void *) NULL);
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d idle hosts\n", i);
+
+  stack = GetHostStack (PCONTROL_HOST_BUSY);
+  Nobject = stack[0].Nobject;
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d busy hosts\n", i);
+  return (TRUE);
+}
+
+/*
+
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check 4: %f seconds\n", dtime);
+
+  gettimeofday (&start, (void *) NULL);
+*/
+
+/** All of the CheckFooBar entries cycle though their respective queues, popping from the top and
+    pushing to the bottom.  if we stop before the loop is done there is no tendancy for bias because
+    we continue where we left off next round **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/GetJobOutput.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 5922)
@@ -0,0 +1,61 @@
+# include "pcontrol.h"
+# define PCLIENT_TIMEOUT 500
+
+/* we read Nbytes from the host, then watch for the prompt */ 
+int GetJobOutput (char *cmd, Host *host, IOBuffer *buffer, int Nbytes) {
+  
+  int i, status, Nstart;
+  char *line;
+  struct timespec request, remain;
+
+  /* flush any earlier messages */
+  ReadtoIOBuffer (buffer, host[0].stdout);
+  FlushIOBuffer (buffer);
+  Nstart = buffer[0].Nbuffer;
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
+  /* send cmd (stdout / stderr) */
+  status = write_fmt (host[0].stdin, "%s\n", cmd);
+
+  /* is pipe still open? */
+  if ((status == -1) && (errno == EPIPE)) return (PCLIENT_DOWN);
+
+  /* read at least Nbytes, then watch for PCLIENT_PROMPT */
+  line = NULL;
+  status = -1;
+  for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) {
+    status = ReadtoIOBuffer (buffer, host[0].stdout);
+    if ((buffer[0].Nbuffer - Nstart) >= Nbytes) {
+      line = memstr (buffer[0].buffer, PCLIENT_PROMPT, buffer[0].Nbuffer);
+    }
+    if (status == -1) nanosleep (&request, &remain);
+  }
+  if (status ==  0) return (PCLIENT_DOWN);
+  if (status == -1) return (PCLIENT_HUNG);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case -1:
+      if (VerboseMode()) fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      return (FALSE);
+
+    case 0:
+      if (VerboseMode()) fprintf (stderr, "host %s is down\n", host[0].hostname);
+      return (FALSE);
+
+    default:
+      if (VerboseMode()) fprintf (stderr, "message received (GetJobOutput : %s)\n", cmd);  
+      /* drop extra bytes from pclient (not pclient:job) */
+      buffer[0].Nbuffer = Nstart + Nbytes;
+      if (buffer[0].Nalloc > buffer[0].Nbuffer) {
+	bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
+      }
+      return (TRUE);
+  }
+
+  fprintf (stderr, "programming error: should not reach here (GetJobOutput)\n");
+  exit (1);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/HostOps.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/HostOps.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/HostOps.c	(revision 5922)
@@ -0,0 +1,204 @@
+# include "pcontrol.h"
+
+Stack *HostPool_Idle;
+Stack *HostPool_Busy;
+Stack *HostPool_Done;
+Stack *HostPool_Down;
+Stack *HostPool_Off;
+
+void InitHostStacks () {
+  HostPool_Idle = InitStack ();
+  HostPool_Busy = InitStack ();
+  HostPool_Done = InitStack ();
+  HostPool_Down = InitStack ();
+  HostPool_Off = InitStack ();
+}
+
+Stack *GetHostStack (int StackID) {
+  switch (StackID) {
+    case PCONTROL_HOST_IDLE:
+      return (HostPool_Idle);
+    case PCONTROL_HOST_DOWN:
+      return (HostPool_Down);
+    case PCONTROL_HOST_DONE:
+      return (HostPool_Done);
+    case PCONTROL_HOST_BUSY:
+      return (HostPool_Busy);
+    case PCONTROL_HOST_OFF:
+      return (HostPool_Off);
+    default:
+      fprintf (stderr, "error: unknown host stack : programming error\n");
+      exit (1);
+  }
+  fprintf (stderr, "error: unknown host stack : programming error\n");
+  exit (1);
+}
+
+Host *FindHostStack (IDtype HostID) {
+
+  Host *host;
+
+  host = FindHostPtr (HostID, PCONTROL_HOST_IDLE);
+  if (host != NULL) return (host);
+
+  host = FindHostPtr (HostID, PCONTROL_HOST_DOWN);
+  if (host != NULL) return (host);
+
+  host = FindHostPtr (HostID, PCONTROL_HOST_DONE);
+  if (host != NULL) return (host);
+
+  host = FindHostPtr (HostID, PCONTROL_HOST_BUSY);
+  if (host != NULL) return (host);
+
+  host = FindHostPtr (HostID, PCONTROL_HOST_OFF);
+  if (host != NULL) return (host);
+
+  return (NULL);
+}
+
+int FindNamedHostStack (char *name) {
+
+  int N;
+
+  N = FindNamedHost (name, PCONTROL_HOST_IDLE);
+  if (N > 0) return (PCONTROL_HOST_IDLE);
+
+  N = FindNamedHost (name, PCONTROL_HOST_DOWN);
+  if (N > 0) return (PCONTROL_HOST_DOWN);
+
+  N = FindNamedHost (name, PCONTROL_HOST_DONE);
+  if (N > 0) return (PCONTROL_HOST_DONE);
+
+  N = FindNamedHost (name, PCONTROL_HOST_BUSY);
+  if (N > 0) return (PCONTROL_HOST_BUSY);
+
+  N = FindNamedHost (name, PCONTROL_HOST_OFF);
+  if (N > 0) return (PCONTROL_HOST_OFF);
+
+  return (-1);
+}
+
+int PutHost (Host *host, int StackID, int where) {
+
+  int stat;
+  Stack *stack;
+
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (FALSE);
+
+  host[0].stack = StackID;
+  stat = PutStack (stack, where, host);
+  /* status (0, NULL); */
+  return (stat);
+}
+  
+Host *GetHost (int StackID, int where) {
+
+  Host *host;
+  Stack *stack;
+
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (NULL);
+
+  host = GetStack (stack, where);
+  return (host);
+}
+
+int FindHost (IDtype HostID, int StackID) {
+
+  int i;
+  Host *host;
+  Stack *stack;
+
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (-2);
+
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = (Host *) stack[0].object[i];
+    if (host[0].HostID == HostID) {
+      return (i);
+    }
+  }
+  return (-1);
+}
+
+Host *FindHostPtr (IDtype HostID, int StackID) {
+
+  int i;
+  Host *host;
+  Stack *stack;
+
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (NULL);
+
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = (Host *) stack[0].object[i];
+    if (host[0].HostID == HostID) {
+      return (host);
+    }
+  }
+  return (NULL);
+}
+
+Host *PullHost (IDtype HostID, int StackID) {
+  
+  int N;
+  Host *host;
+
+  N = FindHost (HostID, StackID);
+  if (N < 0) return (NULL);
+
+  host = GetHost (StackID, N);
+  if (host == NULL) {
+    fprintf (stderr, "programming error! host missing from stack\n");
+    exit (1);
+  }
+  return (host);
+}
+
+int FindNamedHost (char *name, int StackID) {
+
+  int i;
+  Host *host;
+  Stack *stack;
+
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (-2);
+
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = (Host *) stack[0].object[i];
+    if (!strcasecmp (host[0].hostname, name)) {
+      return (i);
+    }
+  }
+  return (-1);
+}
+
+IDtype AddHost (char *hostname) {
+
+  Host *host;
+
+  ALLOCATE (host, Host, 1);
+
+  host[0].hostname = strcreate (hostname);
+  host[0].stdin    = 0;
+  host[0].stdout   = 0;
+  host[0].stderr   = 0;
+  host[0].HostID   = NextHostID();
+
+  host[0].lasttry.tv_sec = 0;
+  host[0].lasttry.tv_usec = 0;
+  host[0].nexttry.tv_sec = 0;
+  host[0].nexttry.tv_usec = 0;
+
+  host[0].markoff  = FALSE;
+  host[0].job      = NULL;
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+  return (host[0].HostID);
+}
+
+void DelHost (Host *host) {
+  FREE (host[0].hostname);
+  FREE (host[0].job);
+  FREE (host);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobID.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobID.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobID.c	(revision 5922)
@@ -0,0 +1,41 @@
+# include "pcontrol.h"
+
+static IDtype CurrentJobID  = 0;
+static IDtype CurrentHostID = 0;
+
+/* for now, no persistence : we could use the date/time to seed the upper byte(s) if needed */
+void InitIDs () {
+  CurrentJobID = 0;
+  CurrentHostID = 0;
+}
+
+IDtype NextJobID () {
+
+  IDtype ID;
+
+  ID = CurrentJobID;
+  CurrentJobID ++;
+  return (ID);
+}
+
+IDtype NextHostID () {
+
+  IDtype ID;
+
+  ID = CurrentHostID;
+  CurrentHostID ++;
+  return (ID);
+}
+
+void PrintID (FILE *f, IDtype ID) {
+
+  unsigned short int word0, word1, word2, word3;
+
+  word0 = 0xffff & ID;
+  word1 = 0xffff & (ID >> 16);
+  word2 = 0xffff & (ID >> 32);
+  word3 = 0xffff & (ID >> 48);
+
+  fprintf (f, "%x.%x.%x.%x", word3, word2, word1, word0);
+}
+
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobOps.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobOps.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/JobOps.c	(revision 5922)
@@ -0,0 +1,231 @@
+# include "pcontrol.h"
+
+Stack *JobPool_Pending;
+Stack *JobPool_Busy;
+Stack *JobPool_Done;
+Stack *JobPool_Exit;
+Stack *JobPool_Crash;
+
+void InitJobStacks () {
+  JobPool_Pending = InitStack ();
+  JobPool_Busy = InitStack ();
+  JobPool_Done = InitStack ();
+  JobPool_Exit = InitStack ();
+  JobPool_Crash = InitStack ();
+}
+
+Stack *GetJobStack (int StackID) {
+  switch (StackID) {
+    case PCONTROL_JOB_PENDING:
+      return (JobPool_Pending);
+    case PCONTROL_JOB_BUSY:
+      return (JobPool_Busy);
+    case PCONTROL_JOB_DONE:
+      return (JobPool_Done);
+    case PCONTROL_JOB_EXIT:
+      return (JobPool_Exit);
+    case PCONTROL_JOB_CRASH:
+      return (JobPool_Crash);
+    default:
+      fprintf (stderr, "error: unknown job stack : programming error\n");
+      exit (1);
+  }
+  fprintf (stderr, "error: unknown job stack : programming error\n");
+  exit (1);
+}
+
+Stack *GetJobStackByName (char *name) {
+
+  if (!strcasecmp (name, "pending")) 
+    return (JobPool_Pending);
+  if (!strcasecmp (name, "busy")) 
+    return (JobPool_Busy);
+  if (!strcasecmp (name, "done")) 
+    return (JobPool_Done);
+  if (!strcasecmp (name, "exit")) 
+    return (JobPool_Exit);
+  if (!strcasecmp (name, "crash")) 
+    return (JobPool_Crash);
+  return (NULL);
+}
+
+Job *FindJobStack (IDtype JobID) {
+
+  Job *job;
+
+  job = FindJobPtr (JobID, PCONTROL_JOB_PENDING);
+  if (job != NULL) return (job);
+
+  job = FindJobPtr (JobID, PCONTROL_JOB_BUSY);
+  if (job != NULL) return (job);
+
+  job = FindJobPtr (JobID, PCONTROL_JOB_EXIT);
+  if (job != NULL) return (job);
+
+  job = FindJobPtr (JobID, PCONTROL_JOB_CRASH);
+  if (job != NULL) return (job);
+
+  job = FindJobPtr (JobID, PCONTROL_JOB_DONE);
+  if (job != NULL) return (job);
+
+  return (NULL);
+}
+
+/* add job to position in stack */
+int PutJob (Job *job, int StackID, int where) {
+
+  int stat;
+  Stack *stack;
+
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (FALSE);
+
+  /* be default, these are both the same - to override, set state after PutJob */
+  job[0].state = StackID;
+  job[0].stack = StackID;
+  stat = PutStack (stack, where, job);
+  /* status (0, NULL); */
+  return (stat);
+}
+  
+/* remove job from position in stack */
+Job *GetJob (int StackID, int where) {
+
+  Job *job;
+  Stack *stack;
+
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (NULL);
+
+  job = GetStack (stack, where);
+  return (job);
+}
+  
+/* return stack position of job */
+int FindJob (IDtype JobID, int StackID) {
+
+  int i;
+  Job *job;
+  Stack *stack;
+
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (-2);
+
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].JobID == JobID) {
+      return (i);
+    }
+  }
+  return (-1);
+}
+
+/* return pointer to job */
+Job *FindJobPtr (IDtype JobID, int StackID) {
+
+  int i;
+  Job *job;
+  Stack *stack;
+
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (NULL);
+
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].JobID == JobID) {
+      return (job);
+    }
+  }
+  return (NULL);
+}
+
+/* remove job from stack, return pointer */
+Job *PullJob (IDtype JobID, int StackID) {
+  
+  int N;
+  Job *job;
+
+  N = FindJob (JobID, StackID);
+  if (N < 0) return (NULL);
+
+  job = GetJob (StackID, N);
+  if (job == NULL) {
+    fprintf (stderr, "programming error! job missing from stack\n");
+    exit (1);
+  }
+  return (job);
+}
+
+IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv) {
+
+  Job *job;
+
+  ALLOCATE (job, Job, 1);
+
+  job[0].argc     = argc;
+  job[0].argv     = argv;
+  job[0].hostname = hostname;
+  job[0].mode     = mode;
+  job[0].host     = NULL;
+  job[0].JobID    = NextJobID();
+  job[0].Reset    = FALSE;
+
+  /* do this step on start? */
+  InitIOBuffer (&job[0].stdout, 0x1000);
+  InitIOBuffer (&job[0].stderr, 0x1000);
+
+  PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+  fprintf (stderr, "added new job\n");
+  return (job[0].JobID);
+}
+
+void DelJob (Job *job) {
+
+  int i;
+
+  FREE (job[0].hostname);
+  for (i = 0; i < job[0].argc; i++) {
+    FREE (job[0].argv[i]);
+  }
+  FREE (job[0].argv);
+
+  FreeIOBuffer (&job[0].stdout);
+  FreeIOBuffer (&job[0].stderr);
+
+  FREE (job);
+}
+
+/* unlink job and host, pull host from its stack */
+Host *UnlinkJobAndHost (Job *job) {
+
+  Host *host;
+
+  host = (Host *) job[0].host;
+  if (host == NULL) {
+    fprintf (stderr, "programming error: job has no host\n");
+    exit (2);
+  }
+
+  /* unlink host & job */
+  job[0].host = NULL;
+  host[0].job = NULL;
+  
+  /* remove host from correct stack */
+  if (PullHost (host[0].HostID, host[0].stack) == NULL) {
+    fprintf (stderr, "programming error: host is not found in current stack\n");
+    exit (2);
+  }
+  return (host);
+}
+
+void LinkJobAndHost (Job *job, Host *host) {
+
+  job[0].host = (struct Host *) host;
+  host[0].job = (struct Job *) job;
+
+  /* remove job from correct stack */
+  if (PullJob (job[0].JobID, job[0].stack) == NULL) {
+    fprintf (stderr, "programming error: job is not found in current stack\n");
+    exit (2);
+  }
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/KillJob.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/KillJob.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/KillJob.c	(revision 5922)
@@ -0,0 +1,74 @@
+# include "pcontrol.h"
+
+int KillJob (Job *job) {
+  
+  Host *host;
+  IOBuffer buffer;
+  int status;
+  char *p;
+
+  /** must have a valid host : if not? **/
+  host = (Host *) job[0].host;
+
+  InitIOBuffer (&buffer, 0x100);
+
+  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      HarvestHost (host[0].pid);
+      UnlinkJobAndHost (job);
+      PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      job[0].state = PCONTROL_JOB_HUNG;
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received (KillJob)\n");  
+      break;
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  /** host is up, need to parse message **/
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "missing STATUS in pclient message : programming error\n");
+    exit (1);
+  }
+  fprintf (stderr, "client message: %s\n", buffer.buffer);
+  sscanf (p, "%*s %d", &status);
+  FreeIOBuffer (&buffer);
+  fprintf (stderr, "client status: %d\n", status);
+
+  switch (status) {
+    case -1:
+      fprintf (stderr, "programming error (syntax error to pclient)\n");
+      return (FALSE);
+      break;
+    case 0:
+      fprintf (stderr, "failure to kill child process\n");
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      return (FALSE);
+    case 1:
+      PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+      job[0].state = PCONTROL_JOB_CRASH;
+      /* this overrides the default assigned by PutJob */
+      return (TRUE);
+    case 2:
+      fprintf (stderr, "programming error (client has no job)\n");
+      return (FALSE);
+  }
+
+  fprintf (stderr, "programming error : should not reach here (CheckJob)\n");
+  exit (1);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/Makefile
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/Makefile	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/Makefile	(revision 5922)
@@ -0,0 +1,79 @@
+include ../../../Configure
+
+HOME    =       $(ROOT)/src/opihi
+BIN     =       $(HOME)/bin
+LIB     =       $(HOME)/lib
+INC     =       $(HOME)/include
+SDIR    =       $(HOME)/pcontrol
+#
+DESTBIN =       $(LBIN)
+DESTLIB =       $(LLIB)
+DESTINC =       $(LINC)
+DESTMAN =       $(LMAN)
+DESTHLP =       $(LHLP)/pcontrol
+#  
+INCS    =       -I$(INC) -I$(LINC) -I$(XINC)
+LFLAGS  =       -L$(LLIB) -L$(LIB)
+LIBS1   =       -lsocket -lnsl -lreadline -ltermcap -lkapa -lFITS -lohana -lX11 -lm
+LIBS2   =       -lbasiccmd -lshell -ldata
+LIBS    =       $(LIBS2) $(LIBS1) 
+CFLAGS  =       $(INCS) -DHELPDIR_DEFAULT=$(DESTHLP)
+CCFLAGS =       $(LIBS) 
+
+# mana user commands and support functions ########################
+
+funcs = \
+$(SDIR)/init.$(ARCH).o \
+$(SDIR)/pclient.$(ARCH).o \
+$(SDIR)/pcontrol.$(ARCH).o \
+$(SDIR)/rconnect.$(ARCH).o \
+$(SDIR)/CheckBusyJob.$(ARCH).o \
+$(SDIR)/CheckDoneHost.$(ARCH).o \
+$(SDIR)/CheckDoneJob.$(ARCH).o \
+$(SDIR)/CheckHost.$(ARCH).o \
+$(SDIR)/CheckIdleHost.$(ARCH).o \
+$(SDIR)/CheckSystem.$(ARCH).o \
+$(SDIR)/GetJobOutput.$(ARCH).o \
+$(SDIR)/HostOps.$(ARCH).o \
+$(SDIR)/JobID.$(ARCH).o \
+$(SDIR)/JobOps.$(ARCH).o \
+$(SDIR)/QueueOps.$(ARCH).o \
+$(SDIR)/ResetJob.$(ARCH).o \
+$(SDIR)/StartHost.$(ARCH).o \
+$(SDIR)/StopHosts.$(ARCH).o \
+$(SDIR)/KillJob.$(ARCH).o \
+$(SDIR)/StartJob.$(ARCH).o
+
+cmds = \
+$(SDIR)/check.$(ARCH).o \
+$(SDIR)/delete.$(ARCH).o \
+$(SDIR)/host.$(ARCH).o \
+$(SDIR)/job.$(ARCH).o \
+$(SDIR)/jobstack.$(ARCH).o \
+$(SDIR)/kill.$(ARCH).o \
+$(SDIR)/pulse.$(ARCH).o \
+$(SDIR)/stop.$(ARCH).o \
+$(SDIR)/run.$(ARCH).o \
+$(SDIR)/status.$(ARCH).o \
+$(SDIR)/stdout.$(ARCH).o \
+$(SDIR)/version.$(ARCH).o \
+$(SDIR)/verbose.$(ARCH).o
+
+pcontrol = $(cmds) $(funcs)
+
+libs = \
+$(DESTLIB)/libbasiccmd.a \
+$(DESTLIB)/libshell.a \
+$(DESTLIB)/libdata.a
+
+pcontrol: $(BIN)/pcontrol.$(ARCH)
+
+$(BIN)/pcontrol.$(ARCH) : $(pcontrol) $(libs)
+
+install: $(DESTBIN)/pcontrol help
+
+help: clean-help cmd.basic.help pcontrol.help
+
+.PHONY: pcontrol
+
+include ../Makefile.Common
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/QueueOps.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/QueueOps.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/QueueOps.c	(revision 5922)
@@ -0,0 +1,63 @@
+# include "pcontrol.h"
+
+/* get object from point in stack (negative == distance from end) */
+void *GetStack (Stack *stack, int where) {
+
+  int i;
+  void *object;
+  
+  /* STACK_TOP == 0, STACK_BOTTOM == -1 */
+  /* this code correctly handles the negative 'where' and Nobject == 0 */
+  if (where < 0) where += stack[0].Nobject;
+  if (where < 0) return (NULL);
+  if (where >= stack[0].Nobject) return (NULL);
+
+  object = stack[0].object[where];
+  stack[0].Nobject --;
+  for (i = where; i < stack[0].Nobject; i++) {
+    stack[0].object[i] = stack[0].object[i+1];
+  }
+  return (object);
+}
+
+/* push object on top of stack */
+int PutStack (Stack *stack, int where, void *object) {
+
+  int i;
+
+  /* STACK_TOP == 0, STACK_BOTTOM == -1 */
+  /* this code correctly handles the negative 'where' and Nobject == 0 */
+  if (where < 0) where += stack[0].Nobject + 1;
+  if (where < 0) return (FALSE);
+  if (where > stack[0].Nobject) return (FALSE);
+
+  /* extend stack as needed */
+  if (stack[0].Nobject >= stack[0].NOBJECT) {
+    stack[0].NOBJECT += 100;
+    REALLOCATE (stack[0].object, void *, stack[0].NOBJECT);
+  }
+
+  for (i = stack[0].Nobject; i > where; i--) {
+    stack[0].object[i] = stack[0].object[i-1];
+  }
+  stack[0].object[where] = object;
+  stack[0].Nobject ++;
+  return (TRUE);
+}
+
+/* allocate stack, setup with default values, allocate data */
+Stack *InitStack () {
+
+  Stack *stack;
+
+  ALLOCATE (stack, Stack, 1);
+
+  stack[0].Nobject = 0;
+  stack[0].NOBJECT = 50;
+  ALLOCATE (stack[0].object, void *, stack[0].NOBJECT);
+  return (stack);
+}
+
+/* these stacks are not super efficient, and should probably be replaced with linked lists, 
+   but I find these easier to get my brain around
+*/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/ResetJob.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 5922)
@@ -0,0 +1,54 @@
+# include "pcontrol.h"
+
+int ResetJob (Job *job) {
+  
+  int       status;
+  IOBuffer  buffer;
+  Host     *host;
+
+  /** must have a valid host : if not, move to pending? **/
+  host = (Host *) job[0].host;
+
+  InitIOBuffer (&buffer, 0x100);
+  
+  /* we have tried to reset the job; may not get status */
+  job[0].Reset = TRUE;
+
+  status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST? ***/
+      fprintf (stderr, "host %s is down\n", host[0].hostname);
+      HarvestHost (host[0].pid);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      /*** should we consider a HUNG host DOWN? ***/
+      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received (ResetJob)\n");  
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  fprintf (stderr, "programming error in GetJobOutput (should not reach here)\n");
+  exit (1);
+}
+
+/* if machine is down, return FALSE
+   this will place job back in BUSY state,
+   next check of job will catch state and
+   put machine down correctly
+
+*/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartHost.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartHost.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartHost.c	(revision 5922)
@@ -0,0 +1,45 @@
+# include "pcontrol.h"
+# define RETRY_BASE 1.0
+
+int StartHost (Host *host) {
+
+  int pid;
+  int stdio[3];
+  char command[64], shell[64];
+  struct timeval now;
+  float delta;
+
+  /* perhaps change the name of these config variables... */
+  if (VarConfig ("COMMAND", "%s", command) == NULL) strcpy (command, "ssh");
+  if (VarConfig ("SHELL", "%s", shell)     == NULL) strcpy (shell, "pclient");
+
+  pid = rconnect (command, host[0].hostname, shell, stdio);
+  if (!pid) {     
+    /** failure to start: extend retry period **/
+    if (VerboseMode()) fprintf (stderr, "failure to start %s\n", host[0].hostname);
+    gettimeofday (&now, (void *) NULL);
+    if (ZTIME(host[0].nexttry) || ZTIME(host[0].lasttry)) {
+      /* reset retry period if either is zero */
+      delta = RETRY_BASE;
+    } else {
+      delta = 2*DTIME (host[0].nexttry, host[0].lasttry);
+    }
+    host[0].nexttry.tv_sec  = now.tv_sec  + delta;
+    host[0].nexttry.tv_usec = now.tv_usec;
+    host[0].lasttry.tv_sec  = now.tv_sec;
+    host[0].lasttry.tv_usec = now.tv_usec;
+    PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+    return (FALSE);
+  }
+  host[0].nexttry.tv_sec  = 0;
+  host[0].nexttry.tv_usec = 0;
+  host[0].lasttry.tv_sec  = 0;
+  host[0].lasttry.tv_usec = 0;
+
+  host[0].stdin  = stdio[0];
+  host[0].stdout = stdio[1];
+  host[0].stderr = stdio[2];
+  host[0].pid    = pid;
+  PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartJob.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartJob.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StartJob.c	(revision 5922)
@@ -0,0 +1,97 @@
+# include "pcontrol.h"
+
+int StartJob (Job *job) {
+
+  int  i, Nline, status;
+  char *line, *p;
+  Host *host;
+  IOBuffer buffer;
+
+  InitIOBuffer (&buffer, 0x100);
+
+  /* job must have assigned host */
+  host = (Host *) job[0].host;
+  if (host == NULL) {
+    fprintf (stderr, "programming error: no assigned host\n");
+    exit (1);
+  }
+
+  /* construct command line : job arg0 arg1 ... argN\n */
+  Nline = 10 + job[0].argc;
+  for (i = 0; i < job[0].argc; i++) {
+    Nline += strlen (job[0].argv[i]);
+  }
+  ALLOCATE (line, char, Nline);
+  bzero (line, Nline);
+  strcpy (line, "job");
+  for (i = 0; i < job[0].argc; i++) {
+    strcat (line, " ");
+    strcat (line, job[0].argv[i]);
+  }
+
+  status = PclientCommand (host, line, PCLIENT_PROMPT, &buffer);
+  free (line);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      if (VerboseMode()) fprintf (stderr, "host %s is down\n", host[0].hostname);
+      goto failure;
+
+    case PCLIENT_HUNG:
+      if (VerboseMode()) fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      goto failure;
+
+    case PCLIENT_GOOD:
+      if (VerboseMode()) fprintf (stderr, "message received (StartJob)\n");  
+      break;
+
+    default:
+      if (VerboseMode()) fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  /* check on result of pclient command */
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "programming error: missing STATUS in pclient message\n");
+    exit (1);
+  }
+  sscanf (p, "%*s %d", &status);
+  switch (status) {
+    case -1:
+      if (VerboseMode()) fprintf (stderr, "error in pclient child\n");
+      goto failure;
+
+    case -2:
+      fprintf (stderr, "programming error: syntax error in pclient command\n");
+      exit (1);
+
+    case -3:
+      fprintf (stderr, "programming error: existing child on pclient\n");
+      exit (1);
+
+    default:
+      job[0].pid = status;
+      PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      /** need to start timers here **/
+      return (TRUE);
+  }
+  /* we should never reach here */
+  fprintf (stderr, "programming error: should not reach here (StartJob)\n");
+  exit (1);
+
+failure:
+  /* unlink host & job */
+  job[0].host = NULL;
+  host[0].job = NULL;
+  HarvestHost (host[0].pid);
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+  PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+  FreeIOBuffer (&buffer);
+  return (FALSE);
+}
+
+/** note : host and job popped off stacks : can't use UnlinkJobAndHost **/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StopHosts.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 5922)
@@ -0,0 +1,123 @@
+# include "pcontrol.h"
+
+int DownHosts () {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
+    StopHost (host);
+    DownHost (host);
+  }
+
+  stack = GetHostStack (PCONTROL_HOST_BUSY);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
+    StopHost (host);
+    DownHost (host);
+  }
+  return (TRUE);
+}
+
+int StopHost (Host *host) {
+
+  int       status;
+  IOBuffer  buffer;
+
+  InitIOBuffer (&buffer, 0x100);
+  status = PclientCommand (host, "exit", "Goodbye", &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      break;
+
+    case PCLIENT_HUNG:
+      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      break;
+
+    case PCLIENT_GOOD:
+      break;
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+  HarvestHost (host[0].pid);
+  return (TRUE);
+}
+
+void DownHost (Host *host) {
+  CLOSE (host[0].stdin);
+  CLOSE (host[0].stdout);
+  CLOSE (host[0].stderr);
+  host[0].job = NULL;
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+}
+
+void OffHost (Host *host) {
+  CLOSE (host[0].stdin);
+  CLOSE (host[0].stdout);
+  CLOSE (host[0].stderr);
+  host[0].job = NULL;
+  PutHost (host, PCONTROL_HOST_OFF, STACK_BOTTOM);
+}
+
+int HarvestHost (int pid) {
+  
+  int       result;
+  int       waitstatus;
+
+  /* I probably should loop a few time with max timeout larger than 10ms... */
+  usleep (10000);
+  result = waitpid (pid, &waitstatus, WNOHANG);
+  switch (result) {
+    case -1:  /* error with waitpid */
+      switch (errno) {
+	case ECHILD:
+	  fprintf (stderr, "unknown PID, not a child proc\n");
+	  fprintf (stderr, "did process already exit?  programming error?\n");
+	  break;
+	case EINVAL:
+	  fprintf (stderr, "error EINVAL (waitpid): programming error\n");
+	  exit (1);
+	case EINTR:
+	  fprintf (stderr, "error EINTR (waitpid): programming error\n");
+	  exit (1);
+	default:
+	  fprintf (stderr, "unknown error for waitpid (%d): programming error\n", errno);
+	  exit (1);
+      }
+      break;
+      
+    case 0:
+      fprintf (stderr, "child did not exit??");
+      /** put back in IDLE state? **/
+      break;
+
+    default:
+      if (result != pid) {
+	fprintf (stderr, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
+	exit (1);
+      }
+      
+      if (WIFEXITED(waitstatus)) {
+	fprintf (stderr, "child exited with status %d\n", WEXITSTATUS(waitstatus));
+      }
+      if (WIFSIGNALED(waitstatus)) {
+	fprintf (stderr, "child crashed with status %d\n", WTERMSIG(waitstatus));
+      }
+      if (WIFSTOPPED(waitstatus)) {
+	fprintf (stderr, "waitpid returns 'stopped': programming error\n");
+	exit (1);
+      }
+  }
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/check.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/check.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/check.c	(revision 5922)
@@ -0,0 +1,45 @@
+# include "pcontrol.h"
+
+char jobstate[6][32] = {"PENDING", "BUSY", "EXIT", "CRASH", "HUNG", "DONE"};
+char hoststate[5][32] = {"IDLE", "BUSY", "DOWN", "DONE", "OFF"};
+
+int check (int argc, char **argv) {
+
+  Job *job;
+  Host *host;
+  int JobID, HostID;
+
+  if (argc != 3) {
+    fprintf (stdout, "USAGE: check job (JobID)\n");
+    fprintf (stdout, "USAGE: check host (HostID)\n");
+    return (FALSE);
+  }
+
+  if (!strcasecmp (argv[1], "JOB")) {
+    JobID = atoi (argv[2]);
+    job = FindJobStack (JobID);
+    if (job == NULL) {
+      fprintf (stdout, "job not found\n");
+      return (FALSE);
+    }
+    fprintf (stdout, "STATUS %s\n", jobstate[job[0].stack]);
+    fprintf (stdout, "EXITST %d\n", job[0].exit_status);
+    fprintf (stdout, "STDOUT %d\n", job[0].stdout_size);
+    fprintf (stdout, "STDERR %d\n", job[0].stderr_size);
+    return (TRUE);
+  }
+
+  if (!strcasecmp (argv[1], "HOST")) {
+    HostID = atoi (argv[2]);
+    host = FindHostStack (HostID);
+    if (host == NULL) {
+      fprintf (stdout, "host not found\n");
+      return (FALSE);
+    }
+    fprintf (stdout, "host %s\n", hoststate[host[0].stack]);
+    return (TRUE);
+  }
+
+  fprintf (stdout, "unknown item to check\n");
+  return (FALSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/delete.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/delete.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/delete.c	(revision 5922)
@@ -0,0 +1,45 @@
+# include "pcontrol.h"
+
+int delete (int argc, char **argv) {
+
+  Job *job;
+  int JobID;
+
+  if (argc < 2) {
+    fprintf (stderr, "USAGE: delete (JobID)\n");
+    return (FALSE);
+  }
+  JobID = atoi (argv[1]);
+  /* use a string interp to convert JobIDs to ints ? */
+
+  job = PullJob (JobID, PCONTROL_JOB_PENDING);
+  if (job != NULL) goto found;
+
+  job = PullJob (JobID, PCONTROL_JOB_CRASH);
+  if (job != NULL) goto found;
+
+  job = PullJob (JobID, PCONTROL_JOB_EXIT);
+  if (job != NULL) goto found;
+
+  fprintf (stderr, "job %s not PENDING, CRASH, EXIT\n", argv[1]);
+  return (FALSE);
+  
+found:
+  {
+    int j;
+    fprintf (stderr, "deleting job  %s  %d  ", job[0].hostname, job[0].argc);
+    for (j = 0; j < job[0].argc; j++) {
+      fprintf (stderr, "%s ", job[0].argv[j]);
+    }
+    PrintID (stderr, job[0].JobID);
+    fprintf (stderr, "\n");
+  }  
+  DelJob (job);
+
+  return (TRUE);
+}
+
+/**** at the moment, this function requires the job to be in the correct state
+      to be deleted.  This should be changed to kill, then delete job if it is 
+      not in the correct state 
+****/
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/help/host
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/help/host	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/help/host	(revision 5922)
@@ -0,0 +1,1 @@
+empty
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/host.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/host.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/host.c	(revision 5922)
@@ -0,0 +1,103 @@
+# include "pcontrol.h"
+
+int host (int argc, char **argv) {
+
+  int N, Ns;
+  IDtype HostID;
+  Host *host;
+
+  if (argc != 3) {
+    fprintf (stdout, "USAGE: host (command) (hostname)\n");
+    return (FALSE);
+  }
+
+  if (!strcasecmp (argv[1], "ADD")) {
+    HostID = AddHost (argv[2]);
+    fprintf (stdout, "HostID: %d\n", (int) HostID);
+    return (TRUE);
+  }
+  if (!strcasecmp (argv[1], "ON")) {
+    N = FindNamedHost (argv[2], PCONTROL_HOST_OFF);
+    if (N < 0) {
+      fprintf (stdout, "host %s is not OFF\n", argv[2]);
+      return (FALSE);
+    }
+    host = GetHost (PCONTROL_HOST_OFF, N);
+    DownHost (host);
+    return (TRUE);
+  }
+  if (!strcasecmp (argv[1], "RETRY")) {
+    N = FindNamedHost (argv[2], PCONTROL_HOST_DOWN);
+    if (N < 0) {
+      fprintf (stdout, "host %s is not DOWN\n", argv[2]);
+      return (FALSE);
+    }
+    host = GetHost (PCONTROL_HOST_DOWN, N);
+    host[0].nexttry.tv_sec  = 0;
+    host[0].nexttry.tv_usec = 0;
+    host[0].lasttry.tv_sec  = 0;
+    host[0].lasttry.tv_usec = 0;
+    StartHost (host);
+    return (TRUE);
+  }
+  if (!strcasecmp (argv[1], "CHECK")) {
+    Ns = FindNamedHostStack (argv[2]);
+    switch (Ns) {
+      case PCONTROL_HOST_IDLE:
+      case PCONTROL_HOST_BUSY:
+      case PCONTROL_HOST_DONE:
+	N = FindNamedHost (argv[2], Ns);
+	host = GetHost (Ns, N);
+	CheckHost (host);
+	return (TRUE);
+      case PCONTROL_HOST_DOWN:
+	fprintf (stdout, "host %s is DOWN\n", argv[2]);
+	return (TRUE);
+      case PCONTROL_HOST_OFF:
+	fprintf (stdout, "host %s is OFF\n", argv[2]);
+	return (TRUE);
+      default:
+	fprintf (stdout, "host %s not found\n", argv[2]);
+	return (FALSE);
+    }
+    return (FALSE);
+  }
+  if (!strcasecmp (argv[1], "OFF")) {
+    N = FindNamedHost (argv[2], PCONTROL_HOST_IDLE);
+    if (N >= 0) {
+      host = GetHost (PCONTROL_HOST_IDLE, N);
+      StopHost (host);
+      OffHost (host);
+      return (TRUE);
+    }
+    N = FindNamedHost (argv[2], PCONTROL_HOST_DOWN);
+    if (N >= 0) {
+      host = GetHost (PCONTROL_HOST_DOWN, N);
+      OffHost (host);
+      return (TRUE);
+    }
+    N = FindNamedHost (argv[2], PCONTROL_HOST_BUSY);
+    if (N >= 0) {
+      host = GetHost (PCONTROL_HOST_BUSY, N);
+      host[0].markoff  = TRUE;
+      PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
+      return (TRUE);
+    }
+    fprintf (stdout, "host %s is not BUSY, IDLE, or DOWN\n", argv[2]);
+    return (FALSE);
+  }
+  if (!strcasecmp (argv[1], "DELETE")) {
+    N = FindNamedHost (argv[2], PCONTROL_HOST_OFF);
+    if (N < 0) {
+      fprintf (stdout, "host %s is not OFF\n", argv[2]);
+      return (FALSE);
+    }
+    host = GetHost (PCONTROL_HOST_OFF, N);
+    DelHost (host);
+    return (TRUE);
+  }
+  
+  fprintf (stderr, "unknown host command %s\n", argv[1]);
+  fprintf (stderr, "valid options: xxx\n");
+  return (FALSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/init.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/init.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/init.c	(revision 5922)
@@ -0,0 +1,47 @@
+# include "pcontrol.h"
+
+int check       PROTO((int, char **));
+int delete      PROTO((int, char **));
+int host        PROTO((int, char **));
+int job	        PROTO((int, char **));
+int jobstack    PROTO((int, char **));
+int kill_pc     PROTO((int, char **));
+int pulse       PROTO((int, char **));
+int status      PROTO((int, char **));
+int run         PROTO((int, char **));
+int stop        PROTO((int, char **));
+int stderr_pc   PROTO((int, char **));
+int stdout_pc   PROTO((int, char **));
+int verbose     PROTO((int, char **));
+int version         PROTO((int, char **));
+
+static Command cmds[] = {  
+  {"check",     check,     "get job or host status"},
+  {"delete",    delete,    "delete job"},
+  {"host",      host,      "add / delete / modify host"},
+  {"job",       job,       "add job"},
+  {"jobstack",  jobstack,  "list jobs for a single stack"},
+  {"kill",      kill_pc,   "kill job"},
+  {"pulse",     pulse,     "set system pulse"},
+  {"status",    status,    "get system status"},
+  {"stop",      stop,      "stop controller processing"},
+  {"run",       run,       "start controller processing"},
+  {"stderr",    stderr_pc, "get stderr buffer for job"},
+  {"stdout",    stdout_pc, "get stdout buffer for job"},
+  {"verbose",   verbose,   "set the verbose mode for job"},
+  {"version",     version,      "show version information"},
+}; 
+
+void InitPcontrol () {
+  
+  int i;
+
+  for (i = 0; i < sizeof (cmds) / sizeof (Command); i++) {
+    AddCommand (&cmds[i]);
+  }
+
+  /* temporary : put this elsewhere? */
+  InitJobStacks ();
+  InitHostStacks ();
+
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/job.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/job.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/job.c	(revision 5922)
@@ -0,0 +1,52 @@
+# include "pcontrol.h"
+
+int job (int argc, char **argv) {
+
+  char *Host, **targv;
+  int i, N, Mode, targc, Timeout;
+  IDtype JobID;
+
+  Host = NULL;
+  Mode = PCONTROL_JOB_ANYHOST;
+  if ((N = get_argument (argc, argv, "-host"))) {
+    remove_argument (N, &argc, argv);
+    Host = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+    Mode = PCONTROL_JOB_WANTHOST;
+  }
+  if ((N = get_argument (argc, argv, "+host"))) {
+    if (Mode == PCONTROL_JOB_WANTHOST) {
+      fprintf (stderr, "ERROR: -host and +host are incompatible\n");
+      FREE (Host);
+      return (FALSE);
+    }
+    remove_argument (N, &argc, argv);
+    Host = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+    Mode = PCONTROL_JOB_NEEDHOST;
+  }
+  if (Host == NULL) Host = strcreate ("anyhost");
+ 
+  Timeout = 100;
+  if ((N = get_argument (argc, argv, "-timeout"))) {
+    remove_argument (N, &argc, argv);
+    Timeout = atoi (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  if (argc < 2) {
+    fprintf (stderr, "USAGE: job [options] (arg0) (arg1) ... (argN)\n");
+    FREE (Host);
+    return (FALSE);
+  }
+  
+  targc = argc - 1;
+  ALLOCATE (targv, char *, targc);
+  for (i = 1; i < argc; i++) {
+    targv[i-1] = strcreate (argv[i]);
+  }
+
+  JobID = AddJob (Host, Mode, Timeout, targc, targv);
+  fprintf (stdout, "JobID: %d\n", (int) JobID);
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/jobstack.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/jobstack.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/jobstack.c	(revision 5922)
@@ -0,0 +1,31 @@
+# include "pcontrol.h"
+
+int jobstack (int argc, char **argv) {
+
+  int i;
+  Stack *stack;
+  Job *job;
+
+  if (argc != 2) {
+    fprintf (stdout, "USAGE: jobstack (jobstack)\n");
+    fprintf (stdout, "       (jobstack) : pending, busy, exit, crash, hung, done\n");
+    return (FALSE);
+  }
+
+  /* select jobstack */
+  stack = GetJobStackByName (argv[1]);
+  if (stack == NULL) {
+    fprintf (stderr, "jobstack not found\n");
+    return (FALSE);
+  }
+
+  /* print list */
+  fprintf (stdout, "Njobs: %d\n", stack[0].Nobject);
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = stack[0].object[i];
+    /* PrintID (stdout, job[0].JobID); */
+    fprintf (stdout, "%lld ", job[0].JobID);
+    fprintf (stdout, "%s  %s\n", job[0].argv[0], job[0].hostname);
+  }
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/kill.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/kill.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/kill.c	(revision 5922)
@@ -0,0 +1,26 @@
+# include "pcontrol.h"
+
+int kill_pc (int argc, char **argv) {
+
+  Job *job;
+  int JobID;
+
+  if (argc < 2) {
+    fprintf (stderr, "USAGE: kill (JobID)\n");
+    return (FALSE);
+  }
+  JobID = atoi (argv[1]);
+
+  /* XXX this function should only fail if a process is hung */
+  job = PullJob (JobID, PCONTROL_JOB_BUSY);
+  if (job == NULL) {
+    fprintf (stderr, "job %s not BUSY\n", argv[1]);
+    /* make output message more readable by scheduler */
+    return (FALSE);
+  }
+
+  /* XXX - check on success / failure of kill */
+  KillJob (job);
+
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pclient.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pclient.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pclient.c	(revision 5922)
@@ -0,0 +1,38 @@
+# include "pcontrol.h"
+# define PCLIENT_TIMEOUT 500
+
+int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer) {
+
+  int i;
+  int status;
+  char *line;
+  struct timespec request, remain;
+
+  /* avoid blocking on read, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
+  ReadtoIOBuffer (buffer, host[0].stdout);
+  FlushIOBuffer (buffer);
+
+  /* send command to client (adding on \n) */
+  status = write_fmt (host[0].stdin, "%s\n", command);
+
+  /* is pipe still open? */
+  if ((status == -1) && (errno == EPIPE)) return (PCLIENT_DOWN);
+  
+  /* watch for response - wait up to 1 second */
+  line = NULL;
+  status = -1;
+  for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) {
+    status = ReadtoIOBuffer (buffer, host[0].stdout);
+    line = memstr (buffer[0].buffer, response, buffer[0].Nbuffer);
+    if (status == -1) nanosleep (&request, &remain);
+  }
+  if (status ==  0) return (PCLIENT_DOWN);
+  if (status == -1) return (PCLIENT_HUNG);
+  /* fprintf (stderr, "buffer.buffer: %s\n", buffer[0].buffer); */
+  return (PCLIENT_GOOD);
+}
+
+/* memstr returns a view, not an allocated string : don't free */
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pcontrol.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 5922)
@@ -0,0 +1,53 @@
+# include "pcontrol.h"
+
+# define opihi_name "PCONTROL"
+# define opihi_prompt "pcontrol: "
+# define opihi_description "pcontrol client shell\n"
+# define opihi_history ".pcontrol"
+# define opihi_rcfile ".pcontrolrc"
+
+/* program-dependent initialization */
+void program_init (int *argc, char **argv) {
+  
+  auto_break = TRUE;
+
+  /* load the commands used by this implementation */
+  InitBasic ();
+  InitPcontrol ();
+
+  rl_readline_name = opihi_name;
+  rl_attempted_completion_function = command_completer;
+  rl_event_hook = CheckSystem;
+  rl_set_keyboard_input_timeout (1000); 
+
+  set_str_variable ("HISTORY", opihi_history);
+  set_str_variable ("PROMPT", opihi_prompt);
+  set_str_variable ("RCFILE", opihi_rcfile);
+
+# ifdef HELPDIR_DEFAULT
+  set_str_variable ("HELPDIR", MACRO_NAME(HELPDIR_DEFAULT));
+# endif
+
+  /* ignore the history file.  to change this, see, eg, mana.c */
+  signal (SIGPIPE, gotsignal);
+  signal (SIGTSTP, gotsignal);
+  signal (SIGTTIN, gotsignal);
+  return;
+}
+
+/* standard welcome message */
+void welcome () {
+  fprintf (stderr, "\n");
+  fprintf (stderr, "Welcome to %s - %s\n\n", opihi_name, opihi_description);
+}
+
+/* add program-dependent exit functions here */
+void cleanup () {
+  DownHosts ();
+  return;
+}
+
+void gotsignal (int signum) {
+  fprintf (stderr, "got signal : %d\n", signum);
+  return;
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pulse.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pulse.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/pulse.c	(revision 5922)
@@ -0,0 +1,16 @@
+# include "psched.h"
+
+int pulse (int argc, char **argv) {
+
+  int Nusec;
+
+  if (argc != 2) {
+    fprintf (stderr, "USAGE: pulse (microseconds)\n");
+    return (FALSE);
+  }
+
+  Nusec = atoi (argv[1]);
+  rl_set_keyboard_input_timeout (Nusec); 
+
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/rconnect.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/rconnect.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/rconnect.c	(revision 5922)
@@ -0,0 +1,143 @@
+# include "pcontrol.h"
+
+/* connection can take a while, allow up to 2 sec */
+# define CONNECT_TIMEOUT 200
+
+/* connect to host, start the shell: ssh hostname pclient -> command hostname shell
+   stdio is an array of file descriptors (stdio[3]) 
+*/
+
+int rconnect (char *command, char *hostname, char *shell, int *stdio) {
+
+  int i, stdin_fd[2], stdout_fd[2], stderr_fd[2], status;
+  int result, waitstatus;
+  pid_t pid;
+  char *p;
+  char **argv;
+  IOBuffer buffer;
+  struct timespec request, remain;
+
+  bzero (stdin_fd,  2*sizeof(int));
+  bzero (stdout_fd, 2*sizeof(int));
+  bzero (stderr_fd, 2*sizeof(int));
+
+  if (pipe (stdin_fd)  < 0) goto pipe_error;
+  if (pipe (stdout_fd) < 0) goto pipe_error;
+  if (pipe (stderr_fd) < 0) goto pipe_error;
+
+  ALLOCATE (argv, char *, 4);
+  argv[0] = command;
+  argv[1] = hostname;
+  argv[2] = shell;
+  argv[3] = 0;
+
+  pid = fork ();
+  if (!pid) { /* must be child process */
+    if (VerboseMode()) fprintf (stderr, "starting remote connection to %s...", hostname);
+
+    /* close the other ends of the pipes */
+    close (stdin_fd[1]);
+    close (stdout_fd[0]);
+    close (stderr_fd[0]);
+
+    /* tie our ends of the pipes to stdin, stdout, stderr */
+    dup2 (stdin_fd[0],  STDIN_FILENO);
+    dup2 (stdout_fd[1], STDOUT_FILENO);
+    dup2 (stderr_fd[1], STDERR_FILENO);
+
+    /* set all three unblocking */
+    setvbuf (stdin,  (char *) NULL, _IONBF, BUFSIZ);
+    setvbuf (stdout, (char *) NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, (char *) NULL, _IONBF, BUFSIZ);
+
+    status = execvp (argv[0], argv); 
+    fprintf (stderr, "error starting remote shell process\n");
+    exit (1);
+  }
+  free (argv);
+
+  /* close the other ends of the pipes */
+  close (stdin_fd[0]);  stdin_fd[0]  = 0;
+  close (stdout_fd[1]); stdout_fd[1] = 0;
+  close (stderr_fd[1]); stderr_fd[1] = 0;
+
+  /* make the pipes non-blocking */
+  fcntl (stdin_fd[1],  F_SETFL, O_NONBLOCK);
+  fcntl (stdout_fd[0], F_SETFL, O_NONBLOCK);
+  fcntl (stderr_fd[0], F_SETFL, O_NONBLOCK);
+
+  /* perform handshake with pclient to verify alive & running */
+  /** this handshake is similar to PclientCommand, but has important differences **/
+  InitIOBuffer (&buffer, 0x100);
+
+  /* send handshake command */
+  status = write_fmt (stdin_fd[1], "echo CONNECTED\n");
+  if ((status == -1) && (errno == EPIPE)) goto connect_error;
+
+  /* try to get evidence connection is alive - wait upto a few seconds */
+  p = NULL;
+  status = -1;
+  for (i = 0; (i < CONNECT_TIMEOUT) && (status != 0) && (p == NULL); i++) {
+    status = ReadtoIOBuffer (&buffer, stdout_fd[0]);
+    p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer);
+    usleep (10000);
+  }
+  if (status == 0) goto connect_error;
+  if (status == -1) goto connect_error;
+  if (VerboseMode()) fprintf (stderr, "%d cycles to connect\n", i);
+  FreeIOBuffer (&buffer);
+
+  if (VerboseMode()) fprintf (stderr, "Connected\n");
+
+  stdio[0] = stdin_fd[1];
+  stdio[1] = stdout_fd[0];
+  stdio[2] = stderr_fd[0];
+
+  return (pid);
+
+pipe_error:
+  perror ("pipe error:");
+  goto close_pipes;
+
+connect_error:
+  if (VerboseMode()) fprintf (stderr, "error while connecting\n");
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
+  /* harvest the child process: kill & wait (< 100 ms) for exit */
+  kill (pid, SIGKILL);
+  result = waitpid (pid, &waitstatus, WNOHANG);
+  for (i = 0; (i < 50) && (result == 0); i++) {
+    nanosleep (&request, &remain);
+    result = waitpid (pid, &waitstatus, WNOHANG);
+  }
+
+  if ((result == -1) && (errno != ECHILD)) {
+    fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno);
+    exit (1);
+  }
+  if (result == 0) {
+    if (VerboseMode()) fprintf (stderr, "child did not exit??");
+  }
+  if (result > 0) {
+    if (result != pid) {
+      fprintf (stderr, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
+      exit (1);
+    }
+    if (WIFSTOPPED(waitstatus)) {
+      fprintf (stderr, "waitpid returns 'stopped': programming error\n");
+      exit (1);
+    }
+  }
+
+close_pipes:
+  if (stdin_fd[0]  != 0) close (stdin_fd[0]);
+  if (stdin_fd[1]  != 0) close (stdin_fd[1]);
+  if (stdout_fd[0] != 0) close (stdout_fd[0]);
+  if (stdout_fd[1] != 0) close (stdout_fd[1]);
+  if (stderr_fd[0] != 0) close (stderr_fd[0]);
+  if (stderr_fd[1] != 0) close (stderr_fd[1]);
+  return (FALSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/run.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/run.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/run.c	(revision 5922)
@@ -0,0 +1,13 @@
+# include "pcontrol.h"
+
+int run (int argc, char **argv) {
+
+  if (argc != 1) {
+    fprintf (stderr, "USAGE: run\n");
+    return (FALSE);
+  }
+
+  rl_event_hook = CheckSystem;
+
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/status.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/status.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/status.c	(revision 5922)
@@ -0,0 +1,62 @@
+# include "pcontrol.h"
+
+char jobname[6][32] = {"PENDING", "BUSY", "EXIT", "CRASH", "HUNG", "DONE"};
+char hostname[5][32] = {"IDLE", "BUSY", "DOWN", "DONE", "OFF"};
+
+int status (int argc, char **argv) {
+
+  PrintJobStack (PCONTROL_JOB_PENDING);
+  PrintJobStack (PCONTROL_JOB_BUSY);
+  PrintJobStack (PCONTROL_JOB_DONE);
+  PrintJobStack (PCONTROL_JOB_EXIT);
+  PrintJobStack (PCONTROL_JOB_CRASH);
+
+  PrintHostStack (PCONTROL_HOST_OFF);
+  PrintHostStack (PCONTROL_HOST_DOWN);
+  PrintHostStack (PCONTROL_HOST_IDLE);
+  PrintHostStack (PCONTROL_HOST_BUSY);
+  PrintHostStack (PCONTROL_HOST_DONE);
+  return (TRUE);
+}
+
+int PrintJobStack (int Nstack) {
+
+  int i, j, Nobject;
+  Stack *stack;
+  Job *job;
+
+  stack = GetJobStack (Nstack);
+  Nobject = stack[0].Nobject;
+  fprintf (stdout, "job stack %s:  %d objects\n", jobname[Nstack], Nobject);
+
+  for (i = 0; i < Nobject; i++) {
+    job = stack[0].object[i];
+    fprintf (stdout, "%d  %s  %d  ", i, job[0].hostname, job[0].argc);
+    for (j = 0; j < job[0].argc; j++) {
+      fprintf (stdout, "%s ", job[0].argv[j]);
+    }
+    PrintID (stdout, job[0].JobID);
+    fprintf (stdout, "\n");
+  }
+  return (TRUE);
+}
+
+int PrintHostStack (int Nstack) {
+
+  int i, Nobject;
+  Stack *stack;
+  Host *host;
+
+  stack = GetHostStack (Nstack);
+  Nobject = stack[0].Nobject;
+  fprintf (stdout, "host stack %s:  %d objects\n", hostname[Nstack], Nobject);
+
+  for (i = 0; i < Nobject; i++) {
+    host = stack[0].object[i];
+    fprintf (stdout, "%d  %s  ", i, host[0].hostname);
+    PrintID (stdout, host[0].HostID);
+    fprintf (stdout, "\n");
+  }
+  return (TRUE);
+}
+
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stdout.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stdout.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stdout.c	(revision 5922)
@@ -0,0 +1,63 @@
+# include "pcontrol.h"
+
+int stdout_pc (int argc, char **argv) {
+
+  int JobID;
+  Job *job;
+  IOBuffer *buffer;
+
+  if (argc != 2) {
+    fprintf (stderr, "USAGE: stdout (JobID)\n");
+    fprintf (stdout, "STATUS %d\n", -1);
+    return (FALSE);
+  }
+  
+  /* find Job of interest (must be EXIT or CRASH) */
+  JobID = atoi (argv[1]);
+  job = FindJobPtr (JobID, PCONTROL_JOB_EXIT);
+  if (job == NULL) {
+    job = FindJobPtr (JobID, PCONTROL_JOB_CRASH);
+    if (job == NULL) goto missing_stdout;
+  }    
+
+  buffer = &job[0].stdout;
+  fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout);
+  fprintf (stdout, "STATUS %d\n", 0);
+  return (TRUE);
+
+missing_stdout:
+  fprintf (stderr, "job not found in EXIT or CRASH\n");
+  fprintf (stdout, "STATUS %d\n", -2);
+  return (FALSE);
+}
+
+int stderr_pc (int argc, char **argv) {
+
+  int JobID;
+  Job *job;
+  IOBuffer *buffer;
+
+  if (argc != 2) {
+    fprintf (stderr, "USAGE: stderr (JobID)\n");
+    fprintf (stdout, "STATUS %d\n", -1);
+    return (FALSE);
+  }
+  
+  /* find Job of interest (must be EXIT or CRASH) */
+  JobID = atoi (argv[1]);
+  job = FindJobPtr (JobID, PCONTROL_JOB_EXIT);
+  if (job == NULL) {
+    job = FindJobPtr (JobID, PCONTROL_JOB_CRASH);
+    if (job == NULL) goto missing_stderr;
+  }    
+
+  buffer = &job[0].stderr;
+  fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout);
+  fprintf (stdout, "STATUS %d\n", 0);
+  return (TRUE);
+
+missing_stderr:
+  fprintf (stderr, "job not found in EXIT or CRASH\n");
+  fprintf (stdout, "STATUS %d\n", -2);
+  return (FALSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stop.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stop.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/stop.c	(revision 5922)
@@ -0,0 +1,13 @@
+# include "pcontrol.h"
+
+int stop (int argc, char **argv) {
+
+  if (argc != 1) {
+    fprintf (stderr, "USAGE: stop\n");
+    return (FALSE);
+  }
+
+  rl_event_hook = NULL;
+
+  return (TRUE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/verbose.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/verbose.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/verbose.c	(revision 5922)
@@ -0,0 +1,37 @@
+# include "pcontrol.h"
+
+static int VERBOSE = FALSE;
+
+int verbose (int argc, char **argv) {
+
+  if (argc == 1) {
+    if (VERBOSE) {
+      fprintf (stderr, "verbose mode ON\n");
+    } else {
+      fprintf (stderr, "verbose mode OFF\n");
+    }
+    return (TRUE);
+  }
+
+  if (argc == 2) {
+    if (!strcasecmp (argv[1], "ON")) {
+      VERBOSE = TRUE;
+      return (TRUE);
+    }
+    if (!strcasecmp (argv[1], "OFF")) {
+      VERBOSE = FALSE;
+      return (TRUE);
+    }
+    if (!strcasecmp (argv[1], "TOGGLE")) {
+      VERBOSE = ~VERBOSE;
+      return (TRUE);
+    }
+  }
+
+  fprintf (stderr, "USAGE: verbose (on/off/toggle)\n");
+  return (FALSE);
+}
+
+int VerboseMode () {
+  return (VERBOSE);
+}
Index: /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/version.c
===================================================================
--- /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/version.c	(revision 5922)
+++ /tags/pcontrol-0-3/Ohana/src/opihi/pcontrol/version.c	(revision 5922)
@@ -0,0 +1,14 @@
+# include "pcontrol.h"
+static char *name = "$Name: not supported by cvs2svn $";
+
+int version (int argc, char **argv) {
+
+  fprintf (stderr, "%s\n", name);
+
+  fprintf (stderr, "%s\n", opihi_version());
+  fprintf (stderr, "%s\n", ohana_version());
+  fprintf (stderr, "%s\n", fits_version());
+
+  fprintf (stderr, "compiled on %s %s\n", __DATE__, __TIME__);
+  return (TRUE);
+}