Index: /trunk/Ohana/src/opihi/include/pcontrol.h
===================================================================
--- /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 3202)
+++ /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 3203)
@@ -7,28 +7,44 @@
 # include <sys/wait.h>
 
-/** pcontrol global data **/
+typedef struct timeval Ptime;
+typedef unsigned long long IDtype;
 
 /** job status values **/
-enum {
+typedef enum {
   PCONTROL_JOB_PENDING,
   PCONTROL_JOB_BUSY,  
   PCONTROL_JOB_EXIT,
   PCONTROL_JOB_CRASH,
+  PCONTROL_JOB_HUNG,  
+  PCONTROL_JOB_DONE,  
 } JobStat;
 
 /** job status values **/
-enum {
+typedef enum {
   PCONTROL_JOB_ANYHOST,
   PCONTROL_JOB_WANTHOST,
-  PCONTROL_JOB_NEEDHOST,  
+  PCONTROL_JOB_NEEDHOST,
 } JobMode;
 
 /** host status values **/
-enum {
+typedef enum {
   PCONTROL_HOST_IDLE,
   PCONTROL_HOST_BUSY,  
   PCONTROL_HOST_DOWN,
+  PCONTROL_HOST_DONE,
   PCONTROL_HOST_OFF,
 } HostStat;
+
+/* stack special positions */
+typedef enum {
+  STACK_TOP = 0,
+  STACK_BOTTOM = -1,
+} StackWhere;
+
+typedef enum {
+  PCLIENT_HUNG = -1,
+  PCLIENT_DOWN = 0,
+  PCLIENT_GOOD = 1,
+} PclientStat;
 
 typedef struct {
@@ -49,36 +65,37 @@
 } Fifo;
 
-typedef struct timeval Ptime;
-
 /* data to define a job */
 typedef struct {
-  int        argc; 
-  char     **argv;
-  char      *hostname;
-  int        exitstatus;
-  JobMode    mode;
-  JobStat    status;
-  IOBuffer   stdin;
-  IOBuffer   stdout;
-  IOBuffer   stderr;
-  Ptime      start;
-  Ptime      accum;
-  Ptime      timer;
+  int          argc; 
+  char       **argv;
+  char        *hostname;
+  int          exit_status;
+  int          Reset;
+  JobMode      mode;
+  JobStat      state;
+  JobStat      stack;
+  IOBuffer     stdout;
+  IOBuffer     stderr;
+  Ptime        start;
+  Ptime        accum;
+  Ptime        timer;
   struct Host *host;
-  int        pid;
-  IDtype     JobID;
+  int          pid;
+  IDtype       JobID;
 } Job;
 
 /* data to define a host machine */
 typedef struct {
-  char      *hostname;
-  int        stdin;
-  int        stdout;
-  int        stderr;
-  int        markoff;
-  HostStat   status;
-  Ptime      start;
-  Ptime      accum;
-  Ptime      timer;
+  char       *hostname;
+  int         stdin;
+  int         stdout;
+  int         stderr;
+  int         markoff;
+  int         pid;
+  HostStat    stack;
+  Ptime       start;
+  Ptime       accum;
+  Ptime       timer;
+  IDtype      HostID;
   struct Job *job;
 } Host;
@@ -88,8 +105,12 @@
   int    Nobject;
   int    NOBJECT;
-} Queue;
+} Stack;
 
+/* these should be loaded from ConfigInit or from opihi variables */
 static char *CONNECT;  /* connection method (ssh, rsh, etc) */
 static char *PCLIENT;  /* shell on remote host (eg, pclient) */
+
+# define FREE(X) if (X != NULL) { free (X); }
+# define CLOSE(FD) { if (FD) close (FD); FD = 0; }
 
 /** IOBuffers allow the following operations (see pclient.h) **/
@@ -99,9 +120,44 @@
 void FreeIOBuffer (IOBuffer *buffer);
 
-# define FREE(X) if (X != NULL) { free (X); }
-
-enum {
-  QUEUE_TOP = 0,
-  QUEUE_BOTTOM = -1,
-} QueueWhere;
-
+void *GetStack (Stack *stack, int where);
+int PutStack (Stack *stack, int where, void *object);
+Stack *InitStack ();
+Stack *GetHostStack (int StackID);
+int PutHost (Host *host, int StackID, int where);
+Host *GetHost (int StackID, int where);
+int FindHost (IDtype HostID, int StackID);
+int FindNamedHost (char *name, int StackID);
+void AddHost (char *hostname);
+void DownHost (Host *host);
+void DelHost (Host *host);
+int CheckSystem ();
+int CheckBusyJobs ();
+int CheckDoneJobs ();
+int CheckDoneHosts ();
+int CheckLiveHosts ();
+int CheckDownHosts ();
+int CheckIdleHosts ();
+int CheckIdleHost (Host *host);
+void InitIDs ();
+IDtype NextJobID ();
+IDtype NextHostID ();
+void PrintID (FILE *f, IDtype ID);
+int CheckBusyJob (Job *job);
+int StartHost (Host *host);
+int CheckDoneHost (Host *host);
+int CheckDoneJob (Job *job);
+int GetJobOutput (Job *job, char *channel);
+int ResetJob (Job *job);
+int PclientCommand (Host *host, char *command, IOBuffer *buffer);
+int rconnect (char *command, char *hostname, char *shell, int *stdio);
+int CheckHost (Host *host);
+char *memstr (char *m1, char *m2, int n);
+Stack *GetJobStack (int StackID);
+int PutJob (Job *job, int StackID, int where);
+Job *GetJob (int StackID, int where);
+int FindJob (IDtype JobID, int StackID);
+IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv);
+void KillJob (Job *job);
+void DelJob (Job *job);
+Host *UnlinkJobAndHost (Job *job);
+void LinkJobAndHost (Job *job, Host *host);
Index: /trunk/Ohana/src/opihi/pclient/reset.c
===================================================================
--- /trunk/Ohana/src/opihi/pclient/reset.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pclient/reset.c	(revision 3203)
@@ -7,5 +7,5 @@
   if (argc != 1) {
     fprintf (stderr, "USAGE: reset\n");
-    fprintf (stdout, "STATUS %d\n", -2);
+    fprintf (stdout, "STATUS -1\n");
     return (FALSE);
   }
Index: /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckBusyJob.c	(revision 3203)
@@ -0,0 +1,85 @@
+# include "pcontrol.h"
+
+int CheckBusyJob (Job *job) {
+  
+  int      i;
+  int      status;
+  char    *p;
+  char     string[64];
+  IOBuffer buffer;
+  Host    *host;
+
+  /** must have a valid host : if not? **/
+  host = job[0].host;
+  /*** why is this a type error? ***/
+
+  InitIOBuffer (&buffer, 0x100);
+
+  status = PclientCommand (host, "status", &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      UnlinkJobAndHost (job);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      job[0].state = PCONTROL_JOB_HUNG;
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received\n");  
+      break;
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  /** host is up, need to parse message **/
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "missing STATUS in pclient message : programming error\n");
+    exit (1);
+  }
+  sscanf (p, "%*s %s", string);
+  if (!strcmp(string, "NONE")) {
+    fprintf (stderr, "no current job : programming error\n");
+    exit (1);
+  }
+  /** no status change, return to BUSY stack **/
+  if (!strcmp(string, "BUSY")) {
+    PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+    FreeIOBuffer (&buffer);
+    return (TRUE);
+  }
+
+  /** job has exited : move to DONE stack (host still BUSY) **/
+  if (!strcmp(string, "EXIT")) {
+    p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
+    sscanf (p, "%*s %d", &job[0].exit_status);
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    job[0].state = PCONTROL_JOB_EXIT;
+    FreeIOBuffer (&buffer);
+    return (TRUE);
+  }
+  /** job has crashed : move to DONE stack (host still BUSY) */
+  if (!strcmp(string, "CRASH")) {
+    p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
+    sscanf (p, "%*s %d", &job[0].exit_status);
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    job[0].state = PCONTROL_JOB_CRASH;
+    FreeIOBuffer (&buffer);
+    return (TRUE);
+  }
+
+  fprintf (stderr, "programming error : should not reach here (CheckJob)\n");
+  exit (1);
+}
+
+/** need to add timeout check here **/
Index: /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckDoneHost.c	(revision 3203)
@@ -0,0 +1,71 @@
+# include "pcontrol.h"
+
+int CheckDoneHost (Host *host) {
+  
+  int       status;
+  char     *p;
+  IOBuffer  buffer;
+
+  InitIOBuffer (&buffer, 0x100);
+  
+  status = PclientCommand (host, "reset", &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      /** do we need to close the connection? **/
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      fprintf (stderr, "host %s is down\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received\n");  
+      break;
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  /** successful command, examine result **/
+  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
+  if (p == NULL) {
+    fprintf (stderr, "missing STATUS in pclient message : programming error in CheckDoneHost\n");
+    exit (1);
+  }
+  sscanf (p, "%*s %d", &status);
+  switch (status) {
+    case -1:
+      fprintf (stderr, "reset syntax error : programming error\n");
+      exit (1);
+      
+    case 0:
+      fprintf (stderr, "reset failed\n");
+      PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+      
+    case 1:
+    case 2:
+      fprintf (stderr, "successful reset\n");
+      PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    default:
+      fprintf (stderr, "programming error in CheckDoneHost (should not reach here)\n");
+      exit (1);
+  }
+  fprintf (stderr, "programming error in CheckDoneHost (should not reach here either)\n");
+  exit (1);
+}
+
+/** probably need to flush the buffer before the command **/
+/** need to add timeout check here **/
Index: /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckDoneJob.c	(revision 3203)
@@ -0,0 +1,25 @@
+# include "pcontrol.h"
+
+int CheckDoneJob (Job *job) {
+  
+  Host *host;
+
+  if (!GetJobOutput (job, "stdout")) {
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    return (FALSE);
+  }
+
+  if (!GetJobOutput (job, "stderr")) {
+    PutJob (job, PCONTROL_JOB_DONE, STACK_BOTTOM);
+    return (FALSE);
+  }
+
+  host = UnlinkJobAndHost (job);
+  PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM);
+  PutJob (job, job[0].state, STACK_BOTTOM);
+  /* job's state is either EXIT or CRASH (verify?) */
+
+  return (TRUE);
+}
+
+/** need to add timeout check here **/
Index: /trunk/Ohana/src/opihi/pcontrol/CheckHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckHost.c	(revision 3203)
@@ -3,5 +3,5 @@
 int CheckHost (Host *host) {
   
-  int i, status;
+  int i, N, status;
   IOBuffer buffer;
   Job *job;
@@ -15,5 +15,5 @@
 
       /* if host has a job, job is dead, push to Pending */
-      if (host[0].status == PCONTROL_HOST_BUSY) {
+      if (host[0].stack == PCONTROL_HOST_BUSY) {
 	job = host[0].job;
 	N = FindJob (job[0].JobID, PCONTROL_JOB_BUSY);
@@ -24,9 +24,9 @@
 	job[0].host = NULL; /* unlink host & job */
 	job = GetJob (PCONTROL_JOB_BUSY, N);
-	PutJob (job, PCONTROL_JOB_PENDING, QUEUE_BOTTOM);
+	PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
       }
       host[0].job = NULL;
-      PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
-      FreeBuffer (&buffer);
+      PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
       return (FALSE);
       
@@ -34,12 +34,12 @@
       fprintf (stderr, "host %s is not responding\n", host[0].hostname);
       /*** do we mark this in some way (HUNG) ? ***/
-      PutHost (host, host[0].status, QUEUE_BOTTOM);
-      FreeBuffer (&buffer);
+      PutHost (host, host[0].stack, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
       return (FALSE);
 
     default:
       fprintf (stderr, "host %s is alive\n", host[0].hostname);
-      PutHost (host, host[0].status, QUEUE_BOTTOM);
-      FreeBuffer (&buffer);
+      PutHost (host, host[0].stack, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
       return (TRUE);
   }
Index: /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 3203)
@@ -0,0 +1,47 @@
+# include "pcontrol.h"
+
+int CheckIdleHost (Host *host) {
+
+  int i;
+  Stack *stack;
+  Job *job;
+
+  stack = GetJobStack (PCONTROL_JOB_PENDING);
+
+  /* look for first NEEDHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].mode != PCONTROL_JOB_NEEDHOST) continue;
+    if (strcasecmp (job[0].hostname, host[0].hostname)) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+
+  /* no NEEDHOST entry, look for first WANTHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
+    if (strcasecmp (job[0].hostname, host[0].hostname)) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+
+  /* no WANTHOST entry, look for first ANYHOST matching this host */
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object[i];
+    if (job[0].mode != PCONTROL_JOB_ANYHOST) continue;
+    if (strcasecmp (job[0].hostname, host[0].hostname)) continue;
+    LinkJobAndHost (job, host);
+    StartJob (job);
+    return (TRUE);
+  }
+}
+
+/** note : host and job popped off stacks **/
+
+/** elsewhere, we will check for timeout on the job and 
+    migrate to ANYHOST from WANTHOST or move to 
+    a failure condition for NEEDHOST 
+**/
Index: unk/Ohana/src/opihi/pcontrol/CheckJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckJob.c	(revision 3202)
+++ 	(revision )
@@ -1,82 +1,0 @@
-# include "pcontrol.h"
-# define CONNECT_TIMEOUT 20
-
-int CheckJob (Job *job) {
-  
-  int i, status;
-  IOBuffer buffer;
-  Host *host;
-
-  host = job[0].host;
-
-  InitIOBuffer (&buffer, 0x100);
-
-  status = PclientCommand (host, "status\n", &buffer);
-
-  /* check on success of pclient command */
-  switch (status) {
-    case DOWN:
-      fprintf (stderr, "host %s is down\n", host[0].hostname);
-      goto escape;
-      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST ***/
-
-    case HUNG:
-      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
-      goto escape;
-      /*** should we consider a HUNG host DOWN? ***/
-
-    case TRUE:
-      fprintf (stderr, "message received\n");  
-      break;
-
-    default:
-      fprintf (stderr, "unknown status for pclient command: programming error\n");  
-      exit (1);
-  }
-
-  /** host is up, need to parse message **/
-  p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer);
-  if (p == NULL) {
-    fprintf (stderr, "missing STATUS in pclient message : programming error\n");
-    exit (1);
-  }
-  fscan (p, "%*s %s", string);
-  if (!strcmp(string, "NONE")) {
-    fprintf (stderr, "no current job : programming error\n");
-    exit (2);
-  }
-  if (!strcmp(string, "BUSY")) {
-    /** no status change, return to BUSY queue **/
-    PutJob (job, PCONTROL_JOB_BUSY, QUEUE_BOTTOM);
-    return (TRUE);
-  }
-  if (!strcmp(string, "EXIT")) {
-    job[0].status = PCONTROL_JOB_EXIT;
-    p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
-    fscan (p, "%*s %d", &job[0].exitstatus);
-    return (TRUE);
-  }
-  if (!strcmp(string, "CRASH")) {
-    job[0].status = PCONTROL_JOB_CRASH;
-    p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer);
-    fscan (p, "%*s %d", &job[0].exitstatus);
-    return (TRUE);
-  }
-
-escape:
-  job[0].host = NULL; /* unlink host & job */
-  host[0].job = NULL;
-
-  /*** need to pop host off of correct queue XXX ***/
-  N = FindHost (host[0].ID, PCONTROL_HOST_BUSY);
-  if (N < 0) {
-    fprintf (stderr, "programming error: host is not found in BUSY list\n");
-    exit (2);
-  }
-  host = GetHost (PCONTROL_HOST_BUSY, N);
-  PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
-
-  PutJob (job, PCONTROL_JOB_PENDING, QUEUE_BOTTOM);
-  FreeBuffer (&buffer);
-  return (FALSE);
-}
Index: /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 3203)
@@ -4,72 +4,113 @@
 
   CheckBusyJobs();
+  CheckDoneJobs();
+  CheckDoneHosts();
+  CheckIdleHosts();
+  CheckDownHosts();
+
+  /** add in different timescales for heartbeat? **/
   CheckLiveHosts();
-  CheckDownHosts();
-  CheckIdleHosts();
-
 }
 
-CheckBusyJobs () {
+int CheckBusyJobs () {
 
-  queue = GetJobQueue (PCONTROL_JOB_BUSY);
-  Nobjects = queue[0].Nobjects;
+  int i, Nobject;
+  Stack *stack;
+  Job   *job;
 
-  for (i = 0; i < Nobjects; i++) {
-    job = GetObject (queue, QUEUE_TOP);
-    CheckJob (job);
+  stack = GetJobStack (PCONTROL_JOB_BUSY);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    job = GetStack (stack, STACK_TOP);
+    CheckBusyJob (job);
   }
 }
 
-CheckLiveHosts () {
+int CheckDoneJobs () {
 
-  queue = GetHostQueue (PCONTROL_HOST_IDLE);
-  Nobjects = queue[0].Nobjects;
+  int i, Nobject;
+  Stack *stack;
+  Job   *job;
 
-  for (i = 0; i < Nobjects; i++) {
-    host = GetObject (queue, QUEUE_TOP);
+  stack = GetJobStack (PCONTROL_JOB_DONE);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    job = GetStack (stack, STACK_TOP);
+    CheckDoneJob (job);
+  }
+}
+
+int CheckDoneHosts () {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+
+  stack = GetHostStack (PCONTROL_HOST_DONE);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
+    StartHost (host);
+  }
+}
+
+/* this is just a heartbeat check */
+int CheckLiveHosts () {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
     CheckHost (host);
   }
 
-  queue = GetHostQueue (PCONTROL_HOST_BUSY);
-  Nobjects = queue[0].Nobjects;
+  stack = GetHostStack (PCONTROL_HOST_BUSY);
+  Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobjects; i++) {
-    host = GetObject (queue, QUEUE_TOP);
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
     CheckHost (host);
   }
 }
 
-CheckDownHosts () {
+int CheckDownHosts () {
 
-  queue = GetHostQueue (PCONTROL_HOST_DOWN);
-  Nobjects = queue[0].Nobjects;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
 
-  for (i = 0; i < Nobjects; i++) {
-    host = GetObject (queue, QUEUE_TOP);
+  stack = GetHostStack (PCONTROL_HOST_DOWN);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
     StartHost (host);
   }
 }
 
-CheckIdleHosts () {
+int CheckIdleHosts () {
 
-  hqueue = HostPool_Idle;
-  jqueue = JobPool_Pending;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
 
-  for (i = 0; i < hqueue[0].Nobjects; i++) {
-    host = (Host *) hqueue[0].object[i];
-    /** need to add a timeout here **/
-    for (j = 0; j < jqueue[0].Nobjects; j++) {
-      job = (Job *) jqueue[0].object[j];
-      if ((job[0].mode == PCONTROL_JOB_ANYHOST) ||
-	  (!strcasecmp (job[0].hostname, host[0].hostname))
+  /* check if there are any pending jobs, otherwise skip step */
+  stack = GetJobStack (PCONTROL_JOB_PENDING);
+  if (!stack[0].Nobject) return;
 
-    StartHost (host);
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  for (i = 0; i < Nobject; i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckIdleHost (host);
   }
 }
-
-/*** this section is tricky: need to have a plan on making WANTHOST likely 
-     but not required  (on average, every host is currently in use.  some possible tricks:
-
-     - search through PENDING for N entries looking for WANTHOST / matching host
-     - keep job through N tries waiting for WANTHOST
-***/
Index: /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 3203)
@@ -0,0 +1,49 @@
+# include "pcontrol.h"
+
+int GetJobOutput (Job *job, char *channel) {
+  
+  int      status;
+  IOBuffer *buffer;
+  Host     *host;
+
+  buffer = NULL;
+  if (!strcasecmp (channel, "stdout")) buffer = &job[0].stdout;
+  if (!strcasecmp (channel, "stderr")) buffer = &job[0].stderr;
+  if (buffer == NULL) {
+    fprintf (stderr, "invalid output channel : programming error\n");
+    exit (1);
+  }
+
+  /* If we already have the output, just return, don't retry */
+  if (buffer[0].Nbuffer) return (TRUE);
+
+  /** must have a valid host : if not, move to pending? **/
+  host = job[0].host;
+  status = PclientCommand (host, channel, buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST ***/
+      buffer[0].Nbuffer = 0;
+      fprintf (stderr, "host %s is down\n", host[0].hostname);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      /*** should we consider a HUNG host DOWN? ***/
+      buffer[0].Nbuffer = 0;
+      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received\n");  
+      return (TRUE);
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  fprintf (stderr, "programming error in GetJobOutput (should not reach here)\n");
+  exit (1);
+}
Index: /trunk/Ohana/src/opihi/pcontrol/HostOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/HostOps.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/HostOps.c	(revision 3203)
@@ -1,11 +1,18 @@
 # include "pcontrol.h"
 
-Queue *HostPool_Idle;
-Queue *HostPool_Busy;
-Queue *HostPool_Down;
-Queue *HostPool_Off;
+Stack *HostPool_Idle;
+Stack *HostPool_Busy;
+Stack *HostPool_Down;
+Stack *HostPool_Off;
 
-Queue *GetHostQueue (int QueueID) {
-  switch (QueueID) {
+void InitHostStacks () {
+  HostPool_Idle = InitStack ();
+  HostPool_Busy = InitStack ();
+  HostPool_Down = InitStack ();
+  HostPool_Off = InitStack ();
+}
+
+Stack *GetHostStack (int StackID) {
+  switch (StackID) {
     case PCONTROL_HOST_IDLE:
       return (HostPool_Idle);
@@ -17,5 +24,5 @@
       return (HostPool_Off);
     default:
-      fprintf (stderr, "error: unknown host queue\n");
+      fprintf (stderr, "error: unknown host stack\n");
       return (NULL);
   }
@@ -23,38 +30,40 @@
 }
 
-int PutHost (Host *host, int QueueID, int where) {
+int PutHost (Host *host, int StackID, int where) {
 
   int status;
-  Queue *queue;
+  Stack *stack;
 
-  queue = GetHostQueue (QueueID);
-  if (queue == NULL) return (FALSE);
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (FALSE);
 
-  host[0].status = QueueID;
-  status = PutObject (queue, where, host);
+  host[0].stack = StackID;
+  status = PutStack (stack, where, host);
   return (status);
 }
   
-Host *GetHost (int QueueID, int where) {
+Host *GetHost (int StackID, int where) {
 
   Host *host;
-  Queue *queue;
+  Stack *stack;
 
-  queue = GetHostQueue (QueueID);
-  if (queue == NULL) return (NULL);
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (NULL);
 
-  host = GetObject (queue, where);
+  host = GetStack (stack, where);
   return (host);
 }
 
-int FindHost (IDtype HostID, int QueueID) {
+int FindHost (IDtype HostID, int StackID) {
 
-  Queue *queue;
+  int i;
+  Host *host;
+  Stack *stack;
 
-  queue = GetHostQueue (QueueID);
-  if (queue == NULL) return (-2);
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (-2);
 
-  for (i = 0; i < queue[0].Nobject; i++) {
-    host = (Host *) queue[0].object;
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = (Host *) stack[0].object;
     if (host[0].HostID == HostID) {
       return (i);
@@ -64,14 +73,16 @@
 }
 
-int FindNamedHost (char *name, int QueueID) {
+int FindNamedHost (char *name, int StackID) {
 
-  Queue *queue;
+  int i;
+  Host *host;
+  Stack *stack;
 
-  queue = GetHostQueue (QueueID);
-  if (queue == NULL) return (-2);
+  stack = GetHostStack (StackID);
+  if (stack == NULL) return (-2);
 
-  for (i = 0; i < queue[0].Nobject; i++) {
-    host = (Host *) queue[0].object;
-    if (!strcasecmp (host[0].hostname, host)) {
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = (Host *) stack[0].object;
+    if (!strcasecmp (host[0].hostname, name)) {
       return (i);
     }
@@ -80,5 +91,5 @@
 }
 
-AddHost (char *hostname) {
+void AddHost (char *hostname) {
 
   Host *host;
@@ -94,8 +105,17 @@
   host[0].markoff  = FALSE;
   host[0].job      = NULL;
-  DownHost (host);
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
 }
 
-DelHost (Host *host) {
+void DownHost (Host *host) {
+
+  CLOSE (host[0].stdin);
+  CLOSE (host[0].stdout);
+  CLOSE (host[0].stderr);
+  host[0].job = NULL;
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+}
+
+void DelHost (Host *host) {
   FREE (host[0].hostname);
   FREE (host[0].job);
@@ -103,10 +123,2 @@
 }
 
-MarkOffHost (Host *host) {
-  if (host[0].status != PCONTROL_HOST_BUSY) {
-    return (FALSE);
-  }
-  host[0].markoff  = TRUE;
-  PutObject (HostPool_Busy, host);  /**** where are we removing hosts? **/
-}
-
Index: /trunk/Ohana/src/opihi/pcontrol/IOBufferOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/IOBufferOps.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/IOBufferOps.c	(revision 3203)
@@ -0,0 +1,66 @@
+# include "pclient.h"
+
+int InitIOBuffer (IOBuffer *buffer, int Nalloc) {
+
+  buffer[0].Nalloc = Nalloc;
+  buffer[0].Nreset = Nalloc;
+  buffer[0].Nblock = Nalloc / 2;
+  buffer[0].Nbuffer = 0;
+
+  ALLOCATE (buffer[0].buffer, char, buffer[0].Nalloc);
+
+  return (TRUE);
+}
+
+int FlushIOBuffer (IOBuffer *buffer) {
+
+  buffer[0].Nbuffer = 0;
+  buffer[0].Nalloc = buffer[0].Nreset;
+  REALLOCATE (buffer[0].buffer, char, buffer[0].Nalloc);
+
+  return (TRUE);
+}
+
+int ReadtoIOBuffer (IOBuffer *buffer, int fd) {
+
+  int Nread, Nfree;
+
+  if (fd == 0) {
+    /* pipe is closed */
+    return (0);
+  }
+
+  Nfree = buffer[0].Nalloc - buffer[0].Nbuffer;
+  if (Nfree < buffer[0].Nblock) {
+    buffer[0].Nalloc += 2*buffer[0].Nblock;
+    REALLOCATE (buffer[0].buffer, char, buffer[0].Nalloc);
+    Nfree = buffer[0].Nalloc - buffer[0].Nbuffer;
+  }
+
+  Nread = read (fd, &buffer[0].buffer[buffer[0].Nbuffer], buffer[0].Nblock);
+
+  if (Nread >= 0) {
+    buffer[0].Nbuffer += Nread;
+    return (Nread);
+  }
+
+  if (Nread == -1) {
+    switch (errno) {
+    case EAGAIN:
+    case EIO:
+      /** no data available in pipe **/
+      return (-1);
+    default:
+      /** error reading from pipe **/
+      perror ("ReadtoIOBuffer read error");
+      return (-2);
+    }
+  }
+}
+
+void FreeIOBuffer (IOBuffer *buffer) {
+
+  if (buffer[0].buffer != (char *) NULL) {
+    free (buffer[0].buffer);
+  }
+}
Index: /trunk/Ohana/src/opihi/pcontrol/JobID.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/JobID.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/JobID.c	(revision 3203)
@@ -1,5 +1,3 @@
 # include "pcontrol.h"
-
-typedef unsigned long long IDtype;
 
 static IDtype CurrentJobID  = 0;
@@ -7,5 +5,5 @@
 
 /* for now, no persistence : we could use the date/time to seed the upper byte(s) if needed */
-InitIDs () {
+void InitIDs () {
   CurrentJobID = 0;
   CurrentHostID = 0;
@@ -32,4 +30,6 @@
 void PrintID (FILE *f, IDtype ID) {
 
+  unsigned short int word0, word1, word2, word3;
+
   word0 = 0xffff & ID;
   word1 = 0xffff & (ID >> 16);
Index: /trunk/Ohana/src/opihi/pcontrol/JobOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 3203)
@@ -1,11 +1,18 @@
 # include "pcontrol.h"
 
-Queue *JobPool_Pending;
-Queue *JobPool_Busy;
-Queue *JobPool_Exit;
-Queue *JobPool_Crash;
+Stack *JobPool_Pending;
+Stack *JobPool_Busy;
+Stack *JobPool_Exit;
+Stack *JobPool_Crash;
 
-Queue *GetJobQueue (int QueueID) {
-  switch (QueueID) {
+void InitJobStacks () {
+  JobPool_Pending = InitStack ();
+  JobPool_Busy = InitStack ();
+  JobPool_Exit = InitStack ();
+  JobPool_Crash = InitStack ();
+}
+
+Stack *GetJobStack (int StackID) {
+  switch (StackID) {
     case PCONTROL_JOB_PENDING:
       return (JobPool_Pending);
@@ -14,8 +21,8 @@
     case PCONTROL_JOB_EXIT:
       return (JobPool_Exit);
-    case PCONTROL_JOB_Crash:
+    case PCONTROL_JOB_CRASH:
       return (JobPool_Crash);
     default:
-      fprintf (stderr, "error: unknown job queue\n");
+      fprintf (stderr, "error: unknown job stack\n");
       return (NULL);
   }
@@ -23,39 +30,42 @@
 }
 
-int PutJob (Job *job, int QueueID, int where) {
+int PutJob (Job *job, int StackID, int where) {
 
   int status;
-  Queue *queue;
+  Stack *stack;
 
-  queue = GetJobQueue (QueueID);
-  if (queue == NULL) return (FALSE);
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (FALSE);
 
-  job[0].status = QueueID;
-  status = PutObject (queue, where, job);
+  /* be default, these are both the same - to override, set state after PutJob */
+  job[0].state = StackID;
+  job[0].stack = StackID;
+  status = PutStack (stack, where, job);
   return (status);
 }
   
-Job *GetJob (int QueueID, int where) {
+Job *GetJob (int StackID, int where) {
 
   Job *job;
-  Queue *queue;
+  Stack *stack;
 
-  queue = GetJobQueue (QueueID);
-  if (queue == NULL) return (NULL);
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (NULL);
 
-  job = GetObject (queue, where);
+  job = GetStack (stack, where);
   return (job);
 }
   
-int FindJob (IDtype JobID, int QueueID) {
+int FindJob (IDtype JobID, int StackID) {
 
+  int i;
   Job *job;
-  Queue *queue;
+  Stack *stack;
 
-  queue = GetJobQueue (QueueID);
-  if (queue == NULL) return (-2);
+  stack = GetJobStack (StackID);
+  if (stack == NULL) return (-2);
 
-  for (i = 0; i < queue[0].Nobject; i++) {
-    job = (Job *) queue[0].object;
+  for (i = 0; i < stack[0].Nobject; i++) {
+    job = (Job *) stack[0].object;
     if (job[0].JobID == JobID) {
       return (i);
@@ -65,5 +75,5 @@
 }
 
-AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv) {
+IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv) {
 
   Job *job;
@@ -76,14 +86,18 @@
   job[0].mode     = mode;
   job[0].host     = NULL;
-  job[0].ID       = NextJobID();
+  job[0].JobID    = NextJobID();
+  job[0].Reset    = FALSE;
 
-  InitIOBuffer (&job[0].stdin,  0x1000);
+  /* do this step on start? */
   InitIOBuffer (&job[0].stdout, 0x1000);
   InitIOBuffer (&job[0].stderr, 0x1000);
 
-  PutJob (job, PCONTROL_JOB_PENDING, QUEUE_BOTTOM);
+  PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+  return (job[0].JobID);
 }
 
-DelJob (Job *job) {
+void DelJob (Job *job) {
+
+  int i;
 
   FREE (job[0].hostname);
@@ -93,5 +107,4 @@
   FREE (job[0].argv);
 
-  FreeIOBuffer (&job[0].stdin);
   FreeIOBuffer (&job[0].stdout);
   FreeIOBuffer (&job[0].stderr);
@@ -100,7 +113,10 @@
 }
 
-KillJob (Job *job) {
+# if (0)
+void KillJob (Job *job) {
 
+  int status;
   char line[64];
+  Host *host;
 
   sprintf (line, "reset\n");
@@ -110,9 +126,37 @@
   if ((status == -1) && (errno == EPIPE)) {
     fprintf (stderr, "host %s is down\n", host[0].hostname);
-    PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
-    PutJob (job, PCONTROL_JOB_PENDING, QUEUE_BOTTOM);
+    PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+    PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
     return (FALSE);
   }
   /* check for a response */
 
+  /** this needs to be cleaned up to handle the slow response case **/
 }
+# endif
+
+Host *UnlinkJobAndHost (Job *job) {
+
+  int N;
+  Host *host;
+
+  host = job[0].host;
+
+  /* unlink host & job */
+  job[0].host = NULL;
+  host[0].job = NULL;
+  
+  /*** need to pop host off of correct stack XXX ***/
+  N = FindHost (host[0].HostID, host[0].stack);
+  if (N < 0) {
+    fprintf (stderr, "programming error: host is not found in current stack\n");
+    exit (2);
+  }
+  host = GetHost (host[0].stack, N);
+  return (host);
+}
+
+void LinkJobAndHost (Job *job, Host *host) {
+  job[0].host = host;
+  host[0].job = job;
+}
Index: /trunk/Ohana/src/opihi/pcontrol/Makefile
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 3203)
@@ -0,0 +1,119 @@
+include ../../../Configure
+
+default: all
+
+HOME    =       $(ROOT)/src/opihi.v2
+BIN     =       $(HOME)/bin
+LIB     =       $(HOME)/lib
+INC     =       $(HOME)/include
+HELP    =       $(HOME)/help
+SDIR    =       $(HOME)/pcontrol
+#
+DESTBIN =       $(LBIN)
+DESTLIB =       $(LLIB)
+DESTINC =       $(LINC)
+DESTMAN =       $(LMAN)
+#  
+INCS    =       -I$(INC) -I$(LINC) -I$(XINC)
+LFLAGS  =       -L$(LLIB) -L$(LIB)
+LIBS1   =       -lFITS -lsocket -lnsl -lreadline -ltermcap -lohana -lm
+LIBS2   =       -lbasiccmd -lshell -ldata
+LIBS    =       $(LIBS2) $(LIBS1) 
+CFLAGS  =       $(INCS)
+CCFLAGS =       $(LIBS) 
+
+# mana user commands and support functions ########################
+
+funcs = \
+$(SDIR)/init.$(ARCH).o \
+$(SDIR)/pclient.$(ARCH).o \
+$(SDIR)/pcontrol.$(ARCH).o \
+$(SDIR)/rconnect.$(ARCH).o \
+$(SDIR)/memstr.$(ARCH).o \
+$(SDIR)/CheckBusyJob.$(ARCH).o \
+$(SDIR)/CheckDoneHost.$(ARCH).o \
+$(SDIR)/CheckDoneJob.$(ARCH).o \
+$(SDIR)/CheckHost.$(ARCH).o \
+$(SDIR)/CheckIdleHost.$(ARCH).o \
+$(SDIR)/CheckSystem.$(ARCH).o \
+$(SDIR)/GetJobOutput.$(ARCH).o \
+$(SDIR)/HostOps.$(ARCH).o \
+$(SDIR)/IOBufferOps.$(ARCH).o \
+$(SDIR)/JobID.$(ARCH).o \
+$(SDIR)/JobOps.$(ARCH).o \
+$(SDIR)/QueueOps.$(ARCH).o \
+$(SDIR)/ResetJob.$(ARCH).o \
+$(SDIR)/StartHost.$(ARCH).o \
+$(SDIR)/StartJob.$(ARCH).o
+
+cmds = \
+$(SDIR)/job.$(ARCH).o \
+$(SDIR)/status.$(ARCH).o \
+$(SDIR)/host.$(ARCH).o 
+
+pcontrol = $(cmds) $(funcs)
+
+libs = \
+$(DESTLIB)/libbasiccmd.a \
+$(DESTLIB)/libshell.a \
+$(DESTLIB)/libdata.a
+
+all: pcontrol
+
+pcontrol: $(BIN)/pcontrol.$(ARCH)
+	@echo done
+
+$(BIN)/pcontrol.$(ARCH) : $(pcontrol) $(libs)
+
+install: $(DESTBIN)/pcontrol
+
+# utilities #################################################
+$(BIN)/%.$(ARCH):
+	@if [ ! -d $(BIN) ]; then mkdir -p $(BIN); fi
+	$(CC) $^ -o $@ $(LFLAGS) $(LIBS)
+	@echo "compiled $*"
+	@echo ""
+
+$(DESTBIN)/%: $(BIN)/%.$(ARCH) 
+	@if [ ! -d $(DESTBIN) ]; then mkdir -p $(DESTBIN); fi
+	rm -f $(DESTBIN)/$*
+	cp $(BIN)/$*.$(ARCH) $(DESTBIN)/$*
+	@echo "installed $*"
+	@echo ""
+
+$(LIB)/%.$(ARCH).a:
+	@if [ ! -d $(LIB) ]; then mkdir -p $(LIB); fi
+	rm -f $@
+	ar rcv $@ $^ 
+	$(RANLIB) $@
+	@echo "compiled library $*"
+	@echo ""
+
+$(DESTLIB)/%.a: $(LIB)/%.$(ARCH).a
+	@if [ ! -d $(DESTLIB) ]; then mkdir -p $(DESTLIB); fi
+	@echo ""
+	rm -f $@
+	cp $^ $@
+
+clean:
+	rm -f $(BIN)/*.$(ARCH)
+	rm -f $(LIB)/*.$(ARCH).a
+	rm -f `find . -name "*.o"`
+	rm -f `find . -name "*~"`
+	rm -f `find . -name "#*"`
+
+dist: clean
+	rm -f $(BIN)/*
+	rm -f $(LIB)/*
+
+lib%.clean:
+	rm -f $(LIB)/lib$*.$(ARCH).a
+	rm -f $($*)
+	@echo ""
+
+%.clean:
+	rm -f $(BIN)/$*.$(ARCH)
+	@echo ""
+
+%.$(ARCH).o : %.c
+	$(CC) $(CFLAGS) -c $*.c -o $@
Index: /trunk/Ohana/src/opihi/pcontrol/QueueOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/QueueOps.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/QueueOps.c	(revision 3203)
@@ -2,19 +2,19 @@
 
 /* get object from point in stack (negative == distance from end) */
-void *GetQueue (Queue *queue, int where) {
+void *GetStack (Stack *stack, int where) {
 
   int i;
   void *object;
   
-  /* QUEUE_TOP == 0, QUEUE_BOTTOM == -1 */
+  /* STACK_TOP == 0, STACK_BOTTOM == -1 */
   /* this code correctly handles the negative 'where' and Nobject == 0 */
-  if (where < 0) where += queue[0].Nobject;
+  if (where < 0) where += stack[0].Nobject;
   if (where < 0) return (NULL);
-  if (where >= queue[0].Nobject) return (NULL);
+  if (where >= stack[0].Nobject) return (NULL);
 
-  object = queue[0].object[where];
-  queue[0].Nobject --;
-  for (i = where; i < queue[0].Nobject; i++) {
-    queue[0].object[i] = queue[0].object[i+1];
+  object = stack[0].object[where];
+  stack[0].Nobject --;
+  for (i = where; i < stack[0].Nobject; i++) {
+    stack[0].object[i] = stack[0].object[i+1];
   }
   return (object);
@@ -22,42 +22,42 @@
 
 /* push object on top of stack */
-int PutQueue (Queue *queue, int where, void *object) {
+int PutStack (Stack *stack, int where, void *object) {
 
   int i;
 
-  /* QUEUE_TOP == 0, QUEUE_BOTTOM == -1 */
+  /* STACK_TOP == 0, STACK_BOTTOM == -1 */
   /* this code correctly handles the negative 'where' and Nobject == 0 */
-  if (where < 0) where += queue[0].Nobject + 1;
+  if (where < 0) where += stack[0].Nobject + 1;
   if (where < 0) return (FALSE);
-  if (where > queue[0].Nobject) return (FALSE);
+  if (where > stack[0].Nobject) return (FALSE);
 
-  /* extend queue as needed */
-  if (queue[0].Nobject >= queue[0].NOBJECT) {
-    queue[0].NOBJECT += 100;
-    REALLOCATE (queue[0].object, void *, queue[0].NOBJECT);
+  /* extend stack as needed */
+  if (stack[0].Nobject >= stack[0].NOBJECT) {
+    stack[0].NOBJECT += 100;
+    REALLOCATE (stack[0].object, void *, stack[0].NOBJECT);
   }
 
-  for (i = queue[0].Nobject; i > where; i--) {
-    queue[0].object[i] = queue[0].object[i-1];
+  for (i = stack[0].Nobject; i > where; i--) {
+    stack[0].object[i] = stack[0].object[i-1];
   }
-  queue[0].object[where] = object;
-  queue[0].Nobject ++;
+  stack[0].object[where] = object;
+  stack[0].Nobject ++;
   return (TRUE);
 }
 
-/* allocate queue, setup with default values, allocate data */
-Queue *InitQueue () {
+/* allocate stack, setup with default values, allocate data */
+Stack *InitStack () {
 
-  Queue *queue;
+  Stack *stack;
 
-  ALLOCATE (queue, Queue, 1);
+  ALLOCATE (stack, Stack, 1);
 
-  queue[0].Nobject = 0;
-  queue[0].NOBJECT = 50;
-  ALLOCATE (queue[0].object, void *, queue[0].NOBJECT);
-  return (queue);
+  stack[0].Nobject = 0;
+  stack[0].NOBJECT = 50;
+  ALLOCATE (stack[0].object, void *, stack[0].NOBJECT);
+  return (stack);
 }
 
-/* these queues are not super efficient, and should probably be replaced with linked lists, 
+/* these stacks are not super efficient, and should probably be replaced with linked lists, 
    but I find these easier to get my brain around
 */
Index: /trunk/Ohana/src/opihi/pcontrol/ResetJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/ResetJob.c	(revision 3203)
@@ -0,0 +1,52 @@
+# include "pcontrol.h"
+
+int ResetJob (Job *job) {
+  
+  int       status;
+  IOBuffer  buffer;
+  Host     *host;
+
+  /** must have a valid host : if not, move to pending? **/
+  host = job[0].host;
+
+  InitIOBuffer (&buffer, 0x100);
+  
+  /* we have tried to reset the job; may not get status */
+  job[0].Reset = TRUE;
+
+  status = PclientCommand (host, "reset", &buffer);
+
+  /* check on success of pclient command */
+  switch (status) {
+    case PCLIENT_DOWN:
+      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST ***/
+      fprintf (stderr, "host %s is down\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_HUNG:
+      /*** should we consider a HUNG host DOWN? ***/
+      fprintf (stderr, "host %s is not responding\n", host[0].hostname);
+      FreeIOBuffer (&buffer);
+      return (FALSE);
+
+    case PCLIENT_GOOD:
+      fprintf (stderr, "message received\n");  
+      FreeIOBuffer (&buffer);
+      return (TRUE);
+
+    default:
+      fprintf (stderr, "unknown status for pclient command: programming error\n");  
+      exit (1);
+  }
+
+  fprintf (stderr, "programming error in GetJobOutput (should not reach here)\n");
+  exit (1);
+}
+
+/* if machine is down, return FALSE
+   this will place job back in BUSY state,
+   next check of job will catch state and
+   put machine down correctly
+
+*/
Index: /trunk/Ohana/src/opihi/pcontrol/StartHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StartHost.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/StartHost.c	(revision 3203)
@@ -3,10 +3,13 @@
 int StartHost (Host *host) {
 
+  int pid;
   int stdio[3];
 
-  pid = rconnect (CONNECT, host[0].hostname, PCLIENT, stdio);
+  /* pid = rconnect (CONNECT, host[0].hostname, PCLIENT, stdio); */
+
+  pid = rconnect ("ssh", host[0].hostname, "pclient", stdio);
   if (!pid) {     /** failure to start **/
     fprintf (stderr, "failure to start %s\n", host[0].hostname);
-    PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
+    PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
     return (FALSE);
   }
@@ -15,5 +18,5 @@
   host[0].stderr = stdio[2];
   host[0].pid    = pid;
-  PutHost (host, PCONTROL_HOST_IDLE, QUEUE_BOTTOM);
+  PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
   return (TRUE);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/StartJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/StartJob.c	(revision 3203)
@@ -3,9 +3,10 @@
 int StartJob (Job *job) {
 
-  int  Nline;
-  char *line;
+  int  i, Nline, status;
+  char *line, *p;
+  Host *host;
   IOBuffer buffer;
 
-  InitIOBuffer (&buffer);
+  InitIOBuffer (&buffer, 0x100);
 
   /* job must have assigned host */
@@ -17,6 +18,6 @@
 
   /* construct command line : job arg0 arg1 ... argN\n */
-  Nline = 10 + argc;
-  for (i = 0; i < argc; i++) {
+  Nline = 10 + job[0].argc;
+  for (i = 0; i < job[0].argc; i++) {
     Nline += strlen (job[0].argv[i]);
   }
@@ -24,9 +25,8 @@
   bzero (line, Nline);
   strcpy (line, "job");
-  for (i = 0; i < argc; i++) {
+  for (i = 0; i < job[0].argc; i++) {
     strcat (line, " ");
     strcat (line, job[0].argv[i]);
   }
-  strcat (line, "\n");
 
   status = PclientCommand (host, line, &buffer);
@@ -35,15 +35,13 @@
   /* check on success of pclient command */
   switch (status) {
-    case DOWN:
+    case PCLIENT_DOWN:
       fprintf (stderr, "host %s is down\n", host[0].hostname);
-      goto escape;
-      /*** different behavior for ANYHOST, WANTHOST, NEEDHOST ***/
+      goto failure;
 
-    case HUNG:
+    case PCLIENT_HUNG:
       fprintf (stderr, "host %s is not responding\n", host[0].hostname);
-      goto escape;
-      /*** should we consider a HUNG host DOWN? ***/
+      goto failure;
 
-    case TRUE:
+    case PCLIENT_GOOD:
       fprintf (stderr, "message received\n");  
       break;
@@ -60,10 +58,9 @@
     exit (1);
   }
-  fscan (p, "%*s %d", &status);
+  sscanf (p, "%*s %d", &status);
   switch (status) {
     case -1:
       fprintf (stderr, "error in pclient child\n");
-      goto escape;
-      /*** should we consider this host DOWN? ***/
+      goto failure;
 
     case -2:
@@ -77,6 +74,8 @@
     default:
       job[0].pid = status;
-      PutHost (host, PCONTROL_HOST_BUSY, QUEUE_BOTTOM);
-      PutJob (job, PCONTROL_JOB_BUSY, QUEUE_BOTTOM);
+      PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
+      PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
+      FreeIOBuffer (&buffer);
+      /** need to start timers here **/
       return (TRUE);
   }
@@ -85,21 +84,13 @@
   exit (1);
 
-escape:
-  job[0].host = NULL; /* unlink host & job */
+failure:
+  /* unlink host & job */
+  job[0].host = NULL;
   host[0].job = NULL;
-  /*** need to pop host from corresponding queue (BUSY) ***/
-  PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
-  PutJob (job, PCONTROL_JOB_PENDING, QUEUE_BOTTOM);
-  FreeBuffer (&buffer);
+  PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
+  PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
+  FreeIOBuffer (&buffer);
   return (FALSE);
 }
 
-/* 
-   this function sends a new command to the pclient.
-   before calling this function:
-     - grab the job from its queue (remove it from that queue with GetQueue)
-     - identify the target host (MUST already be IDLE)
-     - grab the host from its queue (remove it from that queue with GetQueue)
-     - link job and host (job[0].host = host, host[0].job = job)
-     - job must also have a valid command (argc, argv)
-*/
+/** note : host and job popped off stacks : can't use UnlinkJobAndHost **/
Index: /trunk/Ohana/src/opihi/pcontrol/host.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/host.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/host.c	(revision 3203)
@@ -3,7 +3,8 @@
 int host (int argc, char **argv) {
 
-  int pid, status;
+  int N, Delete, Off, On;
+  Host *host;
 
-  /* this section needs some help: find the specified host in the queues */
+  /* this section needs some help: find the specified host in the stacks */
   Delete = FALSE;
   if ((N = get_argument (argc, argv, "-delete"))) {
@@ -12,5 +13,5 @@
   }
  
-  /* this section needs some help: find the specified host in the queues */
+  /* this section needs some help: find the specified host in the stacks */
   Off = FALSE;
   if ((N = get_argument (argc, argv, "-off"))) {
@@ -29,9 +30,10 @@
   
   if (Delete) {
-    host = FindHost (argv[1], PCONTROL_HOST_OFF);
-    if (host == NULL) {
+    N = FindNamedHost (argv[1], PCONTROL_HOST_OFF);
+    if (N < 0) {
       fprintf (stderr, "host %s is not OFF\n", argv[1]);
       return (FALSE);
     }
+    host = GetHost (PCONTROL_HOST_OFF, N);
     DelHost (host);
     return (TRUE);
@@ -45,5 +47,5 @@
     }
     host = GetHost (PCONTROL_HOST_OFF, N);
-    PutHost (host, PCONTROL_HOST_DOWN, QUEUE_BOTTOM);
+    DownHost (host);
     return (TRUE);
   }
@@ -52,18 +54,22 @@
     N = FindNamedHost (argv[1], PCONTROL_HOST_IDLE);
     if (N >= 0) {
-      host = GetHost (PCONTROL_HOST_OFF, N);
-      PutHost (host, PCONTROL_HOST_OFF, QUEUE_BOTTOM);
+      host = GetHost (PCONTROL_HOST_IDLE, N);
+      CLOSE (host[0].stdin);
+      CLOSE (host[0].stdout);
+      CLOSE (host[0].stderr);
+      PutHost (host, PCONTROL_HOST_OFF, STACK_BOTTOM);
       return (TRUE);
     }
     N = FindNamedHost (argv[1], PCONTROL_HOST_DOWN);
     if (N >= 0) {
-      host = GetHost (PCONTROL_HOST_OFF, N);
-      PutHost (host, PCONTROL_HOST_OFF, QUEUE_BOTTOM);
+      host = GetHost (PCONTROL_HOST_DOWN, N);
+      PutHost (host, PCONTROL_HOST_OFF, STACK_BOTTOM);
       return (TRUE);
     }
     N = FindNamedHost (argv[1], PCONTROL_HOST_BUSY);
     if (N >= 0) {
-      host = GetHost (PCONTROL_HOST_OFF, N);
-      PutHost (host, PCONTROL_HOST_OFF, QUEUE_BOTTOM);
+      host = GetHost (PCONTROL_HOST_BUSY, N);
+      host[0].markoff  = TRUE;
+      PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
       return (TRUE);
     }
Index: /trunk/Ohana/src/opihi/pcontrol/init.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/init.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/init.c	(revision 3203)
@@ -0,0 +1,25 @@
+# include "opihi.h"
+
+int job	            PROTO((int, char **));
+int host            PROTO((int, char **));
+int status          PROTO((int, char **));
+
+static Command cmds[] = {  
+  {"job",       job,      "add job"},
+  {"host",      host,     "add / delete / modify host"},
+  {"status",    status,   "get system status"},
+}; 
+
+void InitPcontrol () {
+  
+  int i;
+
+  for (i = 0; i < sizeof (cmds) / sizeof (Command); i++) {
+    AddCommand (&cmds[i]);
+  }
+
+  /* temporary : put this elsewhere? */
+  InitJobStacks ();
+  InitHostStacks ();
+
+}
Index: /trunk/Ohana/src/opihi/pcontrol/job.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/job.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/job.c	(revision 3203)
@@ -3,5 +3,7 @@
 int job (int argc, char **argv) {
 
-  int pid, status;
+  char *Host, **targv;
+  int i, N, Mode, targc, Timeout;
+  IDtype JobID;
 
   Host = NULL;
@@ -16,5 +18,5 @@
     if (Mode == PCONTROL_JOB_WANTHOST) {
       fprintf (stderr, "ERROR: -host and +host are incompatible\n");
-      free (Host);
+      FREE (Host);
       return (FALSE);
     }
@@ -34,4 +36,5 @@
   if (argc < 2) {
     fprintf (stderr, "USAGE: job [options] (arg0) (arg1) ... (argN)\n");
+    FREE (Host);
     return (FALSE);
   }
@@ -46,5 +49,5 @@
   FREE (Host);
 
-  fprintf (stdout, "STATUS %d\n", ChildPID);
+  fprintf (stdout, "STATUS %d\n", JobID);
   return (TRUE);
 
Index: /trunk/Ohana/src/opihi/pcontrol/memstr.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/memstr.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/memstr.c	(revision 3203)
@@ -0,0 +1,13 @@
+# include "pcontrol.h"
+
+/* returns pointer to start of m2 in m1, or NULL if failure */ 
+char *memstr (char *m1, char *m2, int n) {
+
+  int i, N;
+
+  N = strlen (m2);
+  for (i = 0; (i < n - N + 1) && memcmp (m1, m2, N); i++, m1++);
+  if (memcmp (m1, m2, N)) return (NULL);
+  return (m1);
+
+}
Index: /trunk/Ohana/src/opihi/pcontrol/pclient.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/pclient.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/pclient.c	(revision 3203)
@@ -4,22 +4,30 @@
 # define PCLIENT_PROMPT "pclient:"
 
-# define DOWN  0
-# define HUNG -1
-
 int PclientCommand (Host *host, char *command, IOBuffer *buffer) {
 
-  /* send command to client */
-  status = write (host[0].stdin, command, strlen(command));
-  if ((status == -1) && (errno == EPIPE)) return (DOWN);
+  int i;
+  int status;
+  char *line;
+  
+  /* send command to client (adding on \n) */
+  ALLOCATE (line, char, MAX (1, strlen(command)));
+  sprintf (line, "%s\n", command);
+  status = write (host[0].stdin, line, strlen(line));
+  free (line);
+
+  /* is pipe still open? */
+  if ((status == -1) && (errno == EPIPE)) return (PCLIENT_DOWN);
   
   /* watch for response - wait up to 1 second */
-  p = NULL;
+  line = NULL;
   status = -1;
-  for (i = 0; (i < PCLIENT_TIMEOUT) && (status == -1) && (p == NULL); i++) {
+  for (i = 0; (i < PCLIENT_TIMEOUT) && (status == -1) && (line == NULL); i++) {
     status = ReadtoIOBuffer (buffer, host[0].stdout);
-    p = memstr (buffer[0].buffer, PCLIENT_PROMPT, buffer[0].Nbuffer);
+    line = memstr (buffer[0].buffer, PCLIENT_PROMPT, buffer[0].Nbuffer);
   }
-  if (status ==  0) return (DOWN);
-  if (status == -1) return (HUNG);
-  return (TRUE);
+  if (status ==  0) return (PCLIENT_DOWN);
+  if (status == -1) return (PCLIENT_HUNG);
+  return (PCLIENT_GOOD);
 }
+
+/* memstr returns a view, not an allocated string : don't free */
Index: /trunk/Ohana/src/opihi/pcontrol/pcontrol.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 3203)
@@ -41,5 +41,4 @@
   InitOutfile ();
   InitPcontrol ();
-  InitChild ();
 
   rl_readline_name = opihi_name;
Index: /trunk/Ohana/src/opihi/pcontrol/rconnect.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/rconnect.c	(revision 3202)
+++ /trunk/Ohana/src/opihi/pcontrol/rconnect.c	(revision 3203)
@@ -10,6 +10,6 @@
   int i, stdin_fd[2], stdout_fd[2], stderr_fd[2], status;
   pid_t pid;
-  char command[0x100];
   char *p;
+  char **argv;
   IOBuffer buffer;
 
@@ -76,6 +76,6 @@
   status = -1;
   for (i = 0; (i < CONNECT_TIMEOUT) && (status == -1) && (p == NULL); i++) {
-    status = ReadtoIOBuffer (buffer, stdout_fd[0]);
-    p = memstr (buffer[0].buffer, "CONNECTED", buffer[0].Nbuffer);
+    status = ReadtoIOBuffer (&buffer, stdout_fd[0]);
+    p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer);
   }
   if (status == 0) goto pipe_error;
Index: /trunk/Ohana/src/opihi/pcontrol/status.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/status.c	(revision 3203)
+++ /trunk/Ohana/src/opihi/pcontrol/status.c	(revision 3203)
@@ -0,0 +1,60 @@
+# include "pcontrol.h"
+
+int status (int argc, char **argv) {
+
+  int i, Nobject;
+  Stack *stack;
+  Job   *job;
+
+  PrintJobStack (PCONTROL_JOB_PENDING);
+  PrintJobStack (PCONTROL_JOB_BUSY);
+  PrintJobStack (PCONTROL_JOB_DONE);
+  PrintJobStack (PCONTROL_JOB_EXIT);
+  PrintJobStack (PCONTROL_JOB_CRASH);
+
+  PrintHostStack (PCONTROL_HOST_OFF);
+  PrintHostStack (PCONTROL_HOST_DOWN);
+  PrintHostStack (PCONTROL_HOST_IDLE);
+  PrintHostStack (PCONTROL_HOST_BUSY);
+  PrintHostStack (PCONTROL_HOST_DONE);
+}
+
+PrintJobStack (int Nstack) {
+
+  int i, j, Nobject;
+  Stack *stack;
+  Job *job;
+
+  stack = GetJobStack (Nstack);
+  Nobject = stack[0].Nobject;
+  fprintf (stderr, "job stack %d:  %d objects\n", Nstack, Nobject);
+
+  for (i = 0; i < Nobject; i++) {
+    job = stack[0].object[i];
+    fprintf (stderr, "%d  %s  %d  ", i, job[0].hostname, job[0].argc);
+    for (j = 0; j < job[0].argc; j++) {
+      fprintf (stderr, "%s ", job[0].argv[j]);
+    }
+    PrintID (stderr, job[0].JobID);
+    fprintf (stderr, "\n");
+  }
+}
+
+PrintHostStack (int Nstack) {
+
+  int i, j, Nobject;
+  Stack *stack;
+  Host *host;
+
+  stack = GetHostStack (Nstack);
+  Nobject = stack[0].Nobject;
+  fprintf (stderr, "host stack %d:  %d objects\n", Nstack, Nobject);
+
+  for (i = 0; i < Nobject; i++) {
+    host = stack[0].object[i];
+    fprintf (stderr, "%d  %s  ", i, host[0].hostname);
+    PrintID (stderr, host[0].HostID);
+    fprintf (stderr, "\n");
+  }
+}
+