Index: /trunk/Ohana/src/opihi/pcontrol/CheckPoint.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckPoint.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckPoint.c	(revision 8546)
@@ -30,5 +30,5 @@
     }
     pthread_mutex_unlock (&client);
-    usleep (10000);
+    usleep (10000); // wait for client thread to set lock
   }
   // put in a timeout?  (client thread not spinning...)
Index: /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 8546)
@@ -55,6 +55,7 @@
 void *CheckSystem_Threaded (void *data) {
 
-  struct timeval now;
-  float dtime;
+  int Njobchecks, Nhostchecks, Nlivechecks;
+
+  Nlivechecks = 0;
 
   gprintInit ();
@@ -66,17 +67,20 @@
     // don't run the system checks if RunSystem is FALSE
     if (!RunSystem) {
-      usleep (50000);
+      usleep (50000); // idle if RunSystem is FALSE
       continue;
     }
 
+    Njobchecks = 0;
+    Nhostchecks = 0;
+
     // we want to give each block a maximum allowed time
-    CheckIdleHosts(0.020); /* submit a new job */
-
-    CheckBusyJobs(0.020);  /* get job status */
-    CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
-    CheckKillJobs(0.020);  /* harvest job stdout/stderr */
-
-    CheckDoneHosts(0.020); /* reset the host */
-    CheckDownHosts(0.100); /* launch the host */
+    Nhostchecks += CheckIdleHosts(0.020); /* submit a new job */
+
+    Njobchecks  += CheckBusyJobs(0.020);  /* get job status */
+    Njobchecks  += CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
+    Njobchecks  += CheckKillJobs(0.020);  /* harvest job stdout/stderr */
+
+    Nhostchecks += CheckDoneHosts(0.020); /* reset the host */
+    Nhostchecks += CheckDownHosts(0.100); /* launch the host */
 
     /* always allow at least one test */
@@ -84,9 +88,8 @@
        CheckDoneJobs must depend on the size of the output buffer */
 
-    gettimeofday (&now, (void *) NULL);
-    dtime = DTIME (now, lastlive);
-    if (dtime > 1.0) {
+    // there is nothing on the stacks.  test the hosts and wait a bit
+    if (!Njobchecks && !Nhostchecks) {
       CheckLiveHosts(0.040);
-      lastlive = now;
+      usleep (100000); // idle if no jobs are waiting
     } 
 
@@ -145,5 +148,5 @@
   }
   if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
-  return (TRUE);
+  return (i);
 }
 
@@ -182,5 +185,5 @@
   }
   if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
-  return (TRUE);
+  return (i);
 }
 
@@ -219,5 +222,5 @@
   }
   if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
-  return (TRUE);
+  return (i);
 }
 
@@ -245,5 +248,5 @@
   }
   if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
-  return (TRUE);
+  return (i);
 }
 
@@ -281,5 +284,5 @@
   }
   if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
-  return (TRUE);
+  return (i);
 }
 
@@ -294,5 +297,5 @@
   /* check if there are any pending jobs, otherwise skip step */
   stack = GetJobStack (PCONTROL_JOB_PENDING);
-  if (!stack[0].Nobject) return (TRUE);
+  if (!stack[0].Nobject) return (0);
 
   /* Loop through objects on the stack, no more than once. see note above */
@@ -311,5 +314,5 @@
   }
   if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
-  return (TRUE);
+  return (i);
 }
 
Index: /trunk/Ohana/src/opihi/pcontrol/Makefile
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/Makefile	(revision 8546)
@@ -23,5 +23,8 @@
 LFLAGS  =       $(LIBS) $(LIBS2) $(LIBS1) 
 
-# mana user commands and support functions ########################
+# to build the non-threaded version, remove -lpthread and comment out
+# the THREADED line in include/pcontrol.h
+
+# pcontrol user commands and support functions ########################
 
 funcs = \
Index: /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 8546)
@@ -20,5 +20,8 @@
 
   /* is pipe still open? */
-  if ((status == -1) && (errno == EPIPE)) return (PCLIENT_DOWN);
+  if ((status == -1) && (errno == EPIPE)) {
+    // gprint (GP_ERR, "pclient read gives pipe error for %s\n", command);
+    return (PCLIENT_DOWN);
+  }
   
   /* watch for response - wait up to 1 second */
@@ -30,5 +33,8 @@
     if (status == -1) nanosleep (&request, &remain);
   }
-  if (status ==  0) return (PCLIENT_DOWN);
+  if (status ==  0) {
+    // gprint (GP_ERR, "pclient read returns 0 for %s\n", command);
+    return (PCLIENT_DOWN);
+  }
   if (status == -1) return (PCLIENT_HUNG);
   /* gprint (GP_ERR, "buffer.buffer: %s\n", buffer[0].buffer); */
Index: /trunk/Ohana/src/opihi/pcontrol/StartHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StartHost.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/StartHost.c	(revision 8546)
@@ -14,5 +14,5 @@
   if (VarConfig ("SHELL", "%s", shell)     == NULL) strcpy (shell, "pclient");
 
-  gprint (GP_ERR, "starting host within thread %d\n", pthread_self());
+  if (VerboseMode()) gprint (GP_ERR, "starting host within thread %d\n", pthread_self());
 
   pid = rconnect (command, host[0].hostname, shell, stdio);
Index: /trunk/Ohana/src/opihi/pcontrol/StopHosts.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 8546)
@@ -17,19 +17,48 @@
 }
 
+/* for use by shutdown: force machines which are up to go down
+   wait for a little while for the client thread to take care 
+   of them
+*/
+   
 int DownHosts () {
 
+  int i, Nobject, Nwait;
   Stack *stack;
   Host  *host;
 
+  SetCheckPoint (); // ensure we can find the specified host
   stack = GetHostStack (PCONTROL_HOST_IDLE);
-  while ((host = PullStackByLocation (stack, STACK_BOTTOM)) != NULL) {
-    StopHost (host);
-    DownHost (host);
+  Nobject = stack[0].Nobject;
+  for (i = 0; i < Nobject; i++) {
+    host = PullStackByLocation (stack, STACK_TOP);
+    if (host == NULL) continue;
+    host[0].markoff = TRUE;
+    PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
   }
 
   stack = GetHostStack (PCONTROL_HOST_BUSY);
-  while ((host = PullStackByLocation (stack, STACK_BOTTOM)) != NULL) {
-    StopHost (host);
-    DownHost (host);
+  Nobject = stack[0].Nobject;
+  for (i = 0; i < Nobject; i++) {
+    host = PullStackByLocation (stack, STACK_TOP);
+    if (host == NULL) continue;
+    host[0].markoff = TRUE;
+    PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM);
+  }
+  ClearCheckPoint ();
+
+  Nwait = 0;
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  gprint (GP_ERR, "waiting for clients to exit");
+  while ((Nwait < 15) && stack[0].Nobject) {
+    gprint (GP_ERR, ".");
+    usleep (100000); // wait for clients to exit
+    Nwait++;
+  }
+  gprint (GP_ERR, "\n");
+  if (stack[0].Nobject) {
+    gprint (GP_ERR, "trouble shutting down all pclient instances: %d still alive\n", stack[0].Nobject);
+  } else {
+    gprint (GP_ERR, "done\n");
   }
   return (TRUE);
@@ -69,6 +98,6 @@
   int i, result, waitstatus;
 
-  gprint (GP_ERR, "harvesting within thread %d\n", pthread_self());
-  gprint (GP_ERR, "child process %d is down, wait for exit status\n", pid);
+  // gprint (GP_ERR, "harvesting within thread %d\n", pthread_self());
+  // gprint (GP_ERR, "child process %d is down, wait for exit status\n", pid);
   
   // Loop a few times waiting for child to exit
@@ -76,5 +105,5 @@
     result = waitpid (pid, &waitstatus, WNOHANG);
     if ((result == -1) && (errno == ECHILD)) {
-      usleep (10000);
+      usleep (10000); // wait for child to exit
       continue;
     } else {
@@ -109,11 +138,11 @@
       
       if (WIFEXITED(waitstatus)) {
-	gprint (GP_ERR, "child exited with status %d\n", WEXITSTATUS(waitstatus));
+	if (VerboseMode()) gprint (GP_ERR, "child exited with status %d\n", WEXITSTATUS(waitstatus));
       }
       if (WIFSIGNALED(waitstatus)) {
-	gprint (GP_ERR, "child crashed with status %d\n", WTERMSIG(waitstatus));
+	if (VerboseMode()) gprint (GP_ERR, "child crashed with status %d\n", WTERMSIG(waitstatus));
       }
       if (WIFSTOPPED(waitstatus)) {
-	gprint (GP_ERR, "waitpid returns 'stopped': programming error\n");
+	if (VerboseMode()) gprint (GP_ERR, "waitpid returns 'stopped': programming error\n");
 	exit (1);
       }
Index: /trunk/Ohana/src/opihi/pcontrol/pcontrol.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/pcontrol.c	(revision 8546)
@@ -25,7 +25,13 @@
   InitHostStacks ();
 
+  /* set global signal masks (these apply to all threads launched below) */
+  signal (SIGPIPE, gotsignal);
+  signal (SIGTSTP, gotsignal);
+  signal (SIGTTIN, gotsignal);
+
   rl_readline_name = opihi_name;
   rl_attempted_completion_function = command_completer;
 # ifdef THREADED
+  SetRunSystem (TRUE);
   pthread_create (&clientsThread, NULL, &CheckSystem_Threaded, NULL);
   rl_event_hook = NULL;
@@ -45,7 +51,4 @@
 
   /* ignore the history file.  to change this, see, eg, mana.c */
-  signal (SIGPIPE, gotsignal);
-  signal (SIGTSTP, gotsignal);
-  signal (SIGTTIN, gotsignal);
   return;
 }
Index: /trunk/Ohana/src/opihi/pcontrol/rconnect.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/rconnect.c	(revision 8545)
+++ /trunk/Ohana/src/opihi/pcontrol/rconnect.c	(revision 8546)
@@ -81,5 +81,5 @@
     status = ReadtoIOBuffer (&buffer, stdout_fd[0]);
     p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer);
-    usleep (10000);
+    usleep (10000); // wait for client to be connected
   }
   if (status == 0) goto connect_error;