Index: trunk/Ohana/src/opihi/include/pcontrol.h
===================================================================
--- trunk/Ohana/src/opihi/include/pcontrol.h	(revision 28157)
+++ trunk/Ohana/src/opihi/include/pcontrol.h	(revision 28158)
@@ -273,4 +273,7 @@
 int    StopHostResponse (Host *host);
 int    HarvestHost (int pid);
+int    AddZombie(int pid);
+int    DelZombies();
+int    CheckZombies();
 
 /*** JobOps.c ***/
Index: trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 28157)
+++ trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 28158)
@@ -8,5 +8,4 @@
 static float MAX_WANTHOST_WAIT = 10.0;
 static float MAX_CONNECT_TIME = 36000.0;
-static FILE *logfile = NULL;
 
   /* if this host has been connected for too long, disconnect (will automatically reconnect) */
@@ -35,8 +34,4 @@
   struct timeval now;
   float dtime;
-
-  if (logfile == NULL) { 
-    logfile = fopen ("pcontrol.log", "w");
-  }
 
   ASSERT (host, "host not set");
@@ -69,5 +64,5 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -90,5 +85,5 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -111,5 +106,5 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -133,5 +128,5 @@
     if (!CheckMachineJobs (host, job)) continue;
 
-    if (logfile) fprintf (logfile, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    gprint (GP_ERR, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
Index: trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 28157)
+++ trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 28158)
@@ -107,4 +107,6 @@
       TestCheckPoint ();
       Nhostchecks += CheckDownHosts(0.100); /* launch the host */
+      TestCheckPoint ();
+      CheckZombies(); /* launch the host */
       TestCheckPoint ();
     }
Index: trunk/Ohana/src/opihi/pcontrol/StopHosts.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 28157)
+++ trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 28158)
@@ -1,3 +1,10 @@
 # include "pcontrol.h"
+
+// we attempt to harvest the 'down' hosts in HarvestHost.  However, sometimes the
+// child is busy and does not exit in the timeout period.  we need to keep a list and
+// try again occasionally to free up the needed resources
+static int NUNHARVESTED = 0;
+static int Nunharvested = 0;
+static int *unharvested = NULL;
 
 void DownHost (Host *host) {
@@ -140,5 +147,5 @@
       switch (errno) {
 	case ECHILD:
-	  gprint (GP_ERR, "unknown PID, not a child proc\n");
+	  gprint (GP_ERR, "HarvestHost: unknown PID (%d), not a child proc\n", pid);
 	  gprint (GP_ERR, "did process already exit?  programming error?\n");
 	  break;
@@ -152,5 +159,6 @@
       
     case 0:
-      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung");
+      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung\n");
+      AddZombie(pid);
       break;
 
@@ -174,2 +182,99 @@
   return (TRUE);
 }
+
+int AddZombie(int pid) {
+
+  if (unharvested == NULL) {
+    NUNHARVESTED = 128;
+    ALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (unharvested, 0, NUNHARVESTED*sizeof(int));
+  }
+  unharvested[Nunharvested] = pid;
+
+  Nunharvested ++;
+  if (Nunharvested >= NUNHARVESTED) {
+    NUNHARVESTED += 128;
+    REALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (&unharvested[Nunharvested], 0, (NUNHARVESTED - Nunharvested)*sizeof(int));
+  }
+  return TRUE;
+}
+
+int DelZombies() {
+
+  int i, j;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  int *newlist = NULL;
+
+  ALLOCATE (newlist, int, NUNHARVESTED);
+  memset (newlist, 0, NUNHARVESTED*sizeof(int));
+
+  j = 0;
+  for (i = 0; i < NUNHARVESTED; i++) {
+    if (!unharvested[i]) continue;
+    newlist[j] = unharvested[i];
+    j++;
+  }
+  free (unharvested);
+  unharvested = newlist;
+  Nunharvested = j;
+  return TRUE;
+}
+
+int CheckZombies() {
+
+  int pid, i, result, waitstatus;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  for (i = 0; i < Nunharvested; i++) {
+    if (!unharvested[i]) continue;
+    pid = unharvested[i];
+    result = waitpid (pid, &waitstatus, WNOHANG);
+    switch (result) {
+      case -1:  /* error with waitpid */
+	switch (errno) {
+	  case ECHILD:
+	    gprint (GP_ERR, "CheckZombies: unknown PID (%d), not a child proc\n", pid);
+	    gprint (GP_ERR, "did process already exit?  programming error?\n");
+	    break;
+	  case EINTR:
+	  case EINVAL:
+	  default:
+	    perror ("unexpected error");
+	    ABORT ("CheckZombies impossible condition");
+	}
+	break;
+      
+      case 0:
+	if (VerboseMode()) gprint (GP_ERR, "CheckZombies: still waiting on %d\n", pid);
+	break;
+
+      default:
+	if (result != pid) {
+	  gprint (GP_ERR, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
+	  ABORT ("CheckZombies impossible condition");
+	}
+	
+	if (WIFEXITED(waitstatus)) {
+	  if (VerboseMode()) gprint (GP_ERR, "child exited with status %d\n", WEXITSTATUS(waitstatus));
+	}
+	if (WIFSIGNALED(waitstatus)) {
+	  if (VerboseMode()) gprint (GP_ERR, "child crashed with status %d\n", WTERMSIG(waitstatus));
+	}
+	if (WIFSTOPPED(waitstatus)) {
+	  ABORT ("waitpid returns 'stopped': programming error\n");
+	}
+	unharvested[i] = 0;
+	break;
+    }
+  }
+  DelZombies();
+  return (TRUE);
+}
Index: trunk/Ohana/src/opihi/pcontrol/pcontrol.c.in
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/pcontrol.c.in	(revision 28157)
+++ trunk/Ohana/src/opihi/pcontrol/pcontrol.c.in	(revision 28158)
@@ -1,3 +1,5 @@
 # include "pcontrol.h"
+
+# define STDERR_FILE "pcontrol.log"
 
 # define opihi_name "PCONTROL"
@@ -24,4 +26,12 @@
   signal (SIGTSTP, gotsignal);
   signal (SIGTTIN, gotsignal);
+
+  // stdin / stdout are used for communication with pantasks.
+  // redirect stderr so various error messages are saved
+  stderr = freopen (STDERR_FILE, "a", stderr);
+  if (!stderr) {
+      fprintf (stdout, "failed to open %s for error output\n", STDERR_FILE);
+      exit (1);
+  }
 
   rl_readline_name = opihi_name;
Index: trunk/Ohana/src/opihi/pcontrol/test/machines.sh
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/test/machines.sh	(revision 28157)
+++ trunk/Ohana/src/opihi/pcontrol/test/machines.sh	(revision 28158)
@@ -6,5 +6,4 @@
   host add pikake
   host add pikake
-
   host add ipp022
   host add ipp022
@@ -26,2 +25,26 @@
 end
 
+macro load.hosts.zombie
+  parameters connect = 2.0
+
+  host add pikake
+  host add pikake
+  host add ipp022
+  host add ipp022
+  host add ipp022
+  host add ipp022
+
+  machines
+end
+
+macro load.jobs.zombie
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+end
+