Index: trunk/Ohana/src/opihi/pcontrol/StopHosts.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 27592)
+++ trunk/Ohana/src/opihi/pcontrol/StopHosts.c	(revision 28158)
@@ -1,3 +1,10 @@
 # include "pcontrol.h"
+
+// we attempt to harvest the 'down' hosts in HarvestHost.  However, sometimes the
+// child is busy and does not exit in the timeout period.  we need to keep a list and
+// try again occasionally to free up the needed resources
+static int NUNHARVESTED = 0;
+static int Nunharvested = 0;
+static int *unharvested = NULL;
 
 void DownHost (Host *host) {
@@ -140,5 +147,5 @@
       switch (errno) {
 	case ECHILD:
-	  gprint (GP_ERR, "unknown PID, not a child proc\n");
+	  gprint (GP_ERR, "HarvestHost: unknown PID (%d), not a child proc\n", pid);
 	  gprint (GP_ERR, "did process already exit?  programming error?\n");
 	  break;
@@ -152,5 +159,6 @@
       
     case 0:
-      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung");
+      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung\n");
+      AddZombie(pid);
       break;
 
@@ -174,2 +182,99 @@
   return (TRUE);
 }
+
+int AddZombie(int pid) {
+
+  if (unharvested == NULL) {
+    NUNHARVESTED = 128;
+    ALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (unharvested, 0, NUNHARVESTED*sizeof(int));
+  }
+  unharvested[Nunharvested] = pid;
+
+  Nunharvested ++;
+  if (Nunharvested >= NUNHARVESTED) {
+    NUNHARVESTED += 128;
+    REALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (&unharvested[Nunharvested], 0, (NUNHARVESTED - Nunharvested)*sizeof(int));
+  }
+  return TRUE;
+}
+
+int DelZombies() {
+
+  int i, j;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  int *newlist = NULL;
+
+  ALLOCATE (newlist, int, NUNHARVESTED);
+  memset (newlist, 0, NUNHARVESTED*sizeof(int));
+
+  j = 0;
+  for (i = 0; i < NUNHARVESTED; i++) {
+    if (!unharvested[i]) continue;
+    newlist[j] = unharvested[i];
+    j++;
+  }
+  free (unharvested);
+  unharvested = newlist;
+  Nunharvested = j;
+  return TRUE;
+}
+
+int CheckZombies() {
+
+  int pid, i, result, waitstatus;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  for (i = 0; i < Nunharvested; i++) {
+    if (!unharvested[i]) continue;
+    pid = unharvested[i];
+    result = waitpid (pid, &waitstatus, WNOHANG);
+    switch (result) {
+      case -1:  /* error with waitpid */
+	switch (errno) {
+	  case ECHILD:
+	    gprint (GP_ERR, "CheckZombies: unknown PID (%d), not a child proc\n", pid);
+	    gprint (GP_ERR, "did process already exit?  programming error?\n");
+	    break;
+	  case EINTR:
+	  case EINVAL:
+	  default:
+	    perror ("unexpected error");
+	    ABORT ("CheckZombies impossible condition");
+	}
+	break;
+      
+      case 0:
+	if (VerboseMode()) gprint (GP_ERR, "CheckZombies: still waiting on %d\n", pid);
+	break;
+
+      default:
+	if (result != pid) {
+	  gprint (GP_ERR, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
+	  ABORT ("CheckZombies impossible condition");
+	}
+	
+	if (WIFEXITED(waitstatus)) {
+	  if (VerboseMode()) gprint (GP_ERR, "child exited with status %d\n", WEXITSTATUS(waitstatus));
+	}
+	if (WIFSIGNALED(waitstatus)) {
+	  if (VerboseMode()) gprint (GP_ERR, "child crashed with status %d\n", WTERMSIG(waitstatus));
+	}
+	if (WIFSTOPPED(waitstatus)) {
+	  ABORT ("waitpid returns 'stopped': programming error\n");
+	}
+	unharvested[i] = 0;
+	break;
+    }
+  }
+  DelZombies();
+  return (TRUE);
+}