Index: trunk/Ohana/src/opihi/pantasks/ControllerOps.c
===================================================================
--- trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 18085)
+++ trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 19125)
@@ -10,4 +10,43 @@
 static IOBuffer stderr_buffer;
 static int ControllerPID = 0;
+
+/* local static variables to track the controller host properties */
+static Host *hosts = NULL;
+static int Nhosts = 0;
+static int NHOSTS = 0;
+
+int AddHost (char *hostname, int max_threads) {
+
+  int N;
+
+  N = Nhosts;
+  Nhosts ++;
+
+  CHECK_REALLOCATE (hosts, Host, NHOSTS, Nhosts, 16);
+
+  hosts[N].hostname = strcreate (hostname);
+  hosts[N].max_threads = max_threads;
+  
+  return (TRUE);
+}
+
+int DeleteHost (char *hostname) {
+
+  int i, j;
+
+  for (i = 0; i < Nhosts; i++) {
+    if (strcmp (hosts[i].hostname, hostname)) continue;
+    
+    // delete this one
+    free (hosts[i].hostname);
+    for (j = i; j < Nhosts - 1; j++) {
+      hosts[j].hostname = hosts[j+1].hostname;
+      hosts[j].max_threads = hosts[j+1].max_threads;
+    }
+    Nhosts --;
+    return (TRUE);
+  }
+  return (FALSE);
+}
 
 /* test if the controller is running */
@@ -146,5 +185,9 @@
 
   /* is pipe still open? */
-  if ((status == -1) && (errno == EPIPE)) return (CONTROLLER_DOWN);
+  // XXX call StopController() here?
+  if ((status == -1) && (errno == EPIPE)) {
+    StopController ();
+    return (CONTROLLER_DOWN);
+  }
 
   /* read at least Nbytes, then watch for CONTROLLER_PROMPT */
@@ -158,5 +201,8 @@
     if (status == -1) nanosleep (&request, &remain);
   }
-  if (status ==  0) return (CONTROLLER_DOWN);
+  if (status ==  0) {
+    StopController ();
+    return (CONTROLLER_DOWN);
+  }
   if (status == -1) return (CONTROLLER_HUNG);
 
@@ -245,4 +291,9 @@
   if (VarConfig ("CONTROLLER", "%s", cmd) == NULL) strcpy (cmd, "pcontrol");
 
+  if (hosts == NULL) {
+    NHOSTS = 16;
+    ALLOCATE (hosts, Host, NHOSTS);
+  }
+
   bzero (stdin_fd,  2*sizeof(int));
   bzero (stdout_fd, 2*sizeof(int));
@@ -360,5 +411,8 @@
   if ((status == -1) && (errno == EPIPE)) {
     StopController ();
-    gprint (GP_ERR, "controller is down\n");
+    gprint (GP_ERR, "controller is down, restarting\n");
+    if (!RestartController ()) {
+      return (FALSE);
+    }
     return (FALSE);
   }
@@ -509,2 +563,32 @@
   return (TRUE);
 }
+
+int RestartController () {
+
+  int i, status;
+  char command[256];
+  IOBuffer buffer;
+
+  gprint (GP_ERR, "controller is down, restarting\n");
+  if (!StartController ()) {
+    gprint (GP_ERR, "failure to re-start pcontrol\n");
+    return (FALSE);
+  }
+
+  InitIOBuffer (&buffer, 0x100);
+
+  // XXX lock the host table? SerialThreadLock ();
+  fprintf (stderr, "controller is down, restarting\n");
+  for (i = 0; i < Nhosts; i++) {
+    fprintf (stderr, "controller host add %s -threads %d\n", hosts[i].hostname, hosts[i].max_threads);
+    snprintf (command, 256, "host add %s -threads %d\n", hosts[i].hostname, hosts[i].max_threads);
+    status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+  }
+  // SerialThreadUnlock ();
+
+  if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
+
+  FreeIOBuffer (&buffer);
+
+  return (TRUE);
+}