Index: trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 8296)
+++ trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 8424)
@@ -1,5 +1,14 @@
 # include "pcontrol.h"
+# define DEBUG 0
 
 static struct timeval lastlive = {0, 0};
+static int RunSystem = FALSE;
+
+int SetRunSystem (int state) {
+  int oldstate;
+  oldstate = RunSystem;
+  RunSystem = state;
+  return oldstate;
+}
 
 int CheckSystem () {
@@ -29,5 +38,5 @@
   } 
 
-  if (0) { 
+  if (DEBUG) { 
     Stack *stack;
     int Nidle, Ndown, Nbusy;
@@ -44,10 +53,65 @@
 }
 
+void *CheckSystem_Threaded (void *data) {
+
+  struct timeval now;
+  float dtime;
+
+  gprintInit ();
+
+  while (1) {
+    // stop here if the user-thread requests (no objects in flight) 
+    TestCheckPoint ();
+
+    // don't run the system checks if RunSystem is FALSE
+    if (!RunSystem) {
+      usleep (50000);
+      continue;
+    }
+
+    // we want to give each block a maximum allowed time
+    CheckIdleHosts(0.020); /* submit a new job */
+
+    CheckBusyJobs(0.020);  /* get job status */
+    CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
+    CheckKillJobs(0.020);  /* harvest job stdout/stderr */
+
+    CheckDoneHosts(0.020); /* reset the host */
+    CheckDownHosts(0.100); /* launch the host */
+
+    /* always allow at least one test */
+    /* most tests require about 2ms per host.  
+       CheckDoneJobs must depend on the size of the output buffer */
+
+    gettimeofday (&now, (void *) NULL);
+    dtime = DTIME (now, lastlive);
+    if (dtime > 1.0) {
+      CheckLiveHosts(0.040);
+      lastlive = now;
+    } 
+
+    if (DEBUG) { 
+      Stack *stack;
+      int Nidle, Ndown, Nbusy;
+      stack = GetHostStack (PCONTROL_HOST_IDLE);
+      Nidle = stack[0].Nobject;
+      stack = GetHostStack (PCONTROL_HOST_DOWN);
+      Ndown = stack[0].Nobject;
+      stack = GetHostStack (PCONTROL_HOST_BUSY);
+      Nbusy = stack[0].Nobject;
+      gprint (GP_ERR, "busy, idle, down: %2d %2d %2d\n", Nbusy, Nidle, Ndown);
+    }
+  }
+  return (NULL);
+}
+
 int CheckBusyJobs (float MaxDelay) {
 
   struct timeval start, stop;
   int i, Nobject;
-  Stack *stack;
+  Stack *hoststack;
+  Stack *jobstack;
   Job   *job;
+  Host  *host;
   float dtime;
 
@@ -65,6 +129,5 @@
   dtime = 0.0;
   for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
-    /* pull both job and host from their stacks */
-    /* XXX is the subject to the Dangerous Embrace? */
+    // pull both job and host from their stacks
     LockStack (hoststack);
     job = PullStackByLocation (jobstack, STACK_TOP);
@@ -73,12 +136,13 @@
       break;
     }
-    host = RemoveStackByID (hoststack, job[0].host[0].HostID);
+    host = (Host *) job[0].host;
+    RemoveStackByID (hoststack, host[0].HostID);
     UnlockStack (hoststack);
 
-    CheckBusyJob (job);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
-  if (0 && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
+    CheckBusyJob (job, host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
   return (TRUE);
 }
@@ -88,23 +152,34 @@
   struct timeval start, stop;
   int i, Nobject;
-  Stack *stack;
+  Stack *hoststack;
+  Stack *jobstack;
   Job   *job;
-  float dtime;
-
-  /* Loop through objects on the stack, no more than once. see note above */
-  stack = GetJobStack (PCONTROL_JOB_DONE);
-  Nobject = stack[0].Nobject;
-
-  /* always allow at least one test */
-  gettimeofday (&start, (void *) NULL);
-  dtime = 0.0;
-  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
-    job = PullStackByLocation (stack, STACK_TOP);
-    if (job == NULL) break;
-    CheckDoneJob (job);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
-  if (0 && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
+  Host  *host;
+  float dtime;
+
+  /* Loop through objects on the stack, no more than once. see note above */
+  hoststack = GetHostStack (PCONTROL_HOST_BUSY);
+  jobstack  = GetJobStack (PCONTROL_JOB_DONE);
+  Nobject   = jobstack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    LockStack (hoststack);
+    job = PullStackByLocation (jobstack, STACK_TOP);
+    if (job == NULL) {
+      UnlockStack (hoststack);
+      break;
+    }
+    host = (Host *) job[0].host;
+    RemoveStackByID (hoststack, host[0].HostID);
+    UnlockStack (hoststack);
+
+    CheckDoneJob (job, host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
   return (TRUE);
 }
@@ -114,23 +189,34 @@
   struct timeval start, stop;
   int i, Nobject;
-  Stack *stack;
+  Stack *hoststack;
+  Stack *jobstack;
   Job   *job;
-  float dtime;
-
-  /* Loop through objects on the stack, no more than once. see note above */
-  stack = GetJobStack (PCONTROL_JOB_KILL);
-  Nobject = stack[0].Nobject;
-
-  /* always allow at least one test */
-  gettimeofday (&start, (void *) NULL);
-  dtime = 0.0;
-  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
-    job = PullStackByLocation (stack, STACK_TOP);
-    if (job == NULL) break;
-    KillJob (job);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
-  if (0 && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
+  Host  *host;
+  float dtime;
+
+  /* Loop through objects on the stack, no more than once. see note above */
+  hoststack = GetHostStack (PCONTROL_HOST_BUSY);
+  jobstack = GetJobStack (PCONTROL_JOB_KILL);
+  Nobject = jobstack[0].Nobject;
+
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    LockStack (hoststack);
+    job = PullStackByLocation (jobstack, STACK_TOP);
+    if (job == NULL) {
+      UnlockStack (hoststack);
+      break;
+    }
+    host = (Host *) job[0].host;
+    RemoveStackByID (hoststack, host[0].HostID);
+    UnlockStack (hoststack);
+
+    KillJob (job, host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (DEBUG && (Nobject > 0)) gprint (GP_ERR, "checked %d of %d jobs\n", i, Nobject);
   return (TRUE);
 }
@@ -158,5 +244,5 @@
     dtime = DTIME (stop, start);
   }
-  if (0) gprint (GP_ERR, "checked %d hosts\n", i);
+  if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
   return (TRUE);
 }
@@ -180,4 +266,9 @@
     host = PullStackByLocation (stack, STACK_TOP);
     if (host == NULL) break;
+    if (host[0].markoff) {
+      host[0].markoff = FALSE;
+      OffHost (host);
+      return (TRUE);
+    }
     dtime = DTIME (host[0].nexttry, start);
     if (dtime > 0) {
@@ -189,5 +280,5 @@
     dtime = DTIME (stop, start);
   }
-  if (0) gprint (GP_ERR, "checked %d hosts\n", i);
+  if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
   return (TRUE);
 }
@@ -219,9 +310,9 @@
     dtime = DTIME (stop, start);
   }
-  if (0) gprint (GP_ERR, "checked %d hosts\n", i);
-  return (TRUE);
-}
-
-/* this is just a heartbeat check */
+  if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+/* this is just a heartbeat check (only IDLE hosts) */
 int CheckLiveHosts (float MaxDelay) {
 
@@ -246,19 +337,5 @@
     dtime = DTIME (stop, start);
   }
-  if (0) gprint (GP_ERR, "checked %d idle hosts\n", i);
-
-  /* Loop through objects on the stack, no more than once. see note above */
-  stack = GetHostStack (PCONTROL_HOST_BUSY);
-  Nobject = stack[0].Nobject;
-
-  dtime = 0.0;
-  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
-    host = PullStackByLocation (stack, STACK_TOP);
-    if (host == NULL) break;
-    CheckHost (host);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
-  if (0) gprint (GP_ERR, "checked %d busy hosts\n", i);
+  if (DEBUG) gprint (GP_ERR, "checked %d idle hosts\n", i);
   return (TRUE);
 }