Index: trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 4705)
+++ trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 4762)
@@ -1,121 +1,151 @@
 # include "pcontrol.h"
 
+static Npass = 0;
+
 int CheckSystem () {
 
-  CheckBusyJobs();
-  CheckDoneJobs();
-  CheckDoneHosts();
-  CheckIdleHosts();
-  CheckDownHosts();
-
-  /** add in different timescales for heartbeat? **/
-  CheckLiveHosts();
-
-  return (TRUE);
-}
-
-int CheckBusyJobs () {
-
+  /* we want to give each block a maximum allowed time */
+  CheckIdleHosts(0.020); /* submit a new job */
+
+  CheckBusyJobs(0.020);  /* get job status */
+  CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
+
+  CheckDoneHosts(0.020); /* reset the host */
+  CheckDownHosts(0.100); /* launch the host */
+
+  /* most tests require about 2ms per host.  
+     CheckDoneJobs must depend on the size of the output buffer */
+
+  /* this is a waste of cycles: no need to do this every loop */
+  if (Npass > 20) {
+    CheckLiveHosts(0.040);
+    Npass = 0;
+  } else {
+    Npass ++;
+  }
+
+  if (0) { 
+    Stack *stack;
+    int Nidle, Ndown, Nbusy;
+    stack = GetHostStack (PCONTROL_HOST_IDLE);
+    Nidle = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_DOWN);
+    Ndown = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_BUSY);
+    Nbusy = stack[0].Nobject;
+    fprintf (stderr, "busy, idle, down: %2d %2d %2d\n", Nbusy, Nidle, Ndown);
+  }
+
+  return (TRUE);
+}
+
+int CheckBusyJobs (float MaxDelay) {
+
+  struct timeval start, stop;
   int i, Nobject;
   Stack *stack;
   Job   *job;
+  float dtime;
 
   stack = GetJobStack (PCONTROL_JOB_BUSY);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     job = GetStack (stack, STACK_TOP);
     CheckBusyJob (job);
-  }
-  return (TRUE);
-}
-
-int CheckDoneJobs () {
-
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneJobs (float MaxDelay) {
+
+  struct timeval start, stop;
   int i, Nobject;
   Stack *stack;
   Job   *job;
+  float dtime;
 
   stack = GetJobStack (PCONTROL_JOB_DONE);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     job = GetStack (stack, STACK_TOP);
     CheckDoneJob (job);
-  }
-  return (TRUE);
-}
-
-int CheckDoneHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
 
   stack = GetHostStack (PCONTROL_HOST_DONE);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     host = GetStack (stack, STACK_TOP);
     CheckDoneHost (host);
-  }
-  return (TRUE);
-}
-
-/* this is just a heartbeat check */
-int CheckLiveHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
-
-  stack = GetHostStack (PCONTROL_HOST_IDLE);
-  Nobject = stack[0].Nobject;
-
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    CheckHost (host);
-  }
-
-  stack = GetHostStack (PCONTROL_HOST_BUSY);
-  Nobject = stack[0].Nobject;
-
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    CheckHost (host);
-  }
-  return (TRUE);
-}
-
-int CheckDownHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
-  struct timeval now;
-  float delta;
-
-  gettimeofday (&now, (void *) NULL);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckDownHosts (float MaxDelay) {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  struct timeval start, stop;
+  float dtime;
 
   stack = GetHostStack (PCONTROL_HOST_DOWN);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    delta = DTIME (host[0].nexttry, now);
-    if (delta > 0) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    dtime = DTIME (host[0].nexttry, start);
+    if (dtime > 0) {
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
     } else {
       StartHost (host);
     }
-  }
-  return (TRUE);
-}
-
-int CheckIdleHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckIdleHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
 
   /* check if there are any pending jobs, otherwise skip step */
@@ -126,8 +156,64 @@
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     host = GetStack (stack, STACK_TOP);
     CheckIdleHost (host);
-  }
-  return (TRUE);
-}
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+/* this is just a heartbeat check */
+int CheckLiveHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  gettimeofday (&start, (void *) NULL);
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d idle hosts\n", i);
+
+  stack = GetHostStack (PCONTROL_HOST_BUSY);
+  Nobject = stack[0].Nobject;
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d busy hosts\n", i);
+  return (TRUE);
+}
+
+/*
+
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check 4: %f seconds\n", dtime);
+
+  gettimeofday (&start, (void *) NULL);
+*/
+
+/** All of the CheckFooBar entries cycle though their respective queues, popping from the top and
+    pushing to the bottom.  if we stop before the loop is done there is no tendancy for bias because
+    we continue where we left off next round **/