Changeset 4762

Timestamp:

Aug 13, 2005, 10:30:13 AM (21 years ago)

Author:

eugene

Message:

working on speed issues

Location:

trunk/Ohana/src/opihi

Files:

• include/pcontrol.h (modified) (1 diff)
• include/psched.h (modified) (1 diff)
• pantasks/CheckController.c (modified) (4 diffs)
• pantasks/CheckJobs.c (modified) (1 diff)
• pantasks/CheckSystem.c (modified) (1 diff)
• pantasks/CheckTasks.c (modified) (1 diff)
• pantasks/ControllerOps.c (modified) (8 diffs)
• pantasks/controller_output.c (modified) (1 diff)
• pclient/ChildOps.c (modified) (2 diffs)
• pclient/init.c (modified) (1 diff)
• pclient/job.c (modified) (5 diffs)
• pclient/pclient.c (modified) (1 diff)
• pclient/reset.c (modified) (5 diffs)
• pclient/status.c (modified) (1 diff)
• pcontrol/CheckHost.c (modified) (1 diff)
• pcontrol/CheckIdleHost.c (modified) (3 diffs)
• pcontrol/CheckSystem.c (modified) (2 diffs)
• pcontrol/GetJobOutput.c (modified) (4 diffs)
• pcontrol/StopHosts.c (modified) (1 diff)
• pcontrol/pclient.c (modified) (3 diffs)
• pcontrol/pcontrol.c (modified) (1 diff)
• pcontrol/rconnect.c (modified) (4 diffs)

trunk/Ohana/src/opihi/include/pcontrol.h

@@ -124 +124 @@
 void DelHost (Host *host);
 int CheckSystem ();
-int CheckBusyJobs ();
-int CheckDoneJobs ();
-int CheckDoneHosts ();
-int CheckLiveHosts ();
-int CheckDownHosts ();
-int CheckIdleHosts ();
+int CheckBusyJobs (float delay);
+int CheckDoneJobs (float delay);
+int CheckDoneHosts (float delay);
+int CheckLiveHosts (float delay);
+int CheckDownHosts (float delay);
+int CheckIdleHosts (float delay);
 int CheckIdleHost (Host *host);
 void InitIDs ();

trunk/Ohana/src/opihi/include/psched.h

@@ -174 +174 @@
 int KillControllerJob (Job *job);
 int CheckControllerStatus ();
+int TestElapsedCheck ();
+

trunk/Ohana/src/opihi/pantasks/CheckController.c

@@ -3 +3 @@
 int CheckController () {
 
+  float dtime;
+  struct timeval start, stop;
   char *p, *q;
   int i, Njobs, status, JobID;
   Job *job;
   IOBuffer buffer;
+
+  gettimeofday (&start, (void *) NULL);
 
   /* get the list of completed jobs (exit / crash), update the job status */
@@ -32 +36 @@
   }
 
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check stack %f\n", dtime);
+  gettimeofday (&start, (void *) NULL);
+
   p = buffer.buffer;
-  for (i = 0; i < Njobs; i++) {
+  for (i = 0; (i < Njobs) && !TestElapsedCheck(); i++) {
     q = strchr (p, '\n');
     if (q == NULL) {
@@ -50 +59 @@
     CheckControllerJob (job);
   }
- 
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check %d jobs %f\n", i, dtime);
+
+  if (TestElapsedCheck()) return (TRUE);
+
   /*** check CRASH jobs ***/
   FlushIOBuffer (&buffer);
@@ -72 +86 @@
 
   p = buffer.buffer;
-  for (i = 0; i < Njobs; i++) {
+  for (i = 0; (i < Njobs) && !TestElapsedCheck(); i++) {
     q = strchr (p, '\n');
     if (q == NULL) {

trunk/Ohana/src/opihi/pantasks/CheckJobs.c

@@ -83 +83 @@
     /* reset polling clock */
     SetTaskTimer (&job[0].last);
+    if (TestElapsedCheck()) return (TRUE);
   }
   return (TRUE);

trunk/Ohana/src/opihi/pantasks/CheckSystem.c

@@ -1 +1 @@
 # include "psched.h"
+
+# define MAX_DELAY 0.1
+struct timeval start;
+static int Ncheck = 0;
 
 int CheckSystem () {
 
-  /* need to make a timeout: if this process lasts too long, 
-     we should return so the user can get input in.  this cannot
-     just be a return, because we need to hit each of the elements
-     eventually
-  */
-  struct timeval start, stop;
-  double dtime;
-  
   gettimeofday (&start, (void *) NULL);
-  CheckTasks ();
+
+  if (Ncheck < 5) {
+    CheckTasks ();
+    CheckJobs ();
+    Ncheck ++;
+  } else {
+    CheckController ();
+    CheckControllerOutput ();
+    Ncheck = 0;
+  }
+  return (TRUE);
+}
+
+int TestElapsedCheck () {
+
+  struct timeval stop;
+  float dtime;
+
   gettimeofday (&stop, (void *) NULL);
   dtime = DTIME (stop, start);
-  if (VerboseMode()) fprintf (stderr, "check 1: %f seconds\n", dtime);
+  if (dtime > MAX_DELAY) return (TRUE);
+  return (FALSE);
+}
 
-  gettimeofday (&start, (void *) NULL);
-  CheckJobs ();
-  gettimeofday (&stop, (void *) NULL);
-  dtime = DTIME (stop, start);
-  if (VerboseMode()) fprintf (stderr, "check 2: %f seconds\n", dtime);
-
-  gettimeofday (&start, (void *) NULL);
-  CheckController ();
-  gettimeofday (&stop, (void *) NULL);
-  dtime = DTIME (stop, start);
-  if (VerboseMode()) fprintf (stderr, "check 3: %f seconds\n", dtime);
-
-  gettimeofday (&start, (void *) NULL);
-  CheckControllerOutput ();
-  gettimeofday (&stop, (void *) NULL);
-  dtime = DTIME (stop, start);
-  if (VerboseMode()) fprintf (stderr, "check 4: %f seconds\n", dtime);
-
-  return (TRUE);
-}
+  /* do we really need to check the controller on every call?
+     or perhaps alternate?
+  */

trunk/Ohana/src/opihi/pantasks/CheckTasks.c

@@ -33 +33 @@
     gettimeofday (&task[0].last, (void *) NULL);
     task[0].Njobs ++;
+
+    if (TestElapsedCheck()) return (TRUE);
   }
   return (TRUE);

trunk/Ohana/src/opihi/pantasks/ControllerOps.c

@@ -1 +1 @@
 # include "psched.h"
 /* adding a new host can delay controller up to a second or so */
-# define CONTROLLER_TIMEOUT 200
+# define CONTROLLER_TIMEOUT 500
 # define CONNECT_TIMEOUT 300
 
@@ -18 +18 @@
 /* check job / get output if done */
 int CheckControllerJob (Job *job) {
-
+  struct timeval start, stop;
+  float dtime;
+
+  gettimeofday (&start, (void *) NULL);
   CheckControllerJobStatus (job);
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check job status %f\n", dtime);
 
   if ((job[0].state == JOB_EXIT) || (job[0].state == JOB_CRASH)) {
+    gettimeofday (&start, (void *) NULL);
     GetJobOutput ("stdout", job[0].pid, &job[0].stdout, job[0].stdout_size);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+    if (VerboseMode()) fprintf (stderr, "get stdout %f\n", dtime);
+
     GetJobOutput ("stderr", job[0].pid, &job[0].stderr, job[0].stderr_size);
     DeleteControllerJob (job);
@@ -97 +108 @@
   int i, status, Nstart;
   char *line;
+  struct timespec request, remain;
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
 
   /* flush any earlier messages */
@@ -117 +133 @@
       line = memstr (buffer[0].buffer, CONTROLLER_PROMPT, buffer[0].Nbuffer);
     }
-    if (status == -1) usleep (10000);
+    if (status == -1) nanosleep (&request, &remain);
   }
   if (status ==  0) return (CONTROLLER_DOWN);
@@ -256 +272 @@
 
   /* try to get evidence connection is alive - wait upto a few seconds */
+  /* connection is likely slow; don't bother with nanosleep here */
   p = NULL;
   status = -1;
@@ -300 +317 @@
 int ControllerCommand (char *cmd, char *response, IOBuffer *buffer) {
 
-  int i;
-  int status;
+  int i, status;
   char *line;
+  struct timespec request, remain;
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
 
   ReadtoIOBuffer (buffer, stdout_cntl);
@@ -323 +344 @@
     status = ReadtoIOBuffer (buffer, stdout_cntl);
     line = memstr (buffer[0].buffer, response, buffer[0].Nbuffer);
-    if (status == -1) usleep (10000);
+    if (status == -1) nanosleep (&request, &remain);
   }
   if (status ==  0) {
@@ -341 +362 @@
     bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
   }
-  if (VerboseMode()) fprintf (stderr, "message received\n");
+  if (VerboseMode()) fprintf (stderr, "message received, %d cycles\n", i);
   return (TRUE);
 }

trunk/Ohana/src/opihi/pantasks/controller_output.c

@@ -8 +8 @@
   }
 
+  CheckControllerOutput ();
   PrintControllerOutput ();
   return (TRUE);

trunk/Ohana/src/opihi/pclient/ChildOps.c

@@ -29 +29 @@
   struct timeval now;
 
-  /* this is really lame : check if we are calling too quickly */
+  /* this is really lame : check if we are calling too quickly 
+     this is unneeded: pclient.c rl_keyboard_input_timeout limits
+     the rate 
   gettimeofday (&now, NULL);
   dtime = DTIME (now, last);
@@ -36 +38 @@
   if (dtime > 0.01) Nbad = 0;
   if (Nbad > 10) exit (2);
+  */
 
   /* this is a bit lame : we must exit if calling process exits */

trunk/Ohana/src/opihi/pclient/init.c

@@ -16 +16 @@
   {"stdout",    stdout_pclient,   "get stdout buffer"},
   {"stderr",    stderr_pclient,   "get stderr buffer"},
-  {"version",     version,      "show version information"},
+  {"version",   version,      "show version information"},
 }; 
 

trunk/Ohana/src/opihi/pclient/job.c

@@ -8 +8 @@
   if (argc < 2) {
     fprintf (stderr, "USAGE: job (arg0) (arg1) ... (argN)\n");
-    fprintf (stdout, "STATUS %d\n", -2);
+    fprintf (GetOutfile(), "STATUS %d\n", -2);
     return (FALSE);
   }
@@ -14 +14 @@
   if (ChildStatus != PCLIENT_NONE) {
     fprintf (stderr, "need to clear existing child\n");
-    fprintf (stdout, "STATUS %d\n", -3);
+    fprintf (GetOutfile(), "STATUS %d\n", -3);
     return (FALSE);
   }
@@ -31 +31 @@
   pid = fork ();
   if (!pid) { /* must be child process */
-    fprintf (stderr, "starting child process %s...\n", targv[0]);
+    /* fprintf (stderr, "starting child process %s...\n", targv[0]); */
 
     /* close the other ends of the pipes */
@@ -73 +73 @@
   ChildPID = pid;
 
-  fprintf (stdout, "STATUS %d\n", ChildPID);
+  fprintf (GetOutfile(), "STATUS %d\n", ChildPID);
   return (TRUE);
 
@@ -85 +85 @@
   if (child_stderr_fd[1] != 0) close (child_stderr_fd[1]);
 
-  fprintf (stdout, "STATUS %d\n", -1);
+  fprintf (GetOutfile(), "STATUS %d\n", -1);
   return (FALSE);
 }

trunk/Ohana/src/opihi/pclient/pclient.c

@@ -20 +20 @@
   rl_attempted_completion_function = command_completer;
   rl_event_hook = CheckChild;
+  rl_set_keyboard_input_timeout (1000); 
 
   set_str_variable ("HISTORY", opihi_history);

trunk/Ohana/src/opihi/pclient/reset.c

@@ -4 +4 @@
 
   int i, result, waitstatus;
+  struct timespec request, remain;
 
   if (argc != 1) {
     fprintf (stderr, "USAGE: reset\n");
-    fprintf (stdout, "STATUS -1\n");
+    fprintf (GetOutfile(), "STATUS -1\n");
     return (FALSE);
   }
@@ -13 +14 @@
   if (ChildStatus == PCLIENT_NONE) {
     fprintf (stderr, "no child process, cannot reset\n");
-    fprintf (stdout, "STATUS 2\n");
+    fprintf (GetOutfile(), "STATUS 2\n");
     return (TRUE);
   }
 
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
   if (ChildStatus == PCLIENT_BUSY) {
-    /* send SIGTERM signal to job */
+    /* job is still running, send SIGTERM signal to job */
     kill (ChildPID, SIGTERM);
-    result = 0;
-    for (i = 0; (i < 10) && (result == 0); i++) {
-      usleep (10000);  /* 10 ms is min */
+    result = waitpid (ChildPID, &waitstatus, WNOHANG);
+    for (i = 0; (i < 500) && (result == 0); i++) {
+      nanosleep (&request, &remain);
       result = waitpid (ChildPID, &waitstatus, WNOHANG);
     }
     if (result) goto reset_job;
 
-    /* send SIGTERM signal to job */
+    /* job did not exit, send SIGKILL signal to job */
     kill (ChildPID, SIGKILL);
-    result = 0;
-    for (i = 0; (i < 10) && (result == 0); i++) {
-      usleep (10000);  /* 10 ms is min */
+    result = waitpid (ChildPID, &waitstatus, WNOHANG);
+    for (i = 0; (i < 500) && (result == 0); i++) {
+      nanosleep (&request, &remain);
       result = waitpid (ChildPID, &waitstatus, WNOHANG);
     }
@@ -38 +43 @@
     /* total failure, don't reset */
     fprintf (stderr, "child process %d is hung, cannot reset\n", ChildPID);
-    fprintf (stdout, "STATUS 0\n");
+    fprintf (GetOutfile(), "STATUS 0\n");
     return (FALSE);
   }
@@ -57 +62 @@
   ChildExitStatus = 0;
 
-  fprintf (stdout, "STATUS 1\n");
+  fprintf (GetOutfile(), "STATUS 1\n");
   return (TRUE);
 }
@@ -68 +73 @@
 */
 
+/* the linux kernel timer sticks in a 10ms lag between kill and the harvest */

trunk/Ohana/src/opihi/pclient/status.c

@@ -15 +15 @@
   if (ChildStatus == PCLIENT_CRASH) strcpy (status_string, "CRASH");
 
-  fprintf (stdout, "STATUS %s\n", status_string);
-  fprintf (stdout, "EXITST %d\n", ChildExitStatus);
-  fprintf (stdout, "STDOUT %d\n", child_stdout.Nbuffer);
-  fprintf (stdout, "STDERR %d\n", child_stderr.Nbuffer);
+  fprintf (GetOutfile(), "STATUS %s\n", status_string);
+  fprintf (GetOutfile(), "EXITST %d\n", ChildExitStatus);
+  fprintf (GetOutfile(), "STDOUT %d\n", child_stdout.Nbuffer);
+  fprintf (GetOutfile(), "STDERR %d\n", child_stderr.Nbuffer);
+
+  set_str_variable ("JOBSTATUS", status_string);
   return (TRUE);
 }

trunk/Ohana/src/opihi/pcontrol/CheckHost.c

@@ -40 +40 @@
 
     default:
-      if (VerboseMode()) fprintf (stderr, "host %s is alive\n", host[0].hostname);
       PutHost (host, host[0].stack, STACK_BOTTOM);
       FreeIOBuffer (&buffer);

trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c

@@ -8 +8 @@
 
   stack = GetJobStack (PCONTROL_JOB_PENDING);
-
+  
   /* look for first NEEDHOST matching this host */
   for (i = 0; i < stack[0].Nobject; i++) {
@@ -26 +26 @@
   for (i = 0; i < stack[0].Nobject; i++) {
     job = (Job *) stack[0].object[i];
+    /*** this currently never runs WANTHOST jobs on any other machines ***/
     if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
     if (job[0].hostname == NULL) {
@@ -50 +51 @@
 }
 
-/** note : host and job popped off stacks **/
-
-/** elsewhere, we will check for timeout on the job and 
-    migrate to ANYHOST from WANTHOST or move to 
-    a failure condition for NEEDHOST 
-**/
+/** note : host and job popped off IDLE and PENDING stacks, 
+    unless no job is available **/

trunk/Ohana/src/opihi/pcontrol/CheckSystem.c

@@ -1 +1 @@
 # include "pcontrol.h"
 
+static Npass = 0;
+
 int CheckSystem () {
 
-  CheckBusyJobs();
-  CheckDoneJobs();
-  CheckDoneHosts();
-  CheckIdleHosts();
-  CheckDownHosts();
-
-  /** add in different timescales for heartbeat? **/
-  CheckLiveHosts();
-
-  return (TRUE);
-}
-
-int CheckBusyJobs () {
-
+  /* we want to give each block a maximum allowed time */
+  CheckIdleHosts(0.020); /* submit a new job */
+
+  CheckBusyJobs(0.020);  /* get job status */
+  CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
+
+  CheckDoneHosts(0.020); /* reset the host */
+  CheckDownHosts(0.100); /* launch the host */
+
+  /* most tests require about 2ms per host.  
+     CheckDoneJobs must depend on the size of the output buffer */
+
+  /* this is a waste of cycles: no need to do this every loop */
+  if (Npass > 20) {
+    CheckLiveHosts(0.040);
+    Npass = 0;
+  } else {
+    Npass ++;
+  }
+
+  if (0) { 
+    Stack *stack;
+    int Nidle, Ndown, Nbusy;
+    stack = GetHostStack (PCONTROL_HOST_IDLE);
+    Nidle = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_DOWN);
+    Ndown = stack[0].Nobject;
+    stack = GetHostStack (PCONTROL_HOST_BUSY);
+    Nbusy = stack[0].Nobject;
+    fprintf (stderr, "busy, idle, down: %2d %2d %2d\n", Nbusy, Nidle, Ndown);
+  }
+
+  return (TRUE);
+}
+
+int CheckBusyJobs (float MaxDelay) {
+
+  struct timeval start, stop;
   int i, Nobject;
   Stack *stack;
   Job   *job;
+  float dtime;
 
   stack = GetJobStack (PCONTROL_JOB_BUSY);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     job = GetStack (stack, STACK_TOP);
     CheckBusyJob (job);
-  }
-  return (TRUE);
-}
-
-int CheckDoneJobs () {
-
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneJobs (float MaxDelay) {
+
+  struct timeval start, stop;
   int i, Nobject;
   Stack *stack;
   Job   *job;
+  float dtime;
 
   stack = GetJobStack (PCONTROL_JOB_DONE);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     job = GetStack (stack, STACK_TOP);
     CheckDoneJob (job);
-  }
-  return (TRUE);
-}
-
-int CheckDoneHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0 && (Nobject > 0)) fprintf (stderr, "checked %d of %d jobs\n", i, Nobject);
+  return (TRUE);
+}
+
+int CheckDoneHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
 
   stack = GetHostStack (PCONTROL_HOST_DONE);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     host = GetStack (stack, STACK_TOP);
     CheckDoneHost (host);
-  }
-  return (TRUE);
-}
-
-/* this is just a heartbeat check */
-int CheckLiveHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
-
-  stack = GetHostStack (PCONTROL_HOST_IDLE);
-  Nobject = stack[0].Nobject;
-
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    CheckHost (host);
-  }
-
-  stack = GetHostStack (PCONTROL_HOST_BUSY);
-  Nobject = stack[0].Nobject;
-
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    CheckHost (host);
-  }
-  return (TRUE);
-}
-
-int CheckDownHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
-  struct timeval now;
-  float delta;
-
-  gettimeofday (&now, (void *) NULL);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckDownHosts (float MaxDelay) {
+
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  struct timeval start, stop;
+  float dtime;
 
   stack = GetHostStack (PCONTROL_HOST_DOWN);
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
-    host = GetStack (stack, STACK_TOP);
-    delta = DTIME (host[0].nexttry, now);
-    if (delta > 0) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    dtime = DTIME (host[0].nexttry, start);
+    if (dtime > 0) {
       PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM);
     } else {
       StartHost (host);
     }
-  }
-  return (TRUE);
-}
-
-int CheckIdleHosts () {
-
-  int i, Nobject;
-  Stack *stack;
-  Host  *host;
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+int CheckIdleHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
 
   /* check if there are any pending jobs, otherwise skip step */
@@ -126 +156 @@
   Nobject = stack[0].Nobject;
 
-  for (i = 0; i < Nobject; i++) {
+  /* always allow at least one test */
+  gettimeofday (&start, (void *) NULL);
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
     host = GetStack (stack, STACK_TOP);
     CheckIdleHost (host);
-  }
-  return (TRUE);
-}
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d hosts\n", i);
+  return (TRUE);
+}
+
+/* this is just a heartbeat check */
+int CheckLiveHosts (float MaxDelay) {
+
+  struct timeval start, stop;
+  int i, Nobject;
+  Stack *stack;
+  Host  *host;
+  float dtime;
+
+  stack = GetHostStack (PCONTROL_HOST_IDLE);
+  Nobject = stack[0].Nobject;
+
+  gettimeofday (&start, (void *) NULL);
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d idle hosts\n", i);
+
+  stack = GetHostStack (PCONTROL_HOST_BUSY);
+  Nobject = stack[0].Nobject;
+
+  dtime = 0.0;
+  for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) {
+    host = GetStack (stack, STACK_TOP);
+    CheckHost (host);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+  if (0) fprintf (stderr, "checked %d busy hosts\n", i);
+  return (TRUE);
+}
+
+/*
+
+  gettimeofday (&stop, (void *) NULL);
+  dtime = DTIME (stop, start);
+  if (VerboseMode()) fprintf (stderr, "check 4: %f seconds\n", dtime);
+
+  gettimeofday (&start, (void *) NULL);
+*/
+
+/** All of the CheckFooBar entries cycle though their respective queues, popping from the top and
+    pushing to the bottom.  if we stop before the loop is done there is no tendancy for bias because
+    we continue where we left off next round **/

trunk/Ohana/src/opihi/pcontrol/GetJobOutput.c

@@ -1 +1 @@
 # include "pcontrol.h"
-# define PCLIENT_TIMEOUT 20
+# define PCLIENT_TIMEOUT 500
 
 /* we read Nbytes from the host, then watch for the prompt */ 
@@ -7 +7 @@
   int i, status, Nstart;
   char *line;
+  struct timespec request, remain;
 
   /* flush any earlier messages */
@@ -12 +13 @@
   FlushIOBuffer (buffer);
   Nstart = buffer[0].Nbuffer;
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
 
   /* send cmd (stdout / stderr) */
@@ -27 +32 @@
       line = memstr (buffer[0].buffer, PCLIENT_PROMPT, buffer[0].Nbuffer);
     }
-    if (status == -1) usleep (10000);
+    if (status == -1) nanosleep (&request, &remain);
   }
   if (status ==  0) return (PCLIENT_DOWN);

trunk/Ohana/src/opihi/pcontrol/StopHosts.c

@@ -53 +53 @@
       exit (1);
   }
-  usleep (10000);
 
   /* check current child status */
+  /* I probably should loop a few time with max timeout larger than 10ms... */
+  usleep (10000);
   result = waitpid (host[0].pid, &waitstatus, WNOHANG);
   switch (result) {

trunk/Ohana/src/opihi/pcontrol/pclient.c

@@ -1 +1 @@
 # include "pcontrol.h"
-# define PCLIENT_TIMEOUT 20
+# define PCLIENT_TIMEOUT 500
 
 int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer) {
@@ -7 +7 @@
   int status;
   char *line;
+  struct timespec request, remain;
+
+  /* avoid blocking on read, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
 
   ReadtoIOBuffer (buffer, host[0].stdout);
@@ -23 +28 @@
     status = ReadtoIOBuffer (buffer, host[0].stdout);
     line = memstr (buffer[0].buffer, response, buffer[0].Nbuffer);
-    if (status == -1) usleep (10000);
+    if (status == -1) nanosleep (&request, &remain);
   }
   if (status ==  0) return (PCLIENT_DOWN);

trunk/Ohana/src/opihi/pcontrol/pcontrol.c

@@ -19 +19 @@
   rl_attempted_completion_function = command_completer;
   rl_event_hook = CheckSystem;
-  rl_set_keyboard_input_timeout (1000000); 
+  rl_set_keyboard_input_timeout (100000); 
 
   set_str_variable ("HISTORY", opihi_history);

trunk/Ohana/src/opihi/pcontrol/rconnect.c

@@ -1 +1 @@
 # include "pcontrol.h"
 
-# define CONNECT_TIMEOUT 300
+/* connection can take a while, allow up to 2 sec */
+# define CONNECT_TIMEOUT 200
 
 /* connect to host, start the shell: ssh hostname pclient -> command hostname shell
@@ -15 +16 @@
   char **argv;
   IOBuffer buffer;
+  struct timespec request, remain;
 
   bzero (stdin_fd,  2*sizeof(int));
@@ -79 +81 @@
     status = ReadtoIOBuffer (&buffer, stdout_fd[0]);
     p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer);
-    usleep (20000);
+    usleep (10000);
   }
   if (status == 0) goto connect_error;
@@ -101 +103 @@
   if (VerboseMode()) fprintf (stderr, "error while connecting\n");
 
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
   /* harvest the child process: kill & wait (< 100 ms) for exit */
   kill (pid, SIGKILL);
   result = waitpid (pid, &waitstatus, WNOHANG);
-  for (i = 0; (i < 10) && (result == 0); i++) {
-    usleep (10000);  /* 10 ms is min */
+  for (i = 0; (i < 50) && (result == 0); i++) {
+    nanosleep (&request, &remain);
     result = waitpid (pid, &waitstatus, WNOHANG);
   }