Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/Makefile
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/Makefile	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/Makefile	(revision 42897)
@@ -76,4 +76,5 @@
 $(SRC)/controller_jobstack.$(ARCH).o \
 $(SRC)/controller_hoststack.$(ARCH).o \
+$(SRC)/controller_hoststats.$(ARCH).o \
 $(SRC)/controller_machines.$(ARCH).o \
 $(SRC)/controller_parameters.$(ARCH).o \
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller.c	(revision 42897)
@@ -6,4 +6,5 @@
 int controller_jobstack   PROTO((int, char **));
 int controller_hoststack  PROTO((int, char **));
+int controller_hoststats  PROTO((int, char **));
 int controller_machines   PROTO((int, char **));
 int controller_parameters PROTO((int, char **));
@@ -23,4 +24,5 @@
   {1, "host",      controller_host,       "define host for controller"},
   {1, "hoststack", controller_hoststack,  "define host for controller"},
+  {1, "hoststats", controller_hoststats,  "show processing stats for each host"},
   // {1, "job",       controller_job,      "add jobs to controller"},
   {1, "jobstack",  controller_jobstack,   "check controller status"},
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststack.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststack.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststack.c	(revision 42897)
@@ -9,5 +9,5 @@
   if (argc != 2) {
     gprint (GP_ERR, "USAGE: controller hoststack (hoststack)\n");
-    gprint (GP_ERR, "       (hoststack) : idle, busy, done, down, off\n");
+    gprint (GP_ERR, "       (hoststack) : all, idle, busy, done, down, off, resp\n");
     return (FALSE);
   }
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststats.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststats.c	(revision 42897)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pantasks/controller_hoststats.c	(revision 42897)
@@ -0,0 +1,35 @@
+# include "pantasks.h"
+
+int controller_hoststats (int argc, char **argv) {
+
+  int status;
+  char command[1024];
+  IOBuffer buffer;
+
+  if (argc != 2) {
+    gprint (GP_ERR, "USAGE: controller hoststats (stack)\n");
+    gprint (GP_ERR, "       (stack) : all, idle, busy, done, down, off, resp\n");
+    return (FALSE);
+  }
+
+  /* check if controller is running */
+  status = CheckControllerStatus ();
+  if (!status) {
+    gprint (GP_LOG, "controller is not active\n");
+    return (TRUE);
+  }
+
+  // XXX this has an error?  test this out...
+  sprintf (command, "hoststats %s", argv[1]);
+  InitIOBuffer (&buffer, 0x100);
+
+  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+
+  if (status) {
+    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
+  } else {
+    gprint (GP_LOG, "controller is not communicating\n");
+  }
+  FreeIOBuffer (&buffer);
+  return (TRUE);
+}
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 42897)
@@ -131,10 +131,10 @@
       CheckLiveHosts(0.040);
       // fprintf (stderr, "sleep a bit\n");
-      usleep (100000); // idle if no jobs are waiting
+      usleep (10000); // idle if no jobs are waiting
     } else {
       // if we only have busy jobs, pause a moment before trying again
       if (!Ndonejobs) {
 	// fprintf (stderr, "sleep a bit\n");
-	usleep (100000);
+	usleep (10000);
       }
     }
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/GetJobOutput.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/GetJobOutput.c	(revision 42897)
@@ -12,4 +12,7 @@
   int i, status;
   struct timespec request, remain;
+
+  struct timeval start;
+  gettimeofday (&start, (void *) NULL);
 
   ASSERT (command, "command missing");
@@ -30,5 +33,13 @@
   }
 
-  if (output[0].completed) return PCLIENT_GOOD;
+  if (output[0].completed) {
+    struct timeval stop;
+    gettimeofday (&stop, (void *) NULL);
+    
+    float dtime = DTIME (stop, start);
+    host[0].read_time += dtime;
+    host[0].read_N_GetJobOutput ++;
+    return PCLIENT_GOOD;
+  }
 
   // attempt to read the output->size bytes from the host 
@@ -42,8 +53,20 @@
       if (status == 0) {
 	  if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+	  struct timeval stop;
+	  gettimeofday (&stop, (void *) NULL);
+    
+	  float dtime = DTIME (stop, start);
+	  host[0].read_time += dtime;
+	  host[0].read_N_GetJobOutput ++;
 	  return PCLIENT_DOWN;
       }
       if (output[0].buffer.Nbuffer < output[0].size) {
 	  if (VerboseMode()) gprint (GP_ERR, "host %s still has data, keep trying\n", host[0].hostname);
+	  struct timeval stop;
+	  gettimeofday (&stop, (void *) NULL);
+    
+	  float dtime = DTIME (stop, start);
+	  host[0].read_time += dtime;
+	  host[0].read_N_GetJobOutput ++;
 	  return PCLIENT_HUNG;
       }
@@ -61,10 +84,33 @@
   if (status == 0) {
     if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname);
+
+    struct timeval stop;
+    gettimeofday (&stop, (void *) NULL);
+    
+    float dtime = DTIME (stop, start);
+    host[0].read_time += dtime;
+    host[0].read_N_GetJobOutput ++;
+
     return PCLIENT_DOWN;
   }
   if (line == NULL) {
     if (VerboseMode()) gprint (GP_ERR, "host %s not yet at prompt, keep trying\n", host[0].hostname);
+
+    struct timeval stop;
+    gettimeofday (&stop, (void *) NULL);
+    
+    float dtime = DTIME (stop, start);
+    host[0].read_time += dtime;
+    host[0].read_N_GetJobOutput ++;
+
     return PCLIENT_HUNG;
   }
+
+  struct timeval stop;
+  gettimeofday (&stop, (void *) NULL);
+    
+  float dtime = DTIME (stop, start);
+  host[0].read_time += dtime;
+  host[0].read_N_GetJobOutput ++;
 
   output[0].completed = TRUE;
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/HostOps.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/HostOps.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/HostOps.c	(revision 42897)
@@ -211,4 +211,9 @@
   host[0].response = NULL;
 
+  host[0].read_time = 0.0;
+  host[0].read_N_Pclient = 0;
+  host[0].read_N_GetJobOutput = 0;
+  host[0].N_jobs_total = 0;
+
   host[0].markoff  = FALSE;
   host[0].job      = NULL;
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/Makefile
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/Makefile	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/Makefile	(revision 42897)
@@ -51,4 +51,5 @@
 $(SRC)/jobstack.$(ARCH).o \
 $(SRC)/hoststack.$(ARCH).o \
+$(SRC)/hoststats.$(ARCH).o \
 $(SRC)/kill.$(ARCH).o \
 $(SRC)/pulse.$(ARCH).o \
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/PclientCommand.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/PclientCommand.c	(revision 42897)
@@ -45,4 +45,7 @@
   struct timespec request, remain;
 
+  struct timeval start;
+  gettimeofday (&start, (void *) NULL);
+
   ASSERT (response != NULL, "response missing");
   ASSERT (buffer != NULL, "buffer missing");
@@ -64,4 +67,12 @@
     if (status == -1) nanosleep (&request, &remain);
   }
+
+  struct timeval stop;
+  gettimeofday (&stop, (void *) NULL);
+
+  float dtime = DTIME (stop, start);
+  host[0].read_time += dtime;
+  host[0].read_N_Pclient ++;
+
   if (status ==  0) {
     gprint (GP_ERR, "pclient read returns 0 for %s\n", response);
@@ -69,10 +80,12 @@
   }
   if (line == NULL) {
-      // MARKTIME ("-- client hung (line NULL): %s : %f sec\n", host[0].hostname, dtime);
-      return (PCLIENT_HUNG);
+    // this is really a timeout: data was received, but not the full message, in the allotted time (0.1 sec),
+    // MARKTIME ("-- client hung (line NULL): %s : %f sec\n", host[0].hostname, dtime);
+    return (PCLIENT_HUNG);
   }
   if (status == -1) {
-      // MARKTIME ("-- client hung (status -1): %s : %f sec\n", host[0].hostname, dtime);
-      return (PCLIENT_HUNG);
+    // this is really a timeout: no response was received in the allotted time (0.1 sec),
+    // MARKTIME ("-- client hung (status -1): %s : %f sec\n", host[0].hostname, dtime);
+    return (PCLIENT_HUNG);
   }
 
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StackOps.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StackOps.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StackOps.c	(revision 42897)
@@ -52,4 +52,17 @@
 /* this code correctly handles the negative 'where' and Nobject == 0 */
 
+static int   Np_PushStack = 0;
+static float dT_PushStack = 0.0;
+
+void Stats_PushStack (int reset) {
+
+  gprint (GP_LOG, "CPS: Npass: %d, time: %f\n", Np_PushStack, dT_PushStack);
+
+  if (reset) {
+    Np_PushStack = 0;
+    dT_PushStack = 0.0;
+  }
+}
+
 /* push object on stack at given location */
 int PushStack (Stack *stack, int where, void *object, int id, char *name) {
@@ -60,4 +73,7 @@
   ASSERT (stack != NULL, "stack not set");
   LockStack (stack);
+
+  struct timeval start, stop;
+  gettimeofday (&start, (void *) NULL);
 
   if (where < 0) where += stack[0].Nobject + 1;
@@ -89,8 +105,26 @@
   stack[0].Nobject ++;
 
+  gettimeofday (&stop, (void *) NULL);
+  float dtime = DTIME (stop, start);
+  dT_PushStack += dtime;
+  Np_PushStack ++;
+
   UnlockStack (stack);
   return (TRUE);
 }
 
+static int   Np_PullStackLoc = 0;
+static float dT_PullStackLoc = 0.0;
+
+void Stats_PullStackLoc (int reset) {
+
+  gprint (GP_LOG, "CLS: Npass: %d, time: %f\n", Np_PullStackLoc, dT_PullStackLoc);
+
+  if (reset) {
+    Np_PullStackLoc = 0;
+    dT_PullStackLoc = 0.0;
+  }
+}
+
 /* get object from specified point in stack (negative == distance from end) */
 void *PullStackByLocation (Stack *stack, int where) {
@@ -102,4 +136,7 @@
   LockStack (stack);
 
+  struct timeval start, stop;
+  gettimeofday (&start, (void *) NULL);
+
   if (where < 0) where += stack[0].Nobject;
   if (where < 0) { 
@@ -114,4 +151,10 @@
   object = stack[0].object[where];
   RemoveStackEntry (stack, where);
+
+  gettimeofday (&stop, (void *) NULL);
+  float dtime = DTIME (stop, start);
+  dT_PullStackLoc += dtime;
+  Np_PullStackLoc ++;
+
   UnlockStack (stack); 
   return (object);
@@ -141,4 +184,17 @@
 }
 
+static int   Np_PullStack = 0;
+static float dT_PullStack = 0.0;
+
+void Stats_PullStack (int reset) {
+
+  gprint (GP_LOG, "CpS: Npass: %d, time: %f\n", Np_PullStack, dT_PullStack);
+
+  if (reset) {
+    Np_PullStack = 0;
+    dT_PullStack = 0.0;
+  }
+}
+
 /* get object from point in stack (negative == distance from end) */
 void *PullStackByID (Stack *stack, int id) {
@@ -150,4 +206,7 @@
   ASSERT (stack != NULL, "stack not set");
   LockStack (stack);
+
+  struct timeval start, stop;
+  gettimeofday (&start, (void *) NULL);
 
   for (i = 0; i < stack[0].Nobject; i++) {
@@ -157,7 +216,19 @@
     object = stack[0].object[i];
     RemoveStackEntry (stack, i);
+
+    gettimeofday (&stop, (void *) NULL);
+    float dtime = DTIME (stop, start);
+    dT_PullStack += dtime;
+    Np_PullStack ++;
+
     UnlockStack (stack); 
     return (object);
   }
+
+  gettimeofday (&stop, (void *) NULL);
+  float dtime = DTIME (stop, start);
+  dT_PullStack += dtime;
+  Np_PullStack ++;
+
   UnlockStack (stack); 
   return (NULL);
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StartJob.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StartJob.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/StartJob.c	(revision 42897)
@@ -62,4 +62,5 @@
       job[0].pid = -1;
       gettimeofday (&job[0].start, (void *) NULL);
+      host[0].N_jobs_total ++;
 
       if (VerboseMode()) gprint (GP_ERR, "started job on host %s\n", host[0].hostname);  
@@ -95,5 +96,6 @@
       PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-      gettimeofday (&job[0].start, NULL);
+      // XXX this time stamp overrides the original start of the command, in StartJob
+      // gettimeofday (&job[0].start, NULL);
       return (TRUE);
   }
@@ -122,5 +124,6 @@
       PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM);
       PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM);
-      gettimeofday (&job[0].start, NULL);
+      // XXX this time stamp overrides the original start of the command, in StartJob
+      // gettimeofday (&job[0].start, NULL);
       return (TRUE);
   }
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststack.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststack.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststack.c	(revision 42897)
@@ -9,5 +9,5 @@
   if (argc != 2) {
     gprint (GP_ERR, "USAGE: hoststack (hoststack)\n");
-    gprint (GP_ERR, "       (hoststack) : idle, busy, done, down, off\n");
+    gprint (GP_ERR, "       (hoststack) : all, idle, busy, done, down, off, resp\n");
     return (FALSE);
   }
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststats.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststats.c	(revision 42897)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/hoststats.c	(revision 42897)
@@ -0,0 +1,34 @@
+# include "pcontrol.h"
+
+int hoststats (int argc, char **argv) {
+
+  int i;
+  Stack *stack;
+  Host *host;
+
+  if (argc != 2) {
+    gprint (GP_ERR, "USAGE: hoststats (stack)\n");
+    gprint (GP_ERR, "       (stack) : all, idle, busy, done, down, off, resp\n");
+    return (FALSE);
+  }
+
+  /* select hoststack */
+  stack = GetHostStackByName (argv[1]);
+  if (stack == NULL) {
+    gprint (GP_ERR, "hoststack not found\n");
+    return (FALSE);
+  }
+
+  /* print list */
+  LockStack (stack);
+  gprint (GP_LOG, "Nhosts: %d\n", stack[0].Nobject);
+  for (i = 0; i < stack[0].Nobject; i++) {
+    host = stack[0].object[i];
+    gprint (GP_LOG, "%lld %s %d : %f %d %d %d\n", host[0].HostID, host[0].hostname, host[0].pid, host[0].read_time, host[0].read_N_Pclient, host[0].read_N_GetJobOutput, host[0].N_jobs_total);
+  }
+  UnlockStack (stack);
+
+  return (TRUE);
+}
+
+// Safe with PTHREAD_MUTEX_INITIALIZER lock
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/init.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/init.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/init.c	(revision 42897)
@@ -5,4 +5,5 @@
 int host        PROTO((int, char **));
 int hoststack   PROTO((int, char **));
+int hoststats   PROTO((int, char **));
 int job	        PROTO((int, char **));
 int jobstack    PROTO((int, char **));
@@ -26,4 +27,5 @@
   {1, "host",       host,       "add / delete / modify host"},
   {1, "hoststack",  hoststack,  "list hosts for a single stack"},
+  {1, "hoststats",  hoststats,  "list host statistics for a single stack"},
   {1, "job",        job,        "add job"},
   {1, "jobstack",   jobstack,   "list jobs for a single stack"},
Index: /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/pcstats.c
===================================================================
--- /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/pcstats.c	(revision 42896)
+++ /tags/ipp-unions-20250528/Ohana/src/opihi/pcontrol/pcstats.c	(revision 42897)
@@ -26,4 +26,8 @@
   Stats_CheckLiveHosts (Reset);
 
+  Stats_PushStack (Reset);
+  Stats_PullStack (Reset);
+  Stats_PullStackLoc (Reset);
+
   return (TRUE);