Index: /trunk/Ohana/src/opihi/doc/pantasks.txt
===================================================================
--- /trunk/Ohana/src/opihi/doc/pantasks.txt	(revision 11897)
+++ /trunk/Ohana/src/opihi/doc/pantasks.txt	(revision 11898)
@@ -1,2 +1,27 @@
+
+- task spawning speed
+
+  I have been examining things which affect the speed of the pantasks
+  processing. I have learned some interesting things:
+
+  * pcontrol was being slammed with requests for status by pantasks.
+    this may account for Paul's controller hang-ups.  I have added a
+    long (500ms) sleep to the controller thread to limit the rate at
+    which controller checks are run
+
+  * adding even a small usleep to the task_thread or job_thread puts
+    them to sleep for a long time (>> 10ms).  it seems longer than the
+    linux time slicer.  I have removed sleeps from the task and job
+    threads.
+
+  * the job submit rate is apparently limited by two things:
+
+    * when the job is submitted (SubmitJob) the interaction with
+      the controller seems to take ~30ms or more.
+
+    * some thread (controller thread? main readline thread?) seems to
+      introduce timeouts which are very long (up to 100ms).  These
+      introduce bit delays if when they happen during the task_thread
+      loop. 
 
 - updates for queues:
Index: /trunk/Ohana/src/opihi/include/pantasks.h
===================================================================
--- /trunk/Ohana/src/opihi/include/pantasks.h	(revision 11897)
+++ /trunk/Ohana/src/opihi/include/pantasks.h	(revision 11898)
@@ -174,5 +174,5 @@
 Task *GetActiveTask ();
 void SetTaskTimer (struct timeval *timer);
-double GetTaskTimer (struct timeval start);
+double GetTaskTimer (struct timeval start, int verbose);
 void InitTaskTimers ();
 int TaskHash (char *input);
Index: /trunk/Ohana/src/opihi/pantasks/CheckJobs.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/CheckJobs.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/CheckJobs.c	(revision 11898)
@@ -11,11 +11,15 @@
   Queue *queue;
 
+  // int Ncheck;
+  // Ncheck = 0;
+
   /** test all jobs: ready to test?  finished? **/
   while ((job = NextJob ()) != NULL) {
+    // Ncheck ++;
 
     task = job[0].task;
 
     /* check poll period (ready to ask for status?) */
-    if (GetTaskTimer(job[0].last) < task[0].poll_period) continue;
+    if (GetTaskTimer(job[0].last, FALSE) < task[0].poll_period) continue;
 
     /* check current status */
@@ -135,5 +139,5 @@
      */
     if (job[0].mode == JOB_LOCAL) {
-      if (GetTaskTimer(job[0].start) < task[0].timeout_period) continue;
+      if (GetTaskTimer(job[0].start, FALSE) < task[0].timeout_period) continue;
       if (VerboseMode()) gprint (GP_LOG, "timeout on %s\n", task[0].name);
 
@@ -176,6 +180,10 @@
     /* reset polling clock */
     SetTaskTimer (&job[0].last);
-    if (TestElapsedCheck()) return (TRUE);
+    if (TestElapsedCheck()) {
+      // fprintf (stderr, "check %d jobs\n", Ncheck);
+      return (TRUE);
+    }
   }
+  // fprintf (stderr, "check %d jobs\n", Ncheck);
   return (TRUE);
 }
Index: /trunk/Ohana/src/opihi/pantasks/CheckTasks.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/CheckTasks.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/CheckTasks.c	(revision 11898)
@@ -6,4 +6,5 @@
   Task *task;
   int status;
+  struct timeval now;
 
   /** test all tasks: ready to test? ready to run? **/
@@ -13,5 +14,5 @@
 
     /* ready to test? : check exec period */
-    if (GetTaskTimer(task[0].last) < task[0].exec_period) continue;
+    if (GetTaskTimer(task[0].last, FALSE) < task[0].exec_period) continue;
 
     /* need to check if the current time is within valid/invalid periods */
@@ -25,8 +26,11 @@
     }
     if (task[0].NpendingMax && (task[0].Npending >= task[0].NpendingMax)) {
-	fprintf (stderr, "npending: %d, max npending: %d\n", task[0].Npending, task[0].NpendingMax);
+	// fprintf (stderr, "npending: %d, max npending: %d\n", task[0].Npending, task[0].NpendingMax);
 	gettimeofday (&task[0].last, (void *) NULL);
 	continue;
     }
+
+    // gettimeofday (&now, (void *) NULL);
+    // fprintf (stderr, "t0: %d %6d  - \n", now.tv_sec, now.tv_usec);
 
     /* ready to run? : run task.exec macro */
@@ -39,4 +43,7 @@
     }
 
+    // gettimeofday (&now, (void *) NULL);
+    // fprintf (stderr, "t1: %d %6d  - \n", now.tv_sec, now.tv_usec);
+
     /* check if there are errors with this task */
     if (!ValidateTask (task, TRUE)) { 
@@ -44,10 +51,19 @@
 	continue;
     }
+    
+    // gettimeofday (&now, (void *) NULL);
+    // fprintf (stderr, "t2: %d %6d  - \n", now.tv_sec, now.tv_usec);
 
     /* construct job from task */
     job = CreateJob (task);
 
+    // gettimeofday (&now, (void *) NULL);
+    // fprintf (stderr, "t3: %d %6d  - \n", now.tv_sec, now.tv_usec);
+
     /* execute job - XXX add status test */
     SubmitJob (job);
+
+    // fprintf (stderr, "nl: %d %6d  - ",
+    // task[0].last.tv_sec, task[0].last.tv_usec);
 
     /* reset timer on task (don't do this if Create/Submit fails) (why not??) */
@@ -55,4 +71,7 @@
     task[0].Njobs ++;
     task[0].Npending ++;
+
+    // fprintf (stderr, "%d %6d\n", 
+    // task[0].last.tv_sec, task[0].last.tv_usec);
 
     /* increment Nrun for inclusive ranges with Nmax */
Index: /trunk/Ohana/src/opihi/pantasks/ControllerOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 11898)
@@ -352,4 +352,9 @@
   }
   
+  /* for commands which don't return a prompt, don't look for one */
+  if (response == NULL) {
+      return (TRUE);
+  }
+
   /* watch for response - wait up to 1 second */
   line = NULL;
@@ -456,9 +461,9 @@
   sprintf (cmd, "quit");
   InitIOBuffer (&buffer, 0x100);
-  status = ControllerCommand (cmd, "", &buffer);
-  FreeIOBuffer (&buffer);
-
-  /* the quit command does not return a prompt, so we always 
-     get an error on the controller here */
+  status = ControllerCommand (cmd, NULL, &buffer);
+  FreeIOBuffer (&buffer);
+
+  /* the quit command does not return a prompt, 
+     check that the controller exited */
   StopController ();
   return (TRUE);
Index: /trunk/Ohana/src/opihi/pantasks/LocalJob.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/LocalJob.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/LocalJob.c	(revision 11898)
@@ -97,5 +97,5 @@
 	exit (1);
       }
-      job[0].dtime = GetTaskTimer (job[0].start);
+      job[0].dtime = GetTaskTimer (job[0].start, FALSE);
       break;
   }
Index: /trunk/Ohana/src/opihi/pantasks/TaskOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/TaskOps.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/TaskOps.c	(revision 11898)
@@ -547,5 +547,5 @@
 /*** task timer functions ***/
 
-double GetTaskTimer (struct timeval start) {
+double GetTaskTimer (struct timeval start, int verbose) {
 
   double dtime;
@@ -555,4 +555,10 @@
   dtime = DTIME (now, start);
   
+  if (verbose) {
+      fprintf (stderr, "tt: %d %6d  - %d %6d : %f\n", 
+	       now.tv_sec, now.tv_usec, 
+	       start.tv_sec, start.tv_usec, dtime);
+  }
+
   return (dtime);
 }
Index: /trunk/Ohana/src/opihi/pantasks/controller_threads.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/controller_threads.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/controller_threads.c	(revision 11898)
@@ -28,5 +28,7 @@
     CheckControllerOutput ();
     SerialThreadUnlock ();
-    usleep (10000); // allow other threads a chance to run
+    if (VerboseMode() == 2) fprintf (stderr, "C");
+    // fprintf (stderr, "**** C ****");
+    usleep (500000); // allow other threads a chance to run
   }
 }
Index: /trunk/Ohana/src/opihi/pantasks/input_threads.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/input_threads.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/input_threads.c	(revision 11898)
@@ -27,6 +27,6 @@
     CheckInputs ();
     SerialThreadUnlock ();
-    fprintf (stderr, "I");
-    usleep (10000); // allow other threads a chance to run
+    if (VerboseMode() == 2) fprintf (stderr, "I");
+    // usleep (10000); // allow other threads a chance to run
   }
 }
Index: /trunk/Ohana/src/opihi/pantasks/job_threads.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/job_threads.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/job_threads.c	(revision 11898)
@@ -27,5 +27,7 @@
     CheckJobs ();
     SerialThreadUnlock ();
-    usleep (10000); // allow other threads a chance to run
+    if (VerboseMode() == 2) fprintf (stderr, "J");
+    // fprintf (stderr, "J");
+    // usleep (10000); // allow other threads a chance to run
   }
 }
Index: /trunk/Ohana/src/opihi/pantasks/task_threads.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/task_threads.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/task_threads.c	(revision 11898)
@@ -27,5 +27,7 @@
     CheckTasks ();
     SerialThreadUnlock ();
-    usleep (10000); // allow other threads a chance to run
+    if (VerboseMode() == 2) fprintf (stderr, "T");
+    // fprintf (stderr, "T");
+    // usleep (1000); // allow other threads a chance to run
   }
 }
Index: /trunk/Ohana/src/opihi/pantasks/test/sleep.sh
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/test/sleep.sh	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/test/sleep.sh	(revision 11898)
@@ -6,10 +6,14 @@
 
   periods      -poll 0.1
-  periods      -exec 1.0
+  periods      -exec 0.2
   periods      -timeout 20
-  npending 2
+  npending 5
   
   stdout tmp.txt
   stderr tmp.txt
+
+  task.exec
+    echo "create command"
+  end
 
   # success
Index: /trunk/Ohana/src/opihi/pantasks/test/sleep2.sh
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/test/sleep2.sh	(revision 11898)
+++ /trunk/Ohana/src/opihi/pantasks/test/sleep2.sh	(revision 11898)
@@ -0,0 +1,39 @@
+
+$hostname = `hostname`
+
+controller exit true
+controller host add $hostname
+
+# a basic task which just runs 'sleep'
+task	       basic
+  command      sleep 1
+  host         anyhost
+
+  periods      -poll 0.01
+  periods      -exec 0.02
+  periods      -timeout 20
+  npending 100
+  
+  stdout tmp.txt
+  stderr tmp.txt
+
+  task.exec
+    # echo "create command"
+  end
+
+  # success
+  task.exit    0
+    $Npass ++
+    # echo done sleep
+  end
+
+  # default exit status
+  task.exit    default
+    echo       "basic: exit status: $EXIT"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    echo       "basic: timeout"
+  end
+end
Index: /trunk/Ohana/src/opihi/pantasks/verbose.c
===================================================================
--- /trunk/Ohana/src/opihi/pantasks/verbose.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pantasks/verbose.c	(revision 11898)
@@ -23,4 +23,8 @@
       return (TRUE);
     }
+    if (!strcasecmp (argv[1], "THREADS")) {
+      VERBOSE = 2;
+      return (TRUE);
+    }
     if (!strcasecmp (argv[1], "TOGGLE")) {
       VERBOSE = ~VERBOSE;
@@ -29,5 +33,5 @@
   }
 
-  gprint (GP_ERR, "USAGE: verbose (on/off/toggle)\n");
+  gprint (GP_ERR, "USAGE: verbose (on/off/threads/toggle)\n");
   return (FALSE);
 }
Index: /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 11897)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 11898)
@@ -88,7 +88,9 @@
 
     if ((RunLevel == PCONTROL_RUN_ALL) || (RunLevel == PCONTROL_RUN_REAP)) {
-      Njobchecks  += CheckBusyJobs(0.020);  /* get job status */
-      Njobchecks  += CheckDoneJobs(0.020);  /* harvest job stdout/stderr */
-      Njobchecks  += CheckKillJobs(0.020);  /* harvest job stdout/stderr */
+      Njobchecks  += CheckBusyJobs(0.020);  /* get job status (PCLIENT) */
+      TestCheckPoint ();
+      Njobchecks  += CheckDoneJobs(0.020);  /* harvest job stdout/stderr (!PCLIENT) */
+      TestCheckPoint ();
+      Njobchecks  += CheckKillJobs(0.020);  /* harvest job stdout/stderr (PCLIENT) */
       TestCheckPoint ();
     }
@@ -96,4 +98,5 @@
     if (RunLevel != PCONTROL_RUN_NONE) {
       Nhostchecks += CheckDoneHosts(0.020); /* reset the host */
+      TestCheckPoint ();
       Nhostchecks += CheckDownHosts(0.100); /* launch the host */
       TestCheckPoint ();
@@ -102,5 +105,5 @@
     if (RunLevel == PCONTROL_RUN_ALL) {
       // we want to give each block a maximum allowed time
-      Nhostchecks += CheckIdleHosts(0.020); /* submit a new job */
+      Nhostchecks += CheckIdleHosts(0.020); /* submit a new job (PCLIENT) */
       TestCheckPoint ();
     }