Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckController.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckController.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckController.c	(revision 23484)
@@ -1,7 +1,6 @@
 # include "pantasks.h"
 
-static struct timeval start;
-void TimerMark ();
-float TimerElapsed (int reset);
+void TimerMark (struct timeval *start);
+float TimerElapsed (struct timeval *start, int reset);
 
 int CheckController () {
@@ -11,4 +10,5 @@
   Job *job;
   IOBuffer buffer;
+  struct timeval start;
 
   /* get the list of completed jobs (exit / crash), update the job status */
@@ -17,12 +17,10 @@
   /*** check EXIT jobs ***/
   InitIOBuffer (&buffer, 0x100);
-  // TimerMark ();
-  // status = ControllerCommand ("stop", CONTROLLER_PROMPT, &buffer); 
-  // if (VerboseMode()) gprint (GP_ERR, "stop controller %f\n", TimerElapsed(TRUE));
 
-  TimerMark ();
+  TimerMark (&start);
   FlushIOBuffer (&buffer);
+
   status = ControllerCommand ("jobstack exit", CONTROLLER_PROMPT, &buffer);
-  if (VerboseMode()) gprint (GP_ERR, "check exit stack %f\n", TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "check exit stack %f\n", TimerElapsed(&start, TRUE));
   if (!status) goto escape;
 
@@ -34,5 +32,5 @@
   status = sscanf (buffer.buffer, "%*s %d", &Njobs);
   if (status != 1) goto escape;
-  if (VerboseMode()) gprint (GP_ERR, "parse %d jobs on stack %f\n", Njobs, TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "parse %d jobs on stack %f\n", Njobs, TimerElapsed(&start, TRUE));
 
   p = buffer.buffer;
@@ -46,13 +44,18 @@
     status = sscanf (p, "%d", &JobID);
 
+    // the operations within this locked block only interact with the controller or
+    // modify the properties of the selected job
+    JobTaskLock();
     job = FindControllerJob (JobID);
     if (job == NULL) {
       gprint (GP_ERR, "misplaced job? %d not in EXIT job list\n", JobID);
+      JobTaskUnlock();
       continue;
     }
     /* this checks the individual job status, grabs stdout/stderr */
     CheckControllerJob (job);
+    JobTaskUnlock();
   }
-  if (VerboseMode()) gprint (GP_ERR, "clear %d exit jobs %f\n", i, TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "clear %d exit jobs %f\n", i, TimerElapsed(&start, TRUE));
 
   /*** check CRASH jobs ***/
@@ -67,5 +70,5 @@
   status = sscanf (buffer.buffer, "%*s %d", &Njobs);
   if (status != 1) goto escape;
-  if (VerboseMode()) gprint (GP_ERR, "check crash stack %f\n", TimerElapsed(TRUE)); 
+  if (VerboseMode()) gprint (GP_ERR, "check crash stack %f\n", TimerElapsed(&start, TRUE)); 
 
   p = buffer.buffer;
@@ -77,18 +80,22 @@
     }
     p = q + 1;
-    
     status = sscanf (p, "%d", &JobID);
+
+    // the operations within this locked block only interact with the controller or
+    // modify the properties of the selected job
+    JobTaskLock();
     job = FindControllerJob (JobID);
     if (job == NULL) {
       gprint (GP_ERR, "misplaced job? %d not in CRASH job list\n", JobID);
+      JobTaskUnlock();
       continue;
     }
     /* this checks the individual job status, grabs stdout/stderr */
     CheckControllerJob (job);
+    JobTaskUnlock();
   }
-  if (VerboseMode()) gprint (GP_ERR, "clear %d crash jobs %f\n", i, TimerElapsed(TRUE)); 
+  if (VerboseMode()) gprint (GP_ERR, "clear %d crash jobs %f\n", i, TimerElapsed(&start, TRUE)); 
 
   FlushIOBuffer (&buffer);
-  // status = ControllerCommand ("run", CONTROLLER_PROMPT, &buffer);
   FreeIOBuffer (&buffer);
   return (TRUE);
@@ -96,14 +103,13 @@
  escape:
   FlushIOBuffer (&buffer);
-  // status = ControllerCommand ("run", CONTROLLER_PROMPT, &buffer); 
   FreeIOBuffer (&buffer);
   return (FALSE);
 }
 
-void TimerMark () {
-    gettimeofday (&start, (void *) NULL);
+void TimerMark (struct timeval *start) {
+    gettimeofday (start, (void *) NULL);
 }
 
-float TimerElapsed (int reset) {
+float TimerElapsed (struct timeval *start, int reset) {
 
   float dtime;
@@ -111,6 +117,6 @@
 
   gettimeofday (&stop, (void *) NULL);
-  dtime = DTIME (stop, start);
-  if (reset) gettimeofday (&start, (void *) NULL);
+  dtime = DTIME (stop, start[0]);
+  if (reset) gettimeofday (start, (void *) NULL);
   return (dtime);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckJobs.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckJobs.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckJobs.c	(revision 23484)
@@ -18,4 +18,5 @@
   next_timeout = 1.0;
 
+  JobTaskLock();
   /** test all jobs: ready to test?  finished? **/
   while ((job = NextJob ()) != NULL) {
@@ -77,6 +78,6 @@
 	/* set taskarg variables */
 	for (i = 0; i < job[0].argc; i++) {
-	    sprintf (varname, "taskarg:%d", i);
-	    set_str_variable (varname, job[0].argv[i]);
+	  sprintf (varname, "taskarg:%d", i);
+	  set_str_variable (varname, job[0].argv[i]);
 	}
 	set_int_variable ("taskarg:n", job[0].argc);
@@ -84,6 +85,6 @@
 	/* set options variables */
 	for (i = 0; i < job[0].optc; i++) {
-	    sprintf (varname, "options:%d", i);
-	    set_str_variable (varname, job[0].optv[i]);
+	  sprintf (varname, "options:%d", i);
+	  set_str_variable (varname, job[0].optv[i]);
 	}
 	set_int_variable ("options:n", job[0].optc);
@@ -109,5 +110,10 @@
 	  if (VerboseMode()) gprint (GP_LOG, "job %s (%d) crash\n", task[0].name, job[0].JobID);
 	  if (task[0].crash != NULL) {
+	    // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+	    JobTaskUnlock();
+	    CommandLock();
 	    exec_loop (task[0].crash);
+	    CommandUnlock();
+	    JobTaskLock();
 	  }
 	}
@@ -133,5 +139,12 @@
 	    }
 	  }
-	  if (macro != NULL) exec_loop (macro);
+	  if (macro != NULL) {
+	    // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+	    JobTaskUnlock();
+	    CommandLock();
+	    exec_loop (macro);
+	    CommandUnlock();
+	    JobTaskLock();
+	  }
 	}
 
@@ -156,5 +169,5 @@
     /* check for timeout - (local jobs only) 
        we only check timeout after a poll (forces at least one poll)
-     */
+    */
     if (job[0].mode == JOB_LOCAL) {
       if (GetTaskTimer(job[0].start, FALSE) < task[0].timeout_period) { 
@@ -179,6 +192,6 @@
       /* set taskarg variables */
       for (i = 0; i < job[0].argc; i++) {
-	  sprintf (varname, "taskarg:%d", i);
-	  set_str_variable (varname, job[0].argv[i]);
+	sprintf (varname, "taskarg:%d", i);
+	set_str_variable (varname, job[0].argv[i]);
       }
       set_int_variable ("taskarg:n", job[0].argc);
@@ -193,5 +206,10 @@
       /* run task[0].timeout macro, if it exists */
       if (task[0].timeout != NULL) {
+	// we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+	JobTaskUnlock();
+	CommandLock();
 	exec_loop (task[0].timeout);
+	CommandUnlock();
+	JobTaskLock();
       }
 
@@ -205,4 +223,5 @@
   }
   // fprintf (stderr, "check %d jobs\n", Ncheck);
+  JobTaskUnlock();
   return (next_timeout);
 }
@@ -210,21 +229,21 @@
 /* 
 
-  job / task timeline:
-
-  task:
-  0           exec     
-  start       create
-  task clock  new job
-
-  job:
-  0           1xpoll     2xpoll     3xpoll
-  start       check      check      check 
-  job clock   status     status     status
-
-  .           .          .          timeout
-                                    run
-				    timeout
-
-  must be at least one poll before timeout 
-  (timeout >= poll)
+   job / task timeline:
+
+   task:
+   0           exec     
+   start       create
+   task clock  new job
+
+   job:
+   0           1xpoll     2xpoll     3xpoll
+   start       check      check      check 
+   job clock   status     status     status
+
+   .           .          .          timeout
+   run
+   timeout
+
+   must be at least one poll before timeout 
+   (timeout >= poll)
 */
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckPassword.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckPassword.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckPassword.c	(revision 23484)
@@ -2,4 +2,6 @@
 # define DEBUG 0
 
+// this static var is only used by InitPassword and CheckPassword below.
+// Both functions are only called by the main thread.
 static char PASSWORD[256];
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckTasks.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckTasks.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/CheckTasks.c	(revision 23484)
@@ -12,4 +12,5 @@
   next_timeout = 1.0;
 
+  JobTaskLock();
   /** test all tasks: ready to test? ready to run? **/
   while ((task = NextTask ()) != NULL) {
@@ -54,5 +55,10 @@
     /* ready to run? : run task.exec macro */
     if (task[0].exec != NULL) {
+      // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+      JobTaskUnlock();
+      CommandLock();
       status = exec_loop (task[0].exec);
+      CommandUnlock();
+      JobTaskLock();
       if (!status) {
 	continue;
@@ -94,4 +100,5 @@
     BumpTimeRanges (task[0].ranges, task[0].Nranges);
   }
+  JobTaskUnlock();
   return (next_timeout);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ControllerOps.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 23484)
@@ -12,4 +12,6 @@
 
 /* local static variables to track the controller host properties */
+/* these are used by AddHost and DeleteHost, and are only called by controller_host.c (clientThread) */
+/* or by RestartController : lock between these two? */
 static Host *hosts = NULL;
 static int Nhosts = 0;
@@ -31,4 +33,7 @@
 }
 
+// XXX possible race condition problem: if we delete a host while the controller
+// is being restarted.  to fix this, we need to keep the deletion in controller_thread,
+// but perhaps mark it in the client_thread?
 int DeleteHost (char *hostname) {
 
@@ -261,7 +266,14 @@
   }
 
+  // This function is called by the JobTaskThread via SubmitJob.  We need to unlock the
+  // JobTaskLock to avoid a dead lock with the JobTaskLock called in CheckController
+  JobTaskUnlock();
+  ControlLock(__func__);
   InitIOBuffer (&buffer, 0x100);
   status = ControllerCommand (cmd, CONTROLLER_PROMPT, &buffer);
   free (cmd);
+  ControlUnlock(__func__);
+  JobTaskLock();
+
 
   /* extract the job PID from the controller response */
@@ -411,5 +423,5 @@
   if ((status == -1) && (errno == EPIPE)) {
     StopController ();
-    gprint (GP_ERR, "controller is down (pipe closed), restarting\n");
+    fprintf (stderr, "controller is down (pipe closed), restarting\n");
     if (!RestartController ()) {
       return (FALSE);
@@ -434,5 +446,5 @@
     if (status ==  0) {
       StopController ();
-      gprint (GP_ERR, "controller is down (EOF), restarting\n");
+      fprintf (stderr, "controller is down (EOF), restarting\n");
       if (!RestartController ()) {
 	return (FALSE);
@@ -441,10 +453,10 @@
     }
     if (status == -1) {
-      gprint (GP_ERR, "controller is not responding (%d tries)\n", j);
+      fprintf (stderr, "controller is not responding (%d tries)\n", j);
       gwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, GP_ERR);
     }
   }
   if (status == -1) {
-    gprint (GP_ERR, "controller still not responding, giving up\n");
+    fprintf (stderr, "controller still not responding, giving up\n");
     return (FALSE);
   }
@@ -456,5 +468,5 @@
     bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
   }
-  /* if (VerboseMode()) gprint (GP_ERR, "message received, %d cycles\n", i); */
+  if (VerboseMode()) fprintf (stderr, "message received, %d cycles\n", i);
   return (TRUE);
 }
@@ -492,4 +504,31 @@
 }
 
+void PrintControllerBusyJobs () {
+
+  int status;
+  char command[1024];
+  IOBuffer buffer;
+
+  /* check if controller is running */
+  status = CheckControllerStatus ();
+  if (!status) {
+    return;
+  }
+
+  sprintf (command, "jobstack busy");
+  InitIOBuffer (&buffer, 0x100);
+
+  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+
+  if (status) {
+    gprint (GP_LOG, " jobs currently running remotely:\n");
+    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
+  } else {
+    gprint (GP_LOG, "controller is not responding\n");
+  }
+  FreeIOBuffer (&buffer);
+  return;
+}
+
 int PrintControllerOutput () {
 
@@ -581,5 +620,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  // XXX lock the host table? SerialThreadLock ();
+  // XXX lock the host table? no: that would risk a dead lock between client and controller threads:
   gprint (GP_ERR, "pcontrol restarted, reloading hosts\n");
   for (i = 0; i < Nhosts; i++) {
@@ -588,5 +627,4 @@
     status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
   }
-  // SerialThreadUnlock ();
 
   if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/InputQueue.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/InputQueue.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/InputQueue.c	(revision 23484)
@@ -1,3 +1,4 @@
 # include "pantasks.h"
+// we use fprintf for DEBUG statements to avoid deadlocking issues
 
 static int Ninputs = 0;
@@ -23,6 +24,8 @@
 void AddNewInput (char *input) {
 
+  // XXX define the InputMutex
   SerialThreadLock ();
-  if (DEBUG) gprint (GP_LOG, "adding a new input (%s)\n", input);
+
+  if (DEBUG) fprintf (stderr, "adding a new input (%s)\n", input);
   inputs[Ninputs] = input;
   Ninputs ++;
@@ -31,5 +34,5 @@
     REALLOCATE (inputs, char *, NINPUTS);
   }
-  if (DEBUG) gprint (GP_LOG, "done new input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "done new input (%s)\n", input);
   SerialThreadUnlock ();
 }
@@ -40,5 +43,7 @@
   int i, j;
 
-  if (DEBUG) gprint (GP_LOG, "deleting an input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "deleting an input (%s)\n", input);
+
+  // XXX lock here
   for (i = 0; i < Ninputs; i++) {
     if (inputs[i] == input) {
@@ -52,9 +57,11 @@
 	REALLOCATE (inputs, char *, NINPUTS);
       }
-      if (DEBUG) gprint (GP_LOG, "deleted an input\n");
+      // XXX unlock here
+      if (DEBUG) fprintf (stderr, "deleted an input\n");
       return TRUE;
     }
   }
   // did not find the input
+  // XXX unlock here
   return FALSE;
 }
@@ -66,12 +73,17 @@
   char *input, *line, *outline, tmp;
 
-  if (Ninputs < 1) return;
+  // XXX lock here
+  if (Ninputs < 1) {
+    // XXX unlock here
+    return;
+  }
 
   input = inputs[0];
-  if (DEBUG) gprint (GP_LOG, "got an input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "got an input (%s)\n", input);
   
   Nbytes = snprintf (&tmp, 0, "input %s", input);
   ALLOCATE (line, char, Nbytes + 1);
   snprintf (line, Nbytes + 1, "input %s", input);
+  // XXX unlock here
 
   status = command (line, &outline, TRUE);
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/JobOps.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/JobOps.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/JobOps.c	(revision 23484)
@@ -88,24 +88,4 @@
   }
 
-  /* check if controller is running */
-  status = CheckControllerStatus ();
-  if (!status) {
-    return;
-  }
-
-  sprintf (command, "jobstack busy");
-  InitIOBuffer (&buffer, 0x100);
-
-  SerialThreadLock ();
-  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
-
-  if (status) {
-    gprint (GP_LOG, " jobs currently running remotely:\n");
-    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
-  } else {
-    gprint (GP_LOG, "controller is not responding\n");
-  }
-  FreeIOBuffer (&buffer);
   return;
 }
@@ -285,7 +265,5 @@
   if (job[0].mode == JOB_LOCAL) {
     CheckLocalJob (job);
-  } else {
-    /* controller jobs are now checked en masse by CheckController */
-    /* CheckControllerJob (job); */
+    /* controller jobs are checked en masse by CheckController */
   }
   return (job[0].state);
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ListenClients.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ListenClients.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/ListenClients.c	(revision 23484)
@@ -2,4 +2,5 @@
 # define DEBUG 0
 
+// XXX make the calling functions thread-safe
 static int NCLIENTS;
 static int Nclients;
@@ -9,8 +10,10 @@
 void InitClients () {
 
+  ClientLock();
   Nclients = 0;
   NCLIENTS = 10;
   ALLOCATE (clients, int, NCLIENTS);
   ALLOCATE (buffers, IOBuffer *, NCLIENTS);
+  ClientUnlock();
 }
 
@@ -18,5 +21,6 @@
 void AddNewClient (int client) {
 
-  if (DEBUG) gprint (GP_LOG, "adding a new client (%d)\n", client);
+  ClientLock();
+  if (DEBUG) fprintf (stderr, "adding a new client (%d)\n", client);
   clients[Nclients] = client;
   ALLOCATE (buffers[Nclients], IOBuffer, 1);
@@ -28,4 +32,5 @@
     REALLOCATE (buffers, IOBuffer *, NCLIENTS);
   }
+  ClientUnlock();
 }
 
@@ -35,5 +40,6 @@
   int i, j;
 
-  if (DEBUG) gprint (GP_LOG, "deleting a client (%d)\n", client);
+  ClientLock();
+  if (DEBUG) fprintf (stderr, "deleting a client (%d)\n", client);
   for (i = 0; i < Nclients; i++) {
     if (clients[i] == client) {
@@ -51,8 +57,10 @@
 	REALLOCATE (buffers, IOBuffer *, NCLIENTS);
       }
+      ClientUnlock();
       return TRUE;
     }
   }
   // did not find the client
+  ClientUnlock();
   return FALSE;
 }
@@ -70,15 +78,7 @@
   gprintInit ();  // each thread needs to init the printing system
 
-  // define server output log files
-  if (VarConfig ("PANTASKS_SERVER_STDOUT", "%s", log_stdout) != NULL) {
-    gprintSetFileThisThread (GP_LOG, log_stdout);
-  } else {
-    strcpy (log_stdout, "stdout");
-  }
-  if (VarConfig ("PANTASKS_SERVER_STDERR", "%s", log_stderr) != NULL) {
-    gprintSetFileThisThread (GP_ERR, log_stderr);
-  } else {
-    strcpy (log_stderr, "stderr");
-  }
+  /* set buffers for the output for this client */
+  gprintSetBuffer (GP_LOG);
+  gprintSetBuffer (GP_ERR);
 
   while (1) {
@@ -90,7 +90,9 @@
 
     /* place all of the clients in the fdSet */
-    Ncurrent = Nclients;
     Nmax = 0;
     FD_ZERO (&fdSet);
+    ClientLock();
+    Ncurrent = Nclients;
+    ClientUnlock();
     for (i = 0; i < Ncurrent; i++) {
       Nmax = MAX (Nmax, clients[i]);
@@ -100,5 +102,5 @@
 
     /* block until we have some data on the pipes (or timeout) */
-    if (DEBUG) gprint (GP_ERR, "listening to %d clients\n", Ncurrent);
+    if (DEBUG) fprintf (stderr, "listening to %d clients\n", Ncurrent);
     status = select (Nmax, &fdSet, NULL, NULL, &timeout);
     if (status == -1) {
@@ -122,11 +124,11 @@
       if ((Nread == 0) || (Nread == -2)) {
 	/* error: do something */
-	if (DEBUG && (Nread == 0)) gprint (GP_ERR, "socket is closed\n");
-	if (DEBUG && (Nread == -2)) gprint (GP_ERR, "error reading from socket\n");
+	if (DEBUG && (Nread == 0)) fprintf (stderr, "socket is closed\n");
+	if (DEBUG && (Nread == -2)) fprintf (stderr, "error reading from socket\n");
 	DeleteClient (clients[i]);
-	continue;
+	break;  // the other thread could also have modified the list; restart with new Ncurrent
       }
 
-      if (DEBUG) gprint (GP_ERR, "read %d total bytes\n", buffers[i][0].Nbuffer);
+      if (DEBUG) fprintf (stderr, "read %d total bytes\n", buffers[i][0].Nbuffer);
 
       /* see if we have a complete message waiting; if not, keep waiting for messages */
@@ -141,10 +143,8 @@
       if (*line) {
 
-	/* set buffers for the output for this client */
-	gprintSetBuffer (GP_LOG);
-	gprintSetBuffer (GP_ERR);
-
 	/* run the command, return the exit status */
+	CommandLock();
 	status = multicommand (line);
+	CommandUnlock();
 	SendMessage (clients[i], "STATUS %d", status);
 
@@ -156,4 +156,5 @@
 	  SendMessageFixed (clients[i], 0, "");
 	}	  
+	FlushIOBuffer (outbuffer);
 
 	// return the stdout messages first
@@ -164,8 +165,5 @@
 	  SendMessageFixed (clients[i], 0, "");
 	}	  
-	
-	/* clear and reset the output buffers to their last output file names */
-	gprintSetFileAllThreads (GP_LOG, NULL);
-	gprintSetFileAllThreads (GP_ERR, NULL);
+	FlushIOBuffer (outbuffer);
       }
       free (line);
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/Makefile
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/Makefile	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/Makefile	(revision 23484)
@@ -34,6 +34,5 @@
 $(SRC)/pantasks.$(ARCH).o \
 $(SRC)/thread_locks.$(ARCH).o \
-$(SRC)/job_threads.$(ARCH).o \
-$(SRC)/task_threads.$(ARCH).o \
+$(SRC)/jobs_and_tasks_thread.$(ARCH).o \
 $(SRC)/controller_threads.$(ARCH).o 
 
@@ -41,6 +40,4 @@
 $(SRC)/pantasks_server.$(ARCH).o \
 $(SRC)/server_run.$(ARCH).o \
-$(SRC)/server_load.$(ARCH).o \
-$(SRC)/InputQueue.$(ARCH).o \
 $(SRC)/ListenClients.$(ARCH).o \
 $(SRC)/server.$(ARCH).o \
@@ -49,8 +46,6 @@
 $(SRC)/CheckPassword.$(ARCH).o \
 $(SRC)/thread_locks.$(ARCH).o \
-$(SRC)/job_threads.$(ARCH).o \
-$(SRC)/task_threads.$(ARCH).o \
-$(SRC)/controller_threads.$(ARCH).o \
-$(SRC)/input_threads.$(ARCH).o
+$(SRC)/jobs_and_tasks_thread.$(ARCH).o \
+$(SRC)/controller_threads.$(ARCH).o
 
 funcs = \
@@ -66,5 +61,4 @@
 
 cmds = \
-$(SRC)/pulse.$(ARCH).o \
 $(SRC)/status.$(ARCH).o \
 $(SRC)/flush.$(ARCH).o \
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller.c	(revision 23484)
@@ -50,5 +50,7 @@
   }
 
+  ControlLock(__func__);
   status = (*func)(argc - 1, argv + 1);
+  ControlUnlock(__func__);
   return (status);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_host.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_host.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_host.c	(revision 23484)
@@ -15,4 +15,5 @@
 
   if (argc != 3) goto usage;
+  if (max_threads && strcasecmp (argv[1], "ADD")) goto usage;
 
   /* start controller connection (if needed) */
@@ -27,7 +28,5 @@
   if (!strcasecmp (argv[1], "ADD")) {
     AddHost (argv[2], max_threads);
-  } else {
-    if (max_threads) goto usage;
-  }
+  } 
 
   if (!strcasecmp (argv[1], "DELETE")) {
@@ -42,7 +41,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
 
   if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
@@ -57,8 +54,2 @@
   return (FALSE);
 }
-
-/* should I keep an internal host table so I can reload the 
-   hosts if the controller exits?
-
-   alternatively, that could be a user-level choice
-*/
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_jobstack.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_jobstack.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_jobstack.c	(revision 23484)
@@ -24,7 +24,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
 
   if (status) {
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_status.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_status.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_status.c	(revision 23484)
@@ -23,7 +23,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
 
   if (status) {
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_threads.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_threads.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_threads.c	(revision 23484)
@@ -34,8 +34,12 @@
 
     // one run of the task checker
-    SerialThreadLock ();
+    ControlLock(__func__);
     CheckController ();
+    ControlUnlock(__func__);
+
+    ControlLock(__func__);
     CheckControllerOutput ();
-    SerialThreadUnlock ();
+    ControlUnlock(__func__);
+
     if (VerboseMode() == 2) fprintf (stderr, "C");
     // fprintf (stderr, "**** C ****");
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_verbose.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_verbose.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/controller_verbose.c	(revision 23484)
@@ -24,7 +24,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
 
   if (status) {
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/delete.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/delete.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/delete.c	(revision 23484)
@@ -11,7 +11,9 @@
     JobID = atoi (argv[2]);
 
+    JobTaskLock();
     job = FindJob (JobID);
     if (job == NULL) {
       gprint (GP_LOG, "job not found\n");
+      JobTaskUnlock();
       return (TRUE);
     }
@@ -21,4 +23,5 @@
 	job[0].state = JOB_HUNG;
 	if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+	JobTaskUnlock();
 	return (FALSE);
       }
@@ -27,4 +30,5 @@
 	job[0].state = JOB_HUNG;
 	if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+	JobTaskUnlock();
 	return (FALSE);
       }
@@ -32,4 +36,5 @@
     DeleteJob (job);
     gprint (GP_LOG, "job removed\n");
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -41,7 +46,7 @@
 
 usage:
-    gprint (GP_ERR, "USAGE: delete job (JobID)\n");
-    gprint (GP_ERR, "USAGE: delete task (TaskName)\n");
-    return (FALSE);
+  gprint (GP_ERR, "USAGE: delete job (JobID)\n");
+  gprint (GP_ERR, "USAGE: delete task (TaskName)\n");
+  return (FALSE);
 }
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/flush.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/flush.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/flush.c	(revision 23484)
@@ -6,5 +6,7 @@
 
   if (!strcasecmp (argv[1], "jobs")) {
+    JobTaskLock();
     FlushJobs ();
+    JobTaskUnlock();
     return (TRUE);
   }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init.c	(revision 23484)
@@ -18,5 +18,4 @@
 int halt            PROTO((int, char **));
 int flush_jobs      PROTO((int, char **));
-int pulse           PROTO((int, char **));
 int showtask        PROTO((int, char **));
 int status_sys      PROTO((int, char **));
@@ -40,5 +39,4 @@
   {1, "options",    task_options,  "define optional variables associated with the job task"},
   {1, "periods",    task_periods,  "define time scales for a task"},
-  {1, "pulse",      pulse,         "set the scheduler update period"},
   {1, "run",        run,           "run the scheduler"},
   {1, "flush",      flush_jobs,    "flush all jobs from the queue"},
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init_server.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init_server.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/init_server.c	(revision 23484)
@@ -14,5 +14,4 @@
 int task_stdout     PROTO((int, char **));
 int task_stderr     PROTO((int, char **));
-int pulse           PROTO((int, char **));
 int flush_jobs      PROTO((int, char **));
 int status_server   PROTO((int, char **));
@@ -41,5 +40,4 @@
   {1, "npending",   task_npending, "define maximum number of outstanding jobs for a task"},
   {1, "periods",    task_periods,  "define time scales for a task"},
-  {1, "pulse",      pulse,         "set the scheduler update period"},
   {1, "flush",      flush_jobs,    "flush all jobs from the queue"},
   {1, "server",     server,        "server-specific commands"},
@@ -67,5 +65,4 @@
   InitJobs ();
   InitJobIDs ();
-  InitInputs ();
 
   for (i = 0; i < sizeof (cmds) / sizeof (Command); i++) {
@@ -78,4 +75,3 @@
   FreeJobs ();
   FreeJobIDs ();
-  FreeInputs ();
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/jobs_and_tasks_thread.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/jobs_and_tasks_thread.c	(revision 23484)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/jobs_and_tasks_thread.c	(revision 23484)
@@ -0,0 +1,95 @@
+# include "pantasks.h"
+
+void ResetTaskTimers ();
+
+static int CheckTasksRun = FALSE;
+static int CheckJobsRun = FALSE;
+
+void CheckJobsSetState (int state) {
+  CheckJobsRun = state;
+}
+int CheckJobsGetState () {
+  return (CheckJobsRun);
+}
+
+void CheckTasksSetState (int state) {
+  if (state && !CheckTasksRun) {
+    ResetTaskTimers ();
+  }
+  CheckTasksRun = state;
+}
+int CheckTasksGetState () {
+  return (CheckTasksRun);
+}
+
+void *CheckJobsAndTasksThread (void *data) {
+
+  char log_stdout[128], log_stderr[128];
+  float job_timeout, task_timeout, next_timeout;
+
+  gprintInit ();  // each thread needs to init the printing system
+
+  // define server output log files
+  if (VarConfig ("PANTASKS_SERVER_STDOUT", "%s", log_stdout) != NULL) {
+      gprintSetFileThisThread (GP_LOG, log_stdout);
+  }
+  if (VarConfig ("PANTASKS_SERVER_STDERR", "%s", log_stderr) != NULL) {
+      gprintSetFileThisThread (GP_ERR, log_stderr);
+  }
+
+  while (1) {
+
+    // one run of the task checker
+    task_timeout = 0.25;
+    if (CheckTasksRun) {
+      task_timeout = CheckTasks ();
+      if (VerboseMode() == 2) fprintf (stderr, "T");
+    }
+
+    // one run of the task checker
+    job_timeout = 0.25;
+    if (CheckJobsRun) {
+      job_timeout = CheckJobs ();
+      if (VerboseMode() == 2) fprintf (stderr, "J");
+    }
+
+    // job_timeout, task_timeout is time until next job,task is ready
+    // sleep more-or-less that long (but no longer than 250msec)
+    next_timeout = MIN (job_timeout, task_timeout);
+    next_timeout = MIN (next_timeout, 0.25);
+    if (next_timeout > 0.001) {
+      usleep ((int)(1000000*next_timeout)); // allow other threads a chance to run
+    }      
+  }
+}
+
+// I sleep for a small amount of time here, based on how long until the next expected
+// timeout.  this enforces a certain granularity in the task creation, but prevents the task
+// thread from driving the load up to silly levels.
+
+// reset all of the task timers, with a bit of fuzz.  this is called 
+// whenever we transition from 'stop' to 'run'
+void ResetTaskTimers () {
+
+  Task *task;
+  struct timeval now;
+  float fuzz;
+
+  // get the current time
+  gettimeofday (&now, NULL);
+
+  // check all tasks
+  while ((task = NextTask ()) != NULL) {
+    task[0].last.tv_usec = now.tv_usec;
+    task[0].last.tv_sec  = now.tv_sec;
+
+    // add random offset between 0 and 10% of exec_period
+    // XXX this should be optional
+    fuzz = task[0].exec_period*drand48();
+    task[0].last.tv_usec += 1e6*(fuzz - (int)fuzz);
+    task[0].last.tv_sec += (int) fuzz;
+
+    // gprint (GP_LOG, "fuzz: %f, last: %d %d\n", fuzz, task[0].last.tv_sec, task[0].last.tv_usec);
+  }
+  return;
+}
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/kill.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/kill.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/kill.c	(revision 23484)
@@ -12,7 +12,9 @@
   JobID = atoi (argv[1]);
 
+  JobTaskLock();
   job = FindJob (JobID);
   if (job == NULL) {
     gprint (GP_LOG, "job not found\n");
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -22,4 +24,5 @@
       job[0].state = JOB_HUNG;
       if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -28,4 +31,5 @@
       job[0].state = JOB_HUNG;
       if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -33,4 +37,5 @@
   DeleteJob (job);
   gprint (GP_LOG, "job removed\n");
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/notes.txt
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/notes.txt	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/notes.txt	(revision 23484)
@@ -1,2 +1,116 @@
+
+stdout_cntl:
+  GetJobOutout
+    CheckControllerJob
+      CheckController
+  StartController
+  ControllerCommand
+    CheckController
+    controller_check
+    controller_host
+    controller_jobstack
+    controller_run
+    controller_status
+    controller_verbose
+    DeleteControllerJob
+    CheckControllerJobStatus
+    SubmitControllerJob
+    PrintControllerBusyJobs
+    KillControllerJob
+    QuitController
+    RestartController    
+  CheckControllerOutput
+    controller_output
+    controller_threads
+
+
+
+
+
+
+
+20090322
+
+  We've been surviving with a slightly broken threading / locking
+  model.  I would like to clean it up.  Previously, the threads where
+  blocking at a very coarse level, and there were certain interactions
+  that were not thread-safe.  Here are my notes on re-working this:
+
+  * we have 6 threads:
+
+    * main (top-level parent) : after it spawns the 5 child threads,
+      this thread waits for new clients and adds them to the list of
+      active clients with 'AddNewClient'
+
+    * clientsThread (ListenClients): this thread is listening for
+      commands from the clients; when it receives a complete command,
+      it executes the command using 'multicommand'.
+
+    * tasksThread
+      CheckTasks
+	NextTask
+
+
+
+    * jobsThread
+
+    * controllerThread
+      CheckControllerStatus : race condition is irrelevant
+
+    * inputsThread
+      AddNewInput called by clientThread
+      (not sure this is really being used: 
+      	   server input (filename) calls 'input'
+	   server module (filename) calls 'module'
+
+  * functions which need to be thread-safe:
+    * AddNewClient
+    * gprint and supporting functions must be thread safe (extensively
+      used...)
+
+
+* race condition is irrelevant for this variable
+static int ControllerStatus = FALSE;
+
+static int stdin_cntl, stdout_cntl, stderr_cntl;
+GetJobOutput -> CheckControllerJob -> CheckController -> controllerThread
+ControllerCommand -> controllerThread / clientThread
+CheckControllerOutput -> controllerThread / clientThread
+StartController -> controllerThread / clientThread
+StopController
+
+
+
+static IOBuffer stdout_buffer;
+static IOBuffer stderr_buffer;
+static int ControllerPID = 0;
+
+/* local static variables to track the controller host properties */
+static Host *hosts = NULL;
+static int Nhosts = 0;
+static int NHOSTS = 0;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 PanTasks Client / Server design
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks.c.in
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks.c.in	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks.c.in	(revision 23484)
@@ -10,6 +10,5 @@
 void program_init (int *argc, char **argv) {
   
-  pthread_t jobsThread;
-  pthread_t tasksThread;
+  pthread_t JobsAndTasksThread;
   pthread_t controllerThread;
 
@@ -50,7 +49,8 @@
 
   /* start up the background threads here */
-  pthread_create (&tasksThread,      NULL, &CheckTasksThread, 	   NULL);
-  pthread_create (&jobsThread,       NULL, &CheckJobsThread, 	   NULL);
-  pthread_create (&controllerThread, NULL, &CheckControllerThread, NULL);
+  // pthread_create (&tasksThread,      NULL, &CheckTasksThread, 	   NULL);
+  // pthread_create (&jobsThread,       NULL, &CheckJobsThread, 	   NULL);
+  pthread_create (&JobsAndTasksThread, NULL, &CheckJobsAndTasksThread, NULL);
+  pthread_create (&controllerThread,   NULL, &CheckControllerThread,   NULL);
   return;
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks_server.c.in
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks_server.c.in	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/pantasks_server.c.in	(revision 23484)
@@ -17,7 +17,5 @@
   
   char log_stdout[128], log_stderr[128];
-  pthread_t jobsThread;
-  pthread_t tasksThread;
-  pthread_t inputsThread;
+  pthread_t JobsAndTasksThread;
   pthread_t clientsThread;
   pthread_t controllerThread;
@@ -63,9 +61,7 @@
 
   /* start up the background threads here */
-  pthread_create (&clientsThread,    NULL, &ListenClients,    	   NULL);
-  pthread_create (&tasksThread,      NULL, &CheckTasksThread, 	   NULL);
-  pthread_create (&jobsThread,       NULL, &CheckJobsThread,  	   NULL);
-  pthread_create (&controllerThread, NULL, &CheckControllerThread, NULL);
-  pthread_create (&inputsThread,     NULL, &CheckInputsThread,     NULL);
+  pthread_create (&clientsThread,       NULL, &ListenClients,    	NULL);
+  pthread_create (&JobsAndTasksThread,  NULL, &CheckJobsAndTasksThread, NULL);
+  pthread_create (&controllerThread, 	NULL, &CheckControllerThread,   NULL);
 
   /* in this loop, we listen for incoming connections, validate, and
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server.c	(revision 23484)
@@ -5,5 +5,4 @@
 int input      	    PROTO((int, char **));
 int module     	    PROTO((int, char **));
-int server_load	    PROTO((int, char **));
 int server_run	    PROTO((int, char **));
 int server_stop	    PROTO((int, char **));
@@ -20,5 +19,4 @@
   {1, "input",  input,  "load input file on server"},
   {1, "module", module, "load module file on server"},
-  {1, "load",   server_load, "load input file on server"},
   {1, "run",    server_run,  "run scheduler"},
   {1, "stop",   server_stop, "stop scheduler"},
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server_run.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server_run.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/server_run.c	(revision 23484)
@@ -11,5 +11,4 @@
   CheckJobsSetState (TRUE);
   CheckControllerSetState (TRUE);
-  CheckInputsSetState (TRUE);
   return (TRUE);
 }
@@ -23,7 +22,4 @@
 
   CheckTasksSetState (FALSE);
-  // CheckJobsSetState (FALSE);
-  // CheckControllerSetState (FALSE);
-  CheckInputsSetState (FALSE);
   return (TRUE);
 }
@@ -39,5 +35,4 @@
   CheckJobsSetState (FALSE);
   CheckControllerSetState (FALSE);
-  CheckInputsSetState (FALSE);
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/showtask.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/showtask.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/showtask.c	(revision 23484)
@@ -8,5 +8,8 @@
   }
 
+  JobTaskLock();
   ShowTask (argv[1]);
+  JobTaskUnlock();
+
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/status_server.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/status_server.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/status_server.c	(revision 23484)
@@ -11,5 +11,7 @@
   if ((N = get_argument (argc, argv, "-tasks"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     ListTasks (FALSE);
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -17,5 +19,7 @@
   if ((N = get_argument (argc, argv, "-taskinfo"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     ListTasks (TRUE);
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -23,4 +27,5 @@
   if ((N = get_argument (argc, argv, "-taskstats"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     if (argc == 2) {
       ListTaskStats (argv[N]);
@@ -28,4 +33,5 @@
       ListTaskStats (NULL);
     }      
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -33,4 +39,5 @@
   if ((N = get_argument (argc, argv, "-taskstatsreset"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     if (argc == 2) {
       ResetTaskStats (argv[N]);
@@ -38,4 +45,5 @@
       ResetTaskStats (NULL);
     }      
+    JobTaskUnlock();
     return (TRUE);
   }
@@ -56,6 +64,14 @@
     gprint (GP_LOG, " Controller is stopped\n");
   }
+
+  JobTaskLock();
   ListTasks (FALSE);
   ListJobs ();
+  JobTaskUnlock();
+
+  ControlLock(__func__);
+  PrintControllerBusyJobs();
+  ControlUnlock(__func__);
+
   return (TRUE);
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task.c	(revision 23484)
@@ -11,4 +11,5 @@
   if (argc != 2) goto usage;
 
+  JobTaskLock();
   task = FindTask (argv[1]);
   if (task == NULL) { /**** new task ****/
@@ -18,4 +19,6 @@
     SetNewTask (task);
   }
+  JobTaskUnlock();
+
   /* While a task is being defined, it is removed from the task list.  The new task is added to the task list
      when the definition process is complete.  
@@ -55,6 +58,4 @@
 
       case TASK_END:
-	/* I need to add in a test here to see if all task elements 
-	   have been defined.  delete the task if not */
 	free (input);
 	/* validate the new task: all mandatory elements defined? */ 
@@ -63,5 +64,7 @@
 	  return (FALSE);
 	}
+	JobTaskLock();
 	RegisterNewTask ();
+	JobTaskUnlock();
 	return (TRUE);
 	break;
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_active.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_active.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_active.c	(revision 23484)
@@ -7,7 +7,9 @@
   if (argc != 2) goto usage;
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
   }
@@ -15,12 +17,15 @@
   if (!strcasecmp (argv[1], "true")) {
     task[0].active = TRUE;
+    JobTaskUnlock();
     return (TRUE);
   }
   if (!strcasecmp (argv[1], "false")) {
     task[0].active = FALSE;
+    JobTaskUnlock();
     return (TRUE);
   }
 
   gprint (GP_ERR, "ERROR: invalid option: %s\n", argv[1]);
+  JobTaskUnlock();
   return (FALSE);
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_command.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_command.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_command.c	(revision 23484)
@@ -12,4 +12,5 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
@@ -17,4 +18,5 @@
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -35,4 +37,5 @@
     task[0].argv[i] = strcreate (argv[i+1]);
   }
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_host.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_host.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_host.c	(revision 23484)
@@ -22,8 +22,10 @@
 
   task = GetNewTask ();
+  JobTaskLock();
   if (task == NULL) {
     task = GetActiveTask ();
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -34,7 +36,11 @@
   task[0].host = NULL;
 
-  if (!strcasecmp (argv[1], "LOCAL")) return (TRUE);
+  if (!strcasecmp (argv[1], "LOCAL")) {
+    JobTaskUnlock();
+    return (TRUE);
+  }
 
   task[0].host = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_macros.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_macros.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_macros.c	(revision 23484)
@@ -22,7 +22,9 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
   }
@@ -135,4 +137,5 @@
 	free (input);
 	REALLOCATE (macro[0].line, char *, MAX (1, macro[0].Nlines));
+	JobTaskUnlock();
 	return (TRUE);
       }
@@ -148,4 +151,5 @@
     }
   }
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_nmax.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_nmax.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_nmax.c	(revision 23484)
@@ -7,11 +7,14 @@
   if (argc != 2) goto usage;
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
   }
 
   task[0].Nmax = atoi (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 
@@ -27,11 +30,14 @@
   if (argc != 2) goto usage;
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
   }
 
   task[0].NpendingMax = atoi (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_options.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_options.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_options.c	(revision 23484)
@@ -12,4 +12,5 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
@@ -17,4 +18,5 @@
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -35,4 +37,5 @@
     task[0].optv[i] = strcreate (argv[i+1]);
   }
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_periods.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_periods.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_periods.c	(revision 23484)
@@ -40,4 +40,5 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
@@ -45,4 +46,5 @@
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -53,4 +55,5 @@
   if (Timeout) task[0].timeout_period = TimeoutValue;
 
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_stdout.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_stdout.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_stdout.c	(revision 23484)
@@ -11,4 +11,5 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
@@ -16,4 +17,5 @@
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -21,4 +23,5 @@
   if (task[0].stdout_dump != NULL) free (task[0].stdout_dump);
   task[0].stdout_dump = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 }
@@ -34,4 +37,5 @@
   }
 
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
@@ -39,4 +43,5 @@
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -44,4 +49,5 @@
   if (task[0].stderr_dump != NULL) free (task[0].stderr_dump);
   task[0].stderr_dump = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 }
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_trange.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_trange.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/task_trange.c	(revision 23484)
@@ -12,7 +12,9 @@
     if (argc != 1) goto usage;
 
+    JobTaskLock();
     task = GetNewTask ();
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
     }
@@ -20,4 +22,5 @@
     task[0].Nranges = 0;
     REALLOCATE (task[0].ranges, TimeRange, 1);
+    JobTaskUnlock();
     return (TRUE);
   } 
@@ -40,10 +43,4 @@
 
   if (argc != 3) goto usage;
-
-  task = GetNewTask ();
-  if (task == NULL) {
-    gprint (GP_ERR, "ERROR: not defining or running a task\n");
-    return (FALSE);
-  }
 
   /* test for Mon[@HH:MM:SS] - both must match */
@@ -83,4 +80,12 @@
 
 valid:
+  JobTaskLock();
+  task = GetNewTask ();
+  if (task == NULL) {
+    gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
+    return (FALSE);
+  }
+
   N = task[0].Nranges;
   task[0].Nranges ++;
@@ -88,4 +93,5 @@
   
   task[0].ranges[N] = range;
+  JobTaskUnlock();
   return (TRUE);
 
Index: /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/thread_locks.c
===================================================================
--- /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/thread_locks.c	(revision 23483)
+++ /branches/eam_branches/eam_branch_20090322/Ohana/src/opihi/pantasks/thread_locks.c	(revision 23484)
@@ -1,14 +1,52 @@
 # include "pantasks.h"
 
-/* this mutex is used by the serialized threads */
+/* mutex to lock Client table operations */
+static pthread_mutex_t ClientMutex = PTHREAD_MUTEX_INITIALIZER;
 
-static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-
-void SerialThreadLock () {
-    pthread_mutex_lock (&mutex);
+void ClientLock () {
+  pthread_mutex_lock (&ClientMutex);
 }
 
-void SerialThreadUnlock () {
-    pthread_mutex_unlock (&mutex);
+void ClientUnlock () {
+  pthread_mutex_unlock (&ClientMutex);
 }
 
+/* mutex to lock Command / Multicommand operations */
+static pthread_mutex_t CommandMutex = PTHREAD_MUTEX_INITIALIZER;
+
+void CommandLock () {
+  //  fprintf (stderr, "command lock\n");
+  pthread_mutex_lock (&CommandMutex);
+}
+
+void CommandUnlock () {
+  //  fprintf (stderr, "command unlock\n");
+  pthread_mutex_unlock (&CommandMutex);
+}
+
+/* mutex to lock Control operations */
+static pthread_mutex_t ControlMutex = PTHREAD_MUTEX_INITIALIZER;
+
+void ControlLock (char *func) {
+  // fprintf (stderr, "control lock %s\n", func);
+  pthread_mutex_lock (&ControlMutex);
+}
+
+void ControlUnlock (char *func) {
+  // fprintf (stderr, "control unlock %s\n", func);
+  pthread_mutex_unlock (&ControlMutex);
+}
+
+/* mutex to lock Job / Task operations */
+static pthread_mutex_t JobTaskMutex = PTHREAD_MUTEX_INITIALIZER;
+
+void JobTaskLock () {
+  //  fprintf (stderr, "jobtask lock\n");
+  pthread_mutex_lock (&JobTaskMutex);
+}
+
+void JobTaskUnlock () {
+  //  fprintf (stderr, "jobtask unlock\n");
+  pthread_mutex_unlock (&JobTaskMutex);
+}
+