Index: trunk/Ohana/src/opihi/pantasks/ControllerOps.c
===================================================================
--- trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 20032)
+++ trunk/Ohana/src/opihi/pantasks/ControllerOps.c	(revision 23530)
@@ -12,4 +12,6 @@
 
 /* local static variables to track the controller host properties */
+/* these are used by AddHost and DeleteHost, and are only called by controller_host.c (clientThread) */
+/* or by RestartController : lock between these two? */
 static Host *hosts = NULL;
 static int Nhosts = 0;
@@ -31,4 +33,7 @@
 }
 
+// XXX possible race condition problem: if we delete a host while the controller
+// is being restarted.  to fix this, we need to keep the deletion in controller_thread,
+// but perhaps mark it in the client_thread?
 int DeleteHost (char *hostname) {
 
@@ -261,7 +266,14 @@
   }
 
+  // This function is called by the JobTaskThread via SubmitJob.  We need to unlock the
+  // JobTaskLock to avoid a dead lock with the JobTaskLock called in CheckController
+  JobTaskUnlock();
+  ControlLock(__func__);
   InitIOBuffer (&buffer, 0x100);
   status = ControllerCommand (cmd, CONTROLLER_PROMPT, &buffer);
   free (cmd);
+  ControlUnlock(__func__);
+  JobTaskLock();
+
 
   /* extract the job PID from the controller response */
@@ -411,5 +423,5 @@
   if ((status == -1) && (errno == EPIPE)) {
     StopController ();
-    gprint (GP_ERR, "controller is down (pipe closed), restarting\n");
+    fprintf (stderr, "controller is down (pipe closed), restarting\n");
     if (!RestartController ()) {
       return (FALSE);
@@ -434,5 +446,5 @@
     if (status ==  0) {
       StopController ();
-      gprint (GP_ERR, "controller is down (EOF), restarting\n");
+      fprintf (stderr, "controller is down (EOF), restarting\n");
       if (!RestartController ()) {
 	return (FALSE);
@@ -441,10 +453,10 @@
     }
     if (status == -1) {
-      gprint (GP_ERR, "controller is not responding (%d tries)\n", j);
+      fprintf (stderr, "controller is not responding (%d tries)\n", j);
       gwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, GP_ERR);
     }
   }
   if (status == -1) {
-    gprint (GP_ERR, "controller still not responding, giving up\n");
+    fprintf (stderr, "controller still not responding, giving up\n");
     return (FALSE);
   }
@@ -456,5 +468,5 @@
     bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
   }
-  /* if (VerboseMode()) gprint (GP_ERR, "message received, %d cycles\n", i); */
+  if (VerboseMode()) fprintf (stderr, "message received, %d cycles\n", i);
   return (TRUE);
 }
@@ -492,4 +504,31 @@
 }
 
+void PrintControllerBusyJobs () {
+
+  int status;
+  char command[1024];
+  IOBuffer buffer;
+
+  /* check if controller is running */
+  status = CheckControllerStatus ();
+  if (!status) {
+    return;
+  }
+
+  sprintf (command, "jobstack busy");
+  InitIOBuffer (&buffer, 0x100);
+
+  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+
+  if (status) {
+    gprint (GP_LOG, " jobs currently running remotely:\n");
+    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
+  } else {
+    gprint (GP_LOG, "controller is not responding\n");
+  }
+  FreeIOBuffer (&buffer);
+  return;
+}
+
 int PrintControllerOutput () {
 
@@ -581,5 +620,5 @@
   InitIOBuffer (&buffer, 0x100);
 
-  // XXX lock the host table? SerialThreadLock ();
+  // XXX lock the host table? no: that would risk a dead lock between client and controller threads:
   gprint (GP_ERR, "pcontrol restarted, reloading hosts\n");
   for (i = 0; i < Nhosts; i++) {
@@ -588,5 +627,4 @@
     status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
   }
-  // SerialThreadUnlock ();
 
   if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);