Index: /trunk/Ohana/src/opihi/include/pcontrol.h
===================================================================
--- /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 21378)
+++ /trunk/Ohana/src/opihi/include/pcontrol.h	(revision 21379)
@@ -25,4 +25,12 @@
   PCONTROL_JOB_NEEDHOST,
 } JobMode;
+
+/** job mode check stages **/
+typedef enum {
+  PCONTROL_JOB_STAGE_ANYHOST,
+  PCONTROL_JOB_STAGE_WANTHOST,
+  PCONTROL_JOB_STAGE_NEEDHOST,
+  PCONTROL_JOB_STAGE_OLDWANT,
+} JobCheckStage;
 
 /** job thread options values **/
@@ -180,5 +188,5 @@
 int   CheckDoneHosts (float delay);
 int   CheckDownHosts (float delay);
-int   CheckIdleHosts (float delay);
+int   CheckIdleHosts (float delay, int Stage);
 int   CheckLiveHosts (float delay);
 int   SetRunSystem (int state);
@@ -202,5 +210,5 @@
 
 int StartHost (Host *host);
-int CheckIdleHost (Host *host);
+int CheckIdleHost (Host *host, int Stage);
 int CheckDoneJob (Job *job, Host *host);
 int GetJobOutput (char *command, Host *host, JobOutput *output);
Index: /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 21378)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c	(revision 21379)
@@ -4,9 +4,11 @@
 
 /* the supplied host is not on a stack: it cannot be taken by the other thread */
-int CheckIdleHost (Host *host) {
+int CheckIdleHost (Host *host, int Stage) {
 
   int i;
   Stack *stack;
   Job *job;
+  struct timeval now;
+  float dtime;
 
   if (logfile == NULL) { 
@@ -28,5 +30,5 @@
   
   /* look for first NEEDHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_NEEDHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_NEEDHOST) continue;
@@ -48,5 +50,5 @@
 
   /* no NEEDHOST entry, look for first WANTHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_WANTHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
@@ -68,5 +70,5 @@
 
   /* no WANTHOST entry, look for first ANYHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_ANYHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_ANYHOST) continue;
@@ -86,10 +88,12 @@
 
   /* no ANYHOST entry, look for first WANTHOST with old time */
-  /* XXX perhaps I should add this to the conditions for ANYHOST instead of
-     running a separate loop?  ie, WANTHOST && time > X == ANYHOST */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_OLDWANT) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
-    // XXX test the job age and skip if too young
+
+    // allow WANT jobs to wait up to 10.0 sec for the host to be free before giving up
+    gettimeofday (&now, (void *) NULL);
+    dtime = DTIME (now, job[0].start);
+    if (dtime > 10.0) continue;
 
     if (logfile) fprintf (logfile, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
Index: /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 21378)
+++ /trunk/Ohana/src/opihi/pcontrol/CheckSystem.c	(revision 21379)
@@ -22,5 +22,8 @@
 
   /* we want to give each block a maximum allowed time */
-  CheckIdleHosts(0.020); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_NEEDHOST); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_WANTHOST); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_ANYHOST);  /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_OLDWANT);  /* submit a new job */
 
   CheckBusyJobs(0.020);  /* get job status */
@@ -109,5 +112,8 @@
     if (RunLevel == PCONTROL_RUN_ALL) {
       // we want to give each block a maximum allowed time
-      Nhostchecks += CheckIdleHosts(0.020); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_NEEDHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_WANTHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_ANYHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_OLDWANT); /* submit a new job (PCLIENT) */
       TestCheckPoint ();
     }
@@ -367,5 +373,7 @@
 }
 
-int CheckIdleHosts (float MaxDelay) {
+// if we have any IDLE hosts, check if there are jobs to be launched 
+// for each pass, we only check one type of job: stage = NEED, WANT, ANY, OLDWANT
+int CheckIdleHosts (float MaxDelay, int Stage) {
 
   struct timeval start, stop;
@@ -389,8 +397,9 @@
     host = PullStackByLocation (stack, STACK_TOP);
     if (host == NULL) break;
-    CheckIdleHost (host);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
+    CheckIdleHost (host, Stage);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+
   if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
   return (i);
Index: /trunk/Ohana/src/opihi/pcontrol/JobOps.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 21378)
+++ /trunk/Ohana/src/opihi/pcontrol/JobOps.c	(revision 21379)
@@ -231,4 +231,7 @@
   PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
 
+  // until the job is launched, we use 'start' to time how long the job is waiting on the queue
+  gettimeofday (&job[0].start, (void *) NULL);
+
   if (VerboseMode()) gprint (GP_ERR, "added new job\n");
   return (JobID);
Index: /trunk/Ohana/src/opihi/pcontrol/check.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/check.c	(revision 21378)
+++ /trunk/Ohana/src/opihi/pcontrol/check.c	(revision 21379)
@@ -3,4 +3,5 @@
 int check (int argc, char **argv) {
 
+  int N, Save;
   int JobID, HostID;
 
@@ -8,4 +9,10 @@
   Job *job = NULL;
   Host *host = NULL;
+
+  Save = FALSE;
+  if ((N = get_argument (argc, argv, "-save"))) {
+    remove_argument (N, &argc, argv);
+    Save = TRUE;
+  }
 
   if (argc != 3) {
@@ -39,4 +46,19 @@
 	gprint (GP_LOG, "HOSTNAME NONE\n");
     }
+
+    if (Save) {
+	set_str_variable ("JOB_STATUS", GetJobStackName(job[0].stack));
+	set_int_variable ("JOB_EXITST", job[0].exit_status);
+	set_int_variable ("JOB_STDOUT_SIZE", job[0].stdout.size);
+	set_int_variable ("JOB_STDERR_SIZE", job[0].stderr.size);
+	set_variable ("JOB_DTIME", job[0].dtime);
+	set_str_variable ("JOB_HOSTNAME", job[0].hostname);
+	if (job[0].realhost) {
+	    set_str_variable ("JOB_REALHOST", job[0].realhost);
+	} else {
+	    set_str_variable ("JOB_REALHOST", "NONE");
+	}
+    }
+
     PushStack (stack, STACK_BOTTOM, job, job[0].JobID, job[0].argv[0]);
     return (TRUE);
@@ -53,4 +75,9 @@
     }
     gprint (GP_LOG, "host %s\n", GetHostStackName(host[0].stack));
+
+    if (Save) {
+	set_str_variable ("HOST_STATE", GetHostStackName(host[0].stack));
+    }
+
     PushStack (stack, STACK_BOTTOM, host, host[0].HostID, host[0].hostname);
     return (TRUE);
Index: /trunk/Ohana/src/opihi/pcontrol/status.c
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/status.c	(revision 21378)
+++ /trunk/Ohana/src/opihi/pcontrol/status.c	(revision 21379)
@@ -17,4 +17,6 @@
   Stack *stack;
   Job *job;
+  struct timeval now;
+  float dtime;
 
   stack = GetJobStack (Nstack);
@@ -34,4 +36,27 @@
     }
     gprint (GP_LOG, "%7s  ", GetJobStackName (job[0].state));
+
+    switch (job[0].state) {
+	// for active jobs or pending jobs, print time since start (or create in the case of pending)
+      case PCONTROL_JOB_PENDING:
+      case PCONTROL_JOB_BUSY:
+      case PCONTROL_JOB_RESP:
+      case PCONTROL_JOB_HUNG:
+	gettimeofday (&now, (void *) NULL);
+	dtime = DTIME (now, job[0].start);
+	gprint (GP_LOG, "%8.2f ", dtime);
+	break;
+
+	// for active jobs or pending jobs, print time since start (or create in the case of pending)
+      case PCONTROL_JOB_DONE:
+      case PCONTROL_JOB_KILL:
+      case PCONTROL_JOB_EXIT:
+      case PCONTROL_JOB_CRASH:
+      default:
+	dtime = DTIME (job[0].stop, job[0].start);
+	gprint (GP_LOG, "%8.2f ", dtime);
+	break;
+    }
+
     for (j = 0; j < job[0].argc; j++) {
       gprint (GP_LOG, "%s ", job[0].argv[j]);
Index: /trunk/Ohana/src/opihi/pcontrol/test/hosttargets.sh
===================================================================
--- /trunk/Ohana/src/opihi/pcontrol/test/hosttargets.sh	(revision 21379)
+++ /trunk/Ohana/src/opihi/pcontrol/test/hosttargets.sh	(revision 21379)
@@ -0,0 +1,45 @@
+
+# generate a bunch of jobs which are one of the three states NEED, WANT, (ANY?)
+# check the rate at which they are placed on the appropriate type of host.
+
+# ANY : is there a valid test for ANY?
+# NEED must always be sent to the NEED host
+# WANT : report the fraction of time : if job time is << timeout -> WANT host rate should be high
+# WANT : report the fraction of time : if job time is >> timeout -> WANT host rate should be low
+# WANT : report the fraction of time : if job time is  ~ timeout -> WANT host rate should be ~NN%
+
+macro hosttargets
+
+  if ($0 != 1) 
+    echo "USAGE: hosttargets"
+    break
+  end
+
+  if ($?hostlist:n == 0)
+    echo "create a list of target hosts called 'hostlist'"
+    break
+  end
+
+  stop
+
+  for i 0 $hostlist:n
+    # generate a set of jobs with desired targets
+    for j 0 10
+      job -host $hostlist:$i sleep 1
+    end
+    host add $hostlist:$i
+  end
+  
+  run
+
+  # wait 'till jobs complete
+  sleep 12
+
+  # compare desired with actual host
+  # for i 1 20
+  #   check job $i
+  # end
+
+end
+
+## NOTE : dtime in 'status' is not working