Index: trunk/Ohana/src/opihi/include/pcontrol.h =================================================================== --- trunk/Ohana/src/opihi/include/pcontrol.h (revision 25871) +++ trunk/Ohana/src/opihi/include/pcontrol.h (revision 25872) @@ -60,4 +60,5 @@ PCONTROL_RESP_KILL_JOB, PCONTROL_RESP_STOP_HOST, + PCONTROL_RESP_DOWN_HOST, } HostResp; @@ -129,10 +130,11 @@ int pid; HostStat stack; - Ptime lasttry; - Ptime nexttry; IDtype HostID; IOBuffer comms_buffer; char *response; HostResp response_state; + Ptime last_start_try; // last (UNIX) time we attempted to connect to this host (0 on success) + Ptime next_start_try; // next (UNIX) time we should attempt to connect to this host (0 on success) + Ptime connect_time; // (UNIX) time we connected to this host struct Job *job; } Host; @@ -210,4 +212,5 @@ int StartHost (Host *host); +int CheckResetHost (Host *host); int CheckIdleHost (Host *host, int Stage); int CheckDoneJob (Job *job, Host *host); @@ -253,7 +256,8 @@ /*** StopHosts.c ***/ void DownHost (Host *host); +int DownHosts (void); +int DownHostResponse (Host *host); void OffHost (Host *host); -int DownHosts (void); -int StopHost (Host *host); +int StopHost (Host *host, int mode); int StopHostResponse (Host *host); int HarvestHost (int pid); Index: trunk/Ohana/src/opihi/pcontrol/CheckHost.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/CheckHost.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/CheckHost.c (revision 25872) @@ -14,5 +14,5 @@ if (host[0].markoff) { host[0].markoff = FALSE; - StopHost (host); + StopHost (host, PCONTROL_HOST_OFF); return (TRUE); } Index: trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 25872) @@ -1,5 +1,29 @@ # include "pcontrol.h" +// The connection to the remote host is only allow to live for MAX_CONNECT_TIME seconds. We +// disconnect and reconnect if a remote host has been connected for too long. This is a +// (temporary?) work-around for the problem that the remote pclient job tends to grow too large +// over time. + +# define MAX_CONNECT_TIME 36000.0 + static FILE *logfile = NULL; + + /* if this host has been connected for too long, disconnect (will automatically reconnect) */ +int CheckResetHost (Host *host) { + + struct timeval now; + float dtime; + + /* if this host has been connected for too long, disconnect (will automatically reconnect) */ + gettimeofday (&now, (void *) NULL); + dtime = DTIME (now, host[0].connect_time); + if (dtime > MAX_CONNECT_TIME) { + if (VerboseMode()) gprint (GP_ERR, "disconnect from %s\n", host[0].hostname); + StopHost (host, PCONTROL_HOST_DOWN); + return (TRUE); + } + return FALSE; +} /* the supplied host is not on a stack: it cannot be taken by the other thread */ @@ -21,8 +45,13 @@ if (host[0].markoff) { host[0].markoff = FALSE; - StopHost (host); + StopHost (host, PCONTROL_HOST_OFF); return (TRUE); } + /* check if host has been connected for too long */ + if (CheckResetHost (host)) { + return (TRUE); + } + /* search the JOB_PENDING stack for an appropriate job */ stack = GetJobStack (PCONTROL_JOB_PENDING); Index: trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/CheckRespHost.c (revision 25872) @@ -32,5 +32,17 @@ host[0].response = NULL; - // host has shutdown; harvest the defunct process + // if want the host to be shutdown, accept the result + if (host[0].response_state == PCONTROL_RESP_DOWN_HOST) { + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_DOWN_HOST\n"); + DownHostResponse (host); + return TRUE; + } + if (host[0].response_state == PCONTROL_RESP_STOP_HOST) { + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_STOP_HOST\n"); + StopHostResponse (host); + return TRUE; + } + + // host has unexpectedly shutdown; harvest the defunct process HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); @@ -47,5 +59,4 @@ case PCLIENT_GOOD: - if (VerboseMode()) gprint (GP_ERR, "message received (CheckRespHost)\n"); break; @@ -80,4 +91,9 @@ break; + case PCONTROL_RESP_DOWN_HOST: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_DOWN_HOST\n"); + status = DownHostResponse (host); + break; + case PCONTROL_RESP_STOP_HOST: if (DEBUG) fprintf (stderr, "PCONTROL_RESP_STOP_HOST\n"); Index: trunk/Ohana/src/opihi/pcontrol/CheckSystem.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/CheckSystem.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/CheckSystem.c (revision 25872) @@ -357,5 +357,5 @@ return (TRUE); } - dtime = DTIME (host[0].nexttry, start); + dtime = DTIME (host[0].next_start_try, start); if (dtime > 0) { PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); @@ -383,7 +383,25 @@ float dtime; - /* check if there are any pending jobs, otherwise skip step */ + /* check if there are any pending jobs */ stack = GetJobStack (PCONTROL_JOB_PENDING); - if (!stack[0].Nobject) return (0); + + /* if there are no pending jobs and we are not in STAGE_NEEDHOST, skip test */ + if (!stack[0].Nobject && (Stage != PCONTROL_JOB_STAGE_NEEDHOST)) return (0); + + /* if there are no pending jobs, check for hosts that need to be reset */ + if (!stack[0].Nobject) { + /* cycle through IDLE hosts */ + stack = GetHostStack (PCONTROL_HOST_IDLE); + Nobject = stack[0].Nobject; + for (i = 0; i < Nobject; i++) { + host = PullStackByLocation (stack, STACK_TOP); + if (host == NULL) break; + if (CheckResetHost (host)) { + return (1); + } + PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM); + } + return (0); + } /* Loop through objects on the stack, no more than once. see note above */ Index: trunk/Ohana/src/opihi/pcontrol/HostOps.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/HostOps.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/HostOps.c (revision 25872) @@ -202,8 +202,8 @@ host[0].HostID = NextHostID(); - host[0].lasttry.tv_sec = 0; - host[0].lasttry.tv_usec = 0; - host[0].nexttry.tv_sec = 0; - host[0].nexttry.tv_usec = 0; + host[0].last_start_try.tv_sec = 0; + host[0].last_start_try.tv_usec = 0; + host[0].next_start_try.tv_sec = 0; + host[0].next_start_try.tv_usec = 0; InitIOBuffer (&host[0].comms_buffer, 0x100); Index: trunk/Ohana/src/opihi/pcontrol/StartHost.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/StartHost.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/StartHost.c (revision 25872) @@ -21,21 +21,24 @@ if (VerboseMode()) gprint (GP_ERR, "failure to start %s\n", host[0].hostname); gettimeofday (&now, (void *) NULL); - if (ZTIME(host[0].nexttry) || ZTIME(host[0].lasttry)) { + if (ZTIME(host[0].next_start_try) || ZTIME(host[0].last_start_try)) { /* reset retry period if either is zero */ delta = RETRY_BASE; } else { - delta = 2*DTIME (host[0].nexttry, host[0].lasttry); + delta = MAX(1.0, 2*DTIME (host[0].next_start_try, host[0].last_start_try)); } - host[0].nexttry.tv_sec = now.tv_sec + delta; - host[0].nexttry.tv_usec = now.tv_usec; - host[0].lasttry.tv_sec = now.tv_sec; - host[0].lasttry.tv_usec = now.tv_usec; + host[0].next_start_try.tv_sec = now.tv_sec + delta; + host[0].next_start_try.tv_usec = now.tv_usec; + host[0].last_start_try.tv_sec = now.tv_sec; + host[0].last_start_try.tv_usec = now.tv_usec; PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); return (FALSE); } - host[0].nexttry.tv_sec = 0; - host[0].nexttry.tv_usec = 0; - host[0].lasttry.tv_sec = 0; - host[0].lasttry.tv_usec = 0; + host[0].next_start_try.tv_sec = 0; + host[0].next_start_try.tv_usec = 0; + host[0].last_start_try.tv_sec = 0; + host[0].last_start_try.tv_usec = 0; + + // set the connection time + gettimeofday (&host[0].connect_time, (void *) NULL); host[0].stdin_fd = stdio[0]; Index: trunk/Ohana/src/opihi/pcontrol/StopHosts.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/StopHosts.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/StopHosts.c (revision 25872) @@ -69,15 +69,24 @@ } -int StopHost (Host *host) { +int StopHost (Host *host, int mode) { int status; - status = PclientCommand (host, "exit", "Goodbye", PCONTROL_RESP_STOP_HOST); + switch (mode) { + case PCONTROL_HOST_DOWN: + status = PclientCommand (host, "exit", "Goodbye", PCONTROL_RESP_DOWN_HOST); + break; + case PCONTROL_HOST_OFF: + status = PclientCommand (host, "exit", "Goodbye", PCONTROL_RESP_STOP_HOST); + break; + default: + ABORT ("programming error: invalid StopHost mode"); + } /* check on success of pclient command */ switch (status) { case PCLIENT_DOWN: - // XXX this is the desired result in any case, so ignore it - break; + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + return (TRUE); case PCLIENT_GOOD: @@ -96,4 +105,11 @@ OffHost (host); + HarvestHost (host[0].pid); + return (TRUE); +} + +int DownHostResponse (Host *host) { + + DownHost (host); HarvestHost (host[0].pid); return (TRUE); @@ -134,7 +150,5 @@ case 0: - gprint (GP_ERR, "child did not exit??"); - abort (); - /** put back in IDLE state? **/ + gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung"); break; Index: trunk/Ohana/src/opihi/pcontrol/host.c =================================================================== --- trunk/Ohana/src/opihi/pcontrol/host.c (revision 25871) +++ trunk/Ohana/src/opihi/pcontrol/host.c (revision 25872) @@ -53,8 +53,8 @@ } /* reset time, place back on ALLHOSTS stack */ - host[0].nexttry.tv_sec = 0; - host[0].nexttry.tv_usec = 0; - host[0].lasttry.tv_sec = 0; - host[0].lasttry.tv_usec = 0; + host[0].next_start_try.tv_sec = 0; + host[0].next_start_try.tv_usec = 0; + host[0].last_start_try.tv_sec = 0; + host[0].last_start_try.tv_usec = 0; PushStack (AllHosts, STACK_BOTTOM, host, host[0].HostID, host[0].hostname); return (TRUE);