Index: trunk/Ohana/src/libohana/Makefile
===================================================================
--- trunk/Ohana/src/libohana/Makefile	(revision 33645)
+++ trunk/Ohana/src/libohana/Makefile	(revision 33646)
@@ -38,4 +38,6 @@
 $(SRC)/Fread.$(ARCH).o		 \
 $(SRC)/IOBufferOps.$(ARCH).o	 \
+$(SRC)/memstr.$(ARCH).o	 \
+$(SRC)/rconnect.$(ARCH).o	 \
 $(SRC)/CommOps.$(ARCH).o	 \
 $(SRC)/isolate_elements.$(ARCH).o	 \
Index: trunk/Ohana/src/libohana/include/ohana.h
===================================================================
--- trunk/Ohana/src/libohana/include/ohana.h	(revision 33645)
+++ trunk/Ohana/src/libohana/include/ohana.h	(revision 33646)
@@ -215,4 +215,7 @@
 */
 # define OHANA_WHITESPACE(c)(((c) == 0x09) || ((c) == 0x0a) || ((c) == 0x0b) || ((c) == 0x0b) || ((c) == 0x0c) || ((c) == 0x0d) || ((c) == 0x20))
+# define OHANA_ASSERT(LOGIC,...) { if (!(LOGIC)) { fprintf (stderr, __VA_ARGS__); abort(); } }
+# define myAssert(LOGIC,...) { if (!(LOGIC)) { fprintf (stderr, __VA_ARGS__); abort(); } }
+# define myAbort(MSG) { fprintf (stderr, "%s\n", MSG); abort(); }
 
 // sorting is now defined as a macro call
@@ -236,6 +239,11 @@
 char   *strncreate             PROTO((char *string, int n));
 int     scan_line              PROTO((FILE *f, char *line)); 
+int     scan_line_maxlen       PROTO((FILE *f, char *line, int maxlen)); 
 int     dparse                 PROTO((double *X, int NX, char *line));
 int     dparse_csv             PROTO((double *X, int NX, char *line));
+int     iparse                 PROTO((int *X, int NX, char *line));
+int     iparse_csv             PROTO((int *X, int NX, char *line));
+int     tparse                 PROTO((time_t *X, int NX, char *line));
+int     tparse_csv             PROTO((time_t *X, int NX, char *line));
 int     fparse                 PROTO((float *X, int NX, char *line));
 int     get_argument           PROTO((int argc, char **argv, char *arg));
@@ -256,4 +264,5 @@
 int     check_dir_access       PROTO((char *path, int verbose));
 int     check_file_exec        PROTO((char *filename));
+char   *abspath                PROTO((char *oldpath, int maxlength));
 
 /* in glockfile.c */
@@ -323,4 +332,7 @@
 off_t   Fwrite                 PROTO((void *ptr, off_t size, off_t nitems, FILE *f, char *type));
 
+char   *memstr                 PROTO((char *m1, char *m2, int n));
+int     write_fmt              PROTO((int fd, char *format, ...));
+
 char   **isolate_elements      PROTO((int argc, char **argv, int *nstack));
 
@@ -332,4 +344,15 @@
 // gprint gets a stub implementation in libohana. opihi implements the real one.
 int    gprint (gpDest dest, char *format, ...);
+
+// rconnect is used to run remote programs with a pipe for communication
+typedef enum {
+  RCONNECT_ERR_NONE    = 0,
+  RCONNECT_ERR_EXEC    = 1,
+  RCONNECT_ERR_PIPE    = 2,
+  RCONNECT_ERR_CONNECT = 3,
+} RconnectErrors;
+
+void rconnect_set_timeout (int timeout);
+int rconnect (char *command, char *hostname, char *shell, int *stdio, int *errorInfo, int doHandshake);
 
 /*
Index: trunk/Ohana/src/libohana/include/ohana_allocate.h
===================================================================
--- trunk/Ohana/src/libohana/include/ohana_allocate.h	(revision 33645)
+++ trunk/Ohana/src/libohana/include/ohana_allocate.h	(revision 33646)
@@ -12,5 +12,12 @@
 OhanaMemstats ohana_memstats (int mode);
 
+/* EAM 2012.01.18 : comparative tests on my desktop 'pikake' (Intel Core 2 Quad) show a
+ * modest speed loss when using the Ohana memory managment stuff.  the tests runs a ~1.2M
+ * allocations followed by the same number of frees.  with the Ohana memory code, these
+ * ran at ~0.69 sec per block; without the time was ~0.56 sec per block
+ */
+
 /* default is to use basic system memory functions */
+
 # ifdef OHANA_MEMORY
 
Index: trunk/Ohana/src/libohana/src/findexec.c
===================================================================
--- trunk/Ohana/src/libohana/src/findexec.c	(revision 33645)
+++ trunk/Ohana/src/libohana/src/findexec.c	(revision 33646)
@@ -404,2 +404,29 @@
   return TRUE;
 }
+
+// ohana-based equivalent to 'realpath'.  realpath does not guarantee an absolute path (eg on Solaris), 
+// just a 'cannonical path'.  I don't care about resolving out links, just ensuring an absolute path
+// also, realpath is a bit ambiguous on the maxlength argument
+char *abspath (char *oldpath, int maxlength) {
+
+  if (!oldpath) return NULL;
+
+  if (oldpath[0] == '/') {
+    char *newpath = strcreate (oldpath);
+    return newpath;
+  }
+
+  char *cwd  = getcwd(NULL, maxlength);
+  if (cwd == NULL) {
+    // XXX need an error reporting function...
+    fprintf (stderr, "error getting cwd (longer than %d chars?)\n", maxlength);
+    return NULL;
+  }
+
+  char *newpath = NULL;
+
+  int Nbytes = strlen(cwd) + strlen(oldpath) + 2;
+  ALLOCATE (newpath, char, Nbytes);
+  snprintf (newpath, Nbytes, "%s/%s", cwd, oldpath);
+  return newpath;
+}
Index: trunk/Ohana/src/libohana/src/memstr.c
===================================================================
--- trunk/Ohana/src/libohana/src/memstr.c	(revision 33646)
+++ trunk/Ohana/src/libohana/src/memstr.c	(revision 33646)
@@ -0,0 +1,35 @@
+# include "ohana.h"
+
+/* memstr returns a view, not an allocated string : don't free */
+/* returns pointer to start of m2 in m1, or NULL if failure */ 
+char *memstr (char *m1, char *m2, int n) {
+
+  int i, N;
+
+  N = strlen (m2);
+  for (i = 0; (i < n - N + 1) && memcmp (m1, m2, N); i++, m1++);
+  if (memcmp (m1, m2, N)) return (NULL);
+  return (m1);
+
+}
+
+/* formatted write statement, with intelligent allocation */
+int write_fmt (int fd, char *format, ...) {
+
+  int Nbyte, status;
+  char tmp, *line;
+  va_list argp;  
+
+  va_start (argp, format);
+  Nbyte = vsnprintf (&tmp, 0, format, argp);
+  va_end (argp);
+
+  va_start (argp, format);
+  ALLOCATE (line, char, Nbyte + 1);
+  vsnprintf (line, Nbyte + 1, format, argp);
+  status = write (fd, line, strlen(line));
+  va_end (argp);
+
+  free (line);
+  return (status);
+}
Index: trunk/Ohana/src/libohana/src/rconnect.c
===================================================================
--- trunk/Ohana/src/libohana/src/rconnect.c	(revision 33646)
+++ trunk/Ohana/src/libohana/src/rconnect.c	(revision 33646)
@@ -0,0 +1,170 @@
+# include <ohana.h>
+# include <sys/types.h>
+# include <sys/wait.h>
+
+# define DEBUG 1
+
+/* connect to host with the command and start the shell: (command) hostname (shell)
+   eg: ssh hostname pclient
+   stdio is an array of file descriptors for the I/O to the remote shell:
+   stdio[3]: 0 == stdin, 1 == stdout, 2 == stderr
+
+   The handshake test assumes that the remote shell can accept an echo command.  If not,
+   the user must do their own handshake test.
+
+   return value is the PID of the remote job, or FALSE on failure
+   in case of failure, the error condition is provided in errorInfo.
+
+   
+*/
+
+/* connection can take a while, allow up to 5 sec by default */
+static int CONNECT_TIMEOUT = 5000;
+
+void rconnect_set_timeout (int timeout) {
+  CONNECT_TIMEOUT = timeout;
+}
+
+int rconnect (char *command, char *hostname, char *shell, int *stdio, int *errorInfo, int doHandshake) {
+
+  int i = 0, stdin_fd[2], stdout_fd[2], stderr_fd[2], status;
+  int result, waitstatus;
+  pid_t pid;
+  char *p;
+  char **argv;
+  IOBuffer buffer;
+  struct timespec request, remain;
+
+  if (errorInfo) *errorInfo = RCONNECT_ERR_NONE;
+
+  myAssert (command != NULL,  "command is NULL");
+  myAssert (hostname != NULL, "hostname is NULL");
+  myAssert (shell != NULL,    "shell is NULL");
+  myAssert (stdio != NULL,    "stdio is NULL");
+
+  bzero (stdin_fd,  2*sizeof(int));
+  bzero (stdout_fd, 2*sizeof(int));
+  bzero (stderr_fd, 2*sizeof(int));
+
+  if (pipe (stdin_fd)  < 0) goto pipe_error;
+  if (pipe (stdout_fd) < 0) goto pipe_error;
+  if (pipe (stderr_fd) < 0) goto pipe_error;
+
+  char *flag = "-x";
+
+  ALLOCATE (argv, char *, 5);
+  argv[0] = command;
+  argv[1] = flag;
+  argv[2] = hostname;
+  argv[3] = shell;
+  argv[4] = 0;
+
+  pid = fork ();
+  if (!pid) { /* must be child process */
+    if (DEBUG) fprintf (stderr, "starting remote connection to %s...", hostname);
+
+    /* close the other ends of the pipes */
+    close (stdin_fd[1]);
+    close (stdout_fd[0]);
+    close (stderr_fd[0]);
+
+    /* tie our ends of the pipes to stdin, stdout, stderr */
+    dup2 (stdin_fd[0],  STDIN_FILENO);
+    dup2 (stdout_fd[1], STDOUT_FILENO);
+    dup2 (stderr_fd[1], STDERR_FILENO);
+
+    /* set all three unblocking */
+    setvbuf (stdin,  (char *) NULL, _IONBF, BUFSIZ);
+    setvbuf (stdout, (char *) NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, (char *) NULL, _IONBF, BUFSIZ);
+
+    status = execvp (argv[0], argv);
+    if (DEBUG) fprintf (stderr, "error starting remote shell process\n");
+    if (errorInfo) *errorInfo = RCONNECT_ERR_EXEC;
+    return FALSE;
+  }
+  free (argv);
+
+  /* close the other ends of the pipes */
+  close (stdin_fd[0]);  stdin_fd[0]  = 0;
+  close (stdout_fd[1]); stdout_fd[1] = 0;
+  close (stderr_fd[1]); stderr_fd[1] = 0;
+
+  /* make the pipes non-blocking */
+  fcntl (stdin_fd[1],  F_SETFL, O_NONBLOCK);
+  fcntl (stdout_fd[0], F_SETFL, O_NONBLOCK);
+  fcntl (stderr_fd[0], F_SETFL, O_NONBLOCK);
+
+  if (doHandshake) {
+    /* perform handshake with shell to verify alive & running */
+    InitIOBuffer (&buffer, 0x100);
+
+    /* send handshake command */
+    status = write_fmt (stdin_fd[1], "echo CONNECTED\n");
+    if ((status == -1) && (errno == EPIPE)) goto connect_error;
+
+    /* try to get evidence connection is alive - wait upto a few seconds */
+    p = NULL;
+    status = -1;
+    for (i = 0; (i < CONNECT_TIMEOUT) && (status != 0) && (p == NULL); i++) {
+      status = ReadtoIOBuffer (&buffer, stdout_fd[0]);
+      p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer);
+      usleep (10000); // wait for client to be connected
+    }
+    if (status == 0) goto connect_error;
+    if (status == -1) goto connect_error;
+    if (DEBUG) fprintf (stderr, "%d cycles to connect\n", i);
+    FreeIOBuffer (&buffer);
+  }
+
+  if (DEBUG) fprintf (stderr, "Connected\n");
+
+  stdio[0] = stdin_fd[1];
+  stdio[1] = stdout_fd[0];
+  stdio[2] = stderr_fd[0];
+
+  if (errorInfo) *errorInfo = RCONNECT_ERR_NONE;
+  return (pid);
+
+pipe_error:
+  perror ("pipe error:");
+  if (errorInfo) *errorInfo = RCONNECT_ERR_PIPE;
+  goto close_pipes;
+
+connect_error:
+  if (DEBUG) fprintf (stderr, "error while connecting, status: %d, ncycles: %d\n", status, i);
+
+  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
+  request.tv_sec = 0;
+  request.tv_nsec = 100000;
+
+  /* harvest the child process: kill & wait (< 100 ms) for exit */
+  kill (pid, SIGKILL);
+  result = waitpid (pid, &waitstatus, WNOHANG);
+  for (i = 0; (i < 50) && (result == 0); i++) {
+    nanosleep (&request, &remain);
+    result = waitpid (pid, &waitstatus, WNOHANG);
+  }
+
+  if ((result == -1) && (errno != ECHILD)) {
+    fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno);
+    abort();
+  }
+  if (result == 0) {
+    if (DEBUG) fprintf (stderr, "child did not exit (rconnect)??");
+  }
+  if (result > 0) {
+    myAssert (result == pid, "waitpid error: mis-matched PID (%d vs %d).  programming error", result, pid);
+    myAssert (!WIFSTOPPED(waitstatus), "waitpid returns 'stopped': programming error");
+  }
+  if (errorInfo) *errorInfo = RCONNECT_ERR_CONNECT;
+
+close_pipes:
+  if (stdin_fd[0]  != 0) close (stdin_fd[0]);
+  if (stdin_fd[1]  != 0) close (stdin_fd[1]);
+  if (stdout_fd[0] != 0) close (stdout_fd[0]);
+  if (stdout_fd[1] != 0) close (stdout_fd[1]);
+  if (stderr_fd[0] != 0) close (stderr_fd[0]);
+  if (stderr_fd[1] != 0) close (stderr_fd[1]);
+  return (FALSE);
+}
Index: trunk/Ohana/src/libohana/src/string.c
===================================================================
--- trunk/Ohana/src/libohana/src/string.c	(revision 33645)
+++ trunk/Ohana/src/libohana/src/string.c	(revision 33646)
@@ -143,4 +143,24 @@
 }
 
+int scan_line_maxlen (FILE *f, char *line, int maxlen) {
+
+  int i, status;
+  char c;
+  
+  status = EOF + 1;
+  
+  for (i = 0, c = 0; (c != '\n') && (status != EOF) && (i < maxlen); i++) {
+    status = fscanf (f, "%c", &c);
+    line[i] = c;
+  }
+  line[i - 1] = 0;  /* this could make things crash! */
+
+  if (i > 1) {
+    status = EOF + 1;
+  }
+
+  return (status);
+}
+
 char *_parse_nextword (char *string) {
 
@@ -203,4 +223,76 @@
   if (word[0] == '-') return (-1);
   return (1);
+}
+
+int iparse (int *X, int NX, char *line) {
+
+  int i;
+  char *word;
+  char *ptr;
+
+  word = line;
+  for (i = 0; i < NX - 1; i++)
+    word = _parse_nextword (word);
+
+  *X = strtol (word, &ptr, 0);
+  if (ptr == word) return (FALSE);
+  if (word[0] == '-') return (-1);
+  return (1);
+}
+
+int iparse_csv (int *X, int NX, char *line) {
+
+  int i;
+  char *word;
+  char *ptr;
+
+  word = line;
+  for (i = 0; i < NX - 1; i++)
+    word = _parse_nextword_csv (word);
+  
+  if (word[0] == '"') word[0] = ' ';
+  if (word[0] == ',') {
+      *X = 0;
+      return 1;
+  }
+
+  *X = strtol (word, &ptr, 0);
+  if (ptr == word) return (FALSE);
+  if (word[0] == '-') return (-1);
+  return (1);
+}
+
+int tparse (time_t *X, int NX, char *line) {
+
+  int i, status;
+  char *word;
+
+  word = line;
+  for (i = 0; i < NX - 1; i++)
+    word = _parse_nextword (word);
+
+  status = ohana_str_to_time (word, X);
+  if (!status) return (FALSE);
+  return (TRUE);
+}
+
+int tparse_csv (time_t *X, int NX, char *line) {
+
+  int i, status;
+  char *word;
+
+  word = line;
+  for (i = 0; i < NX - 1; i++)
+    word = _parse_nextword_csv (word);
+  
+  if (word[0] == '"') word[0] = ' ';
+  if (word[0] == ',') {
+      *X = 0;
+      return 1;
+  }
+
+  status = ohana_str_to_time (word, X);
+  if (!status) return (FALSE);
+  return (TRUE);
 }
 
Index: trunk/Ohana/src/libohana/test/memspeed.c
===================================================================
--- trunk/Ohana/src/libohana/test/memspeed.c	(revision 33646)
+++ trunk/Ohana/src/libohana/test/memspeed.c	(revision 33646)
@@ -0,0 +1,86 @@
+
+# define TIMESTAMP(TIME) \
+    gettimeofday (&stop, (void *) NULL);	\
+    dtime = DTIME (stop, start);		\
+    TIME += dtime;				\
+    gettimeofday (&start, (void *) NULL);
+
+
+  // XXX quick and stupid test:
+  if (1) {
+    int i, j; 
+    off_t *Nlist, *NLIST, **mlist;
+    off_t *NlistI, *NLISTI, **mlistI;
+    int **clist;
+    int **clistI;
+    float dtime;
+    float time1 = 0.0;
+    float time2 = 0.0;
+
+    int Nmosaic = 10000;
+    int Nimage = 600000;
+
+    for (j = 0; j < 19000; j++) {
+
+      // mosaic indexes
+      ALLOCATE (Nlist, off_t,   Nmosaic);
+      ALLOCATE (NLIST, off_t,   Nmosaic);
+      ALLOCATE (clist, int *,   Nmosaic);
+      ALLOCATE (mlist, off_t *, Nmosaic);
+    
+      for (i = 0; i < Nmosaic; i++) {
+	Nlist[i] = 0;
+	NLIST[i] = 100;
+	ALLOCATE (clist[i], int,   NLIST[i]);
+	ALLOCATE (mlist[i], off_t, NLIST[i]);
+      }
+
+      // image indexes
+      ALLOCATE (NlistI, off_t,   Nimage);
+      ALLOCATE (NLISTI, off_t,   Nimage);
+      ALLOCATE (clistI, int *,   Nimage);
+      ALLOCATE (mlistI, off_t *, Nimage);
+    
+      for (i = 0; i < Nimage; i++) {
+	NlistI[i] = 0;
+	NLISTI[i] = 100;
+	ALLOCATE (clistI[i], int,   NLISTI[i]);
+	ALLOCATE (mlistI[i], off_t, NLISTI[i]);
+      }
+
+      TIMESTAMP(time1);
+
+      // free mosaic indexes
+      for (i = 0; i < Nmosaic; i++) {
+	free (clist[i]);
+	free (mlist[i]);
+      }
+      free (Nlist);
+      free (NLIST);
+      free (clist);
+      free (mlist);
+
+      // free image indexes
+      for (i = 0; i < Nimage; i++) {
+	free (clistI[i]);
+	free (mlistI[i]);
+      }
+      free (NlistI);
+      free (NLISTI);
+      free (clistI);
+      free (mlistI);
+
+      TIMESTAMP(time2);
+
+      if (j % 10 == 0) {
+	fprintf (stderr, "done with %d\n", j);
+	fprintf (stderr, "time1  %f : init mosaic\n", time1);
+	fprintf (stderr, "time2  %f : free mosaic\n", time2);
+      }
+    }
+    
+    fprintf (stderr, "time1  %f : init mosaic\n", time1);
+    fprintf (stderr, "time2  %f : free mosaic\n", time2);
+    exit (1);
+  }
+