Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.basic/shell.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.basic/shell.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.basic/shell.c	(revision 33308)
@@ -13,5 +13,5 @@
   int exit_status;
   int wait_status;
-  int result, length;
+  int result;
   char **args, *shell;
   struct timeval start, now;
@@ -33,16 +33,5 @@
   args[0] = shell;
   args[1] = cmdflag;
-
-  length = 0;
-  for (i = 1; i < argc; i++) {
-    length += strlen(argv[i]) + 1;
-  }
-  
-  ALLOCATE (args[2], char, length);
-  args[2][0] = 0;
-  for (i = 1; i < argc; i++) {
-    strcat (args[2], argv[i]);
-    if (i < argc - 1) strcat (args[2], " ");
-  }
+  args[2] = paste_args (argc - 1, &argv[1]);
   args[3] = NULL;
 
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/write_vectors.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/write_vectors.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/write_vectors.c	(revision 33308)
@@ -37,21 +37,8 @@
   }
 
-  /* open file for outuput */
-  if (append) {
-    f = fopen (argv[1], "a");
-  } else {
-    f = fopen (argv[1], "w");
-  }
-  if (f == (FILE *) NULL) {
-    gprint (GP_ERR, "can't open file for write\n");
-    return (FALSE);
-  }
-
   /* find number of output vectors */
   Nvec = (argc - 2);
   if (Nvec < 1) {
     gprint (GP_ERR, "USAGE: write (file) vector vector ...\n");
-    fclose (f);
-    fflush (f);
     return (FALSE);
   }
@@ -64,6 +51,4 @@
       gprint (GP_ERR, "USAGE: write (file) vector vector ...\n");
       free (vec);
-      fclose (f);
-      fflush (f);
       return (FALSE);    
     }
@@ -76,6 +61,4 @@
       gprint (GP_ERR, "error: vectors must all be the same size\n");
       free (vec);
-      fclose (f);
-      fflush (f);
       return (FALSE);    
     }
@@ -83,102 +66,17 @@
 
   if (FITS) {
-    char *tformat = NULL;
-    Header header;
-    Matrix matrix;
-    Header theader;
-    FTable ftable;
+    int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, format);
+    free (vec);
+    return status;
+  }
 
-    if (!append) {
-      gfits_init_header (&header);
-      header.extend = TRUE;
-      gfits_create_header (&header);
-      gfits_create_matrix (&header, &matrix);
-    }
-
-    gfits_create_table_header (&theader, "BINTABLE", FITS);
-
-    ALLOCATE (tformat, char, 2*Nvec);
-    if (format) {
-      // the bintable format string can only define the byte-width of each field.  valid output columns are currently:
-      // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double).
-      // the format string is just the sequence of types, eg: LIIJEED
-      // validate the format string
-      char *ptr = format;
-      for (j = 0; j < Nvec; j++) {
-	while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
-	if (*ptr == 0) {
-	  gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format);
-	  goto escape;
-	}
-	if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) {
-	  gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr);
-	  goto escape;
-	}
-	tformat[2*j + 0] = *ptr;
-	tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
-	ptr ++;
-      }
-      while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
-      if (*ptr) {
-	gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format);
-	goto escape;
-      }
-    } else {
-      for (j = 0; j < Nvec; j++) {
-	// if the format is not defined, just use the native byte-widths
-	tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J';
-	tformat[2*j + 1] = 0;
-      }
-    }
-
-    // define the columns of the table.  XXX NOTE: we cannot have duplicate names in
-    // output table (because the data goes to the named column below).  need to enforce
-    // this somehow
-    for (j = 0; j < Nvec; j++) {
-      gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0);
-    }
-    free (tformat);
-
-    // generate the output array that carries the data
-    gfits_create_table (&theader, &ftable);
-
-    // add the vectors to the output array
-    for (j = 0; j < Nvec; j++) {
-      if (vec[j][0].type == OPIHI_FLT) {
-	gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements);
-      } else {
-	gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements);
-      }
-    }
-
-    if (!append) {
-      gfits_fwrite_header  (f, &header);
-      gfits_fwrite_matrix  (f, &matrix);
-      gfits_free_header (&header);
-      gfits_free_matrix (&matrix);
-    }
-    gfits_fwrite_Theader (f, &theader);
-    gfits_fwrite_table  (f, &ftable);
-
-    gfits_free_header (&theader);
-    gfits_free_table (&ftable);
-
-    fclose (f);
-    free (vec);
-    fflush (f);
-    return (TRUE);
-
-  escape:
-    if (!append) {
-      gfits_free_header (&header);
-      gfits_free_matrix (&matrix);
-    }
-    gfits_free_header (&theader);
-    gfits_free_table (&ftable);
-
-    if (tformat) free (tformat);
-    fclose (f);
-    free (vec);
-    fflush (f);
+  /* open file for outuput */
+  if (append) {
+    f = fopen (argv[1], "a");
+  } else {
+    f = fopen (argv[1], "w");
+  }
+  if (f == (FILE *) NULL) {
+    gprint (GP_ERR, "can't open file for write\n");
     return (FALSE);
   }
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo.c.in
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo.c.in	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo.c.in	(revision 33308)
@@ -45,4 +45,8 @@
   }
 
+  // dvo_client uses the following; they should be NULL for dvo
+  HOST_ID = 0;
+  HOSTDIR = NULL;
+
   return;
 }
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo_client.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo_client.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo_client.c	(revision 33308)
@@ -4,4 +4,6 @@
 void program_init (int *argc, char **argv) {
   
+  int N;
+
   /* load the commands used by this implementation */
   InitBasic ();
@@ -23,4 +25,19 @@
     set_int_variable ("MODULES:n", 1);
     free (modules);
+  }
+
+  // dvo_client should have 2 standard arguments: -hostID and -hostdir
+  HOST_ID = 0;
+  if ((N = get_argument (*argc, argv, "-hostID"))) {
+    remove_argument (N, argc, argv);
+    HOST_ID = atoi (argv[N]);;
+    remove_argument (N, argc, argv);
+  }
+
+  HOSTDIR = NULL;
+  if ((N = get_argument (*argc, argv, "-hostdir"))) {
+    remove_argument (N, argc, argv);
+    HOSTDIR = strcreate (argv[N]);;
+    remove_argument (N, argc, argv);
   }
 
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c	(revision 33308)
@@ -1,3 +1,8 @@
 # include "dvoshell.h"
+
+# define DEBUG 1
+# define PARALLEL_MANUAL 1
+# define PARALLEL_SERIAL 0
+# define MAX_PATH_LENGTH 1024
 
 int mextract (int argc, char **argv) {
@@ -34,11 +39,59 @@
   }
 
+  int PARALLEL = FALSE;
+  if ((N = get_argument (argc, argv, "-parallel"))) {
+    remove_argument (N, &argc, argv);
+    PARALLEL = TRUE;
+  }
+
+  // dump results directly to fits file (esp for parallel dvo)
+  char *ResultFile = NULL;
+  if ((N = get_argument (argc, argv, "-result"))) {
+    remove_argument (N, &argc, argv);
+    ResultFile = strcreate(argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   // XXX not yet thought through (where are these set?)
   if (PARALLEL && !HOST_ID) {
-    HostTableLaunchJobs (CATDIR, argc, argv);
-    HostTableWaitJobs (table, __FILE__, __LINE__);
+    // load the list of hosts
+    SkyTable *sky = GetSkyTable();
+    if (!sky) return FALSE;
+
+    char *CATDIR = GetCATDIR ();
+    if (!CATDIR) return FALSE;
+
+    HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+    if (!table) {
+      fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+      return FALSE;
+    }    
+
+    // other things I need to append?
+    char *basecmd = paste_args (argc, argv);
+    double Rmin, Rmax, Dmin, Dmax;
+    get_skyregion (&Rmin, &Rmax, &Dmin, &Dmax);
+
+    char tmp;
+    char *command = NULL;
+    int length = snprintf (&tmp, 0, "%s -D CATDIR %s -skyregion %f %f %f %f", basecmd, CATDIR, Rmin, Rmax, Dmin, Dmax);
+
+    ALLOCATE (command, char, length);
+    snprintf (command, length, "%s -D CATDIR %s -skyregion %f %f %f %f", basecmd, CATDIR, Rmin, Rmax, Dmin, Dmax);
+
+    // launch this command remotely
+    HostTableLaunchJobs (table, command);
+    free (command);
+
+    if (PARALLEL_MANUAL) {
+      fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
+      getchar();
+    }
+    if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+      HostTableWaitJobs (table, __FILE__, __LINE__);
+    }
 
     // load fields from file
-    ReadVectorTables
+    // ReadVectorTables
     return TRUE;
   }
@@ -132,9 +185,11 @@
   for (i = 0; (i < skylist[0].Nregions) && !interrupt; i++) {
 
-    // XXX parallel case, wrong client
+    // parallel case, wrong client
     if (HOST_ID && (HOST_ID != skylist[0].regions[i]->hostID)) continue;
 
     /* lock, load, unlock catalog */
-    catalog.filename = skylist[0].filename[i];
+    char hostfile[1024];
+    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+    catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i];
     catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
     catalog.Nsecfilt = Nsecfilt;
@@ -205,8 +260,9 @@
   }
 
-  if (HOST_ID) {
-    // write table
-    WriteVectorTable (vec);
-    return TRUE;
+  if (ResultFile) {
+    // write vectors to a table
+    // char *resultFile?
+    int status = WriteVectorTableFITS (ResultFile, "MEXTRACT", vec, Nreturn, FALSE, NULL);
+    if (!status) goto escape;
   }
 
@@ -222,4 +278,5 @@
 
 escape:
+  if (ResultFile) free (ResultFile);
   if (vec) free (vec);
   free (values);
@@ -349,2 +406,45 @@
   return (FALSE);
 }
+
+int HostTableLaunchJobs (HostTable *table, char *basecmd) {
+
+  int i;
+  for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    char resultFile[MAX_PATH_LENGTH];
+    snprintf (resultFile, MAX_PATH_LENGTH, "%s/dvo.results.fits", table->hosts[i].pathname);
+
+    char command[1024];
+    snprintf (command, 1024, "%s -result %s -hostID %d -hostdir %s", basecmd, resultFile, table->hosts[i].hostID, table->hosts[i].pathname);
+
+    fprintf (stderr, "command: %s\n", command);
+
+    if (PARALLEL_MANUAL) {
+      continue;
+    }
+
+    if (PARALLEL_SERIAL) {
+      int status = system (command);
+      if (status) {
+	fprintf (stderr, "ERROR running relphot_client\n");
+	exit (2);
+      }
+    } else {
+      // launch the job on the remote machine (no handshake)
+      int errorInfo = 0;
+      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
+      if (!pid) {
+	if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+	exit (1);
+      }
+      table->hosts[i].pid = pid; // save for future reference
+    }
+  }
+  return TRUE;
+}
+
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h	(revision 33308)
@@ -127,4 +127,5 @@
 int           ListVectors           PROTO((void));
 Vector       *SelectVector          PROTO((char *name, int mode, int verbose));
+int           WriteVectorTableFITS  PROTO((char *filename, char *extname, Vector **vec, int Nvec, int append, char *format));
 
 /* buffer handling */
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvoshell.h
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvoshell.h	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvoshell.h	(revision 33308)
@@ -292,4 +292,8 @@
 } CMPstars;
 
+/** some globals used particularly by DVO_CLIENT **/
+int   HOST_ID;
+char *HOSTDIR;
+
 /*** dvo prototypes ***/
 int           DetermineTypeCode     PROTO((Average *average, Measure *measure, int code));
@@ -354,4 +358,6 @@
 int wordhash (char *word);
 
+int HostTableLaunchJobs (HostTable *table, char *basecmd);
+
 #ifdef NOT_MOVED_TO_LIBDVO
 // dvo DB field functions
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/shell.h
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/shell.h	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/shell.h	(revision 33308)
@@ -148,4 +148,5 @@
 char         *opihi_append              PROTO((char *output, int *Noutput, char *start, char *stop));
 void          interpolate_slash         PROTO((char *line));
+char         *paste_args                PROTO((int argc, char **argv));
 
 /* macro functions (mapped to commands) */
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c	(revision 33308)
@@ -288,2 +288,117 @@
 }
   
+// write a set of vectors to a FITS file
+int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *format) {
+  
+  char *tformat = NULL;
+  Header header;
+  Matrix matrix;
+  Header theader;
+  FTable ftable;
+
+  int j;
+  FILE *f = NULL;
+
+  /* open file for outuput */
+  if (append) {
+    f = fopen (filename, "a");
+  } else {
+    f = fopen (filename, "w");
+  }
+  if (f == (FILE *) NULL) {
+    gprint (GP_ERR, "can't open file for write\n");
+    return (FALSE);
+  }
+
+  if (!append) {
+    gfits_init_header (&header);
+    header.extend = TRUE;
+    gfits_create_header (&header);
+    gfits_create_matrix (&header, &matrix);
+  }
+
+  gfits_create_table_header (&theader, "BINTABLE", extname);
+
+  ALLOCATE (tformat, char, 2*Nvec);
+  if (format) {
+    // the bintable format string can only define the byte-width of each field.  valid output columns are currently:
+    // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double).
+    // the format string is just the sequence of types, eg: LIIJEED
+    // validate the format string
+    char *ptr = format;
+    for (j = 0; j < Nvec; j++) {
+      while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
+      if (*ptr == 0) {
+	gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format);
+	goto escape;
+      }
+      if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) {
+	gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr);
+	goto escape;
+      }
+      tformat[2*j + 0] = *ptr;
+      tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
+      ptr ++;
+    }
+    while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
+    if (*ptr) {
+      gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format);
+      goto escape;
+    }
+  } else {
+    for (j = 0; j < Nvec; j++) {
+      // if the format is not defined, just use the native byte-widths
+      tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J';
+      tformat[2*j + 1] = 0;
+    }
+  }
+
+  // define the columns of the table.  XXX NOTE: we cannot have duplicate names in
+  // output table (because the data goes to the named column below).  need to enforce
+  // this somehow
+  for (j = 0; j < Nvec; j++) {
+    gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0);
+  }
+  free (tformat);
+
+  // generate the output array that carries the data
+  gfits_create_table (&theader, &ftable);
+
+  // add the vectors to the output array
+  for (j = 0; j < Nvec; j++) {
+    if (vec[j][0].type == OPIHI_FLT) {
+      gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements);
+    } else {
+      gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements);
+    }
+  }
+
+  if (!append) {
+    gfits_fwrite_header  (f, &header);
+    gfits_fwrite_matrix  (f, &matrix);
+    gfits_free_header (&header);
+    gfits_free_matrix (&matrix);
+  }
+  gfits_fwrite_Theader (f, &theader);
+  gfits_fwrite_table  (f, &ftable);
+
+  gfits_free_header (&theader);
+  gfits_free_table (&ftable);
+
+  fclose (f);
+  fflush (f);
+  return (TRUE);
+
+ escape:
+  if (!append) {
+    gfits_free_header (&header);
+    gfits_free_matrix (&matrix);
+  }
+  gfits_free_header (&theader);
+  gfits_free_table (&ftable);
+
+  if (tformat) free (tformat);
+  fclose (f);
+  fflush (f);
+  return (FALSE);
+}
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/string.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/string.c	(revision 33307)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/string.c	(revision 33308)
@@ -301,2 +301,23 @@
   *out = *in;
 }
+
+// paste together argv[0] .. argv[N] into a single string
+char *paste_args (int argc, char **argv) {
+
+  int i;
+
+  int length = 0;
+  for (i = 0; i < argc; i++) {
+    length += strlen(argv[i]) + 1;
+  }
+  
+  char *string = NULL;
+  ALLOCATE (string, char, length);
+  string[0] = 0;
+  for (i = 0; i < argc; i++) {
+    strcat (string, argv[i]);
+    if (i < argc - 1) strcat (string, " ");
+  }
+  return string;
+}
+