Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/include/relastro.h
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/include/relastro.h	(revision 36563)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/include/relastro.h	(revision 36564)
@@ -30,4 +30,11 @@
   MARK_BIG_OFFSET    = 0x0010,
 } MeasurementMask;
+
+typedef struct {
+  float R;
+  float D;
+  unsigned int objID;
+  unsigned int catID;
+} MeanPos;
 
 typedef struct {
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MeanPosIO.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MeanPosIO.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MeanPosIO.c	(revision 36564)
@@ -0,0 +1,177 @@
+# include "relastro.h"
+
+# define GET_COLUMN(OUT,NAME,TYPE) \
+  TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \
+  myAssert (!strcmp(type, #TYPE), "wrong column type");
+
+// this is nearly identical to the one in 'uniphot/src' used by setphot / setphot_client.
+// Here, we use a handful of different columns (if not, we could move to libdvo)
+MeanPos *MeanPosLoad(char *filename, off_t *nmeanpos) {
+
+  int i, Ncol;
+  off_t Nrow;
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  *nmeanpos = 0;
+  MeanPos *meanpos = NULL;
+
+  FILE *f = fopen (filename, "r");
+  if (!f) {
+    fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename);
+    return NULL;
+  }
+
+  /* load in PHU segment (ignore) */
+  if (!gfits_fread_header (f, &header)) {
+    if (VERBOSE) fprintf (stderr, "can't read image subset header\n");
+    fclose (f);
+    return NULL;
+  }
+  if (!gfits_fread_matrix (f, &matrix, &header)) {
+    if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n");
+    gfits_free_header (&header);
+    fclose (f);
+    return NULL;
+  }
+
+  ftable.header = &theader;
+
+  // load data for this header 
+  if (!gfits_load_header (f, &theader)) {
+    fclose (f);
+    return NULL;
+  }
+
+  // read the fits table bytes
+  if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
+    fclose (f);
+    return (NULL);
+  }
+  fclose (f);
+
+  // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values.
+  // this means I need 3 copies in memory at some point.  ugh.
+
+  char type[16];
+
+  GET_COLUMN (R,    	 "RA",           double);
+  GET_COLUMN (D,   	 "DEC",          double);
+  GET_COLUMN (objID, 	 "OBJ_ID",       int);
+  GET_COLUMN (catID, 	 "CAT_ID",       int);
+
+  ALLOCATE (meanpos, MeanPos, Nrow);
+  for (i = 0; i < Nrow; i++) {
+    meanpos[i].R              = R    [i];
+    meanpos[i].D              = D    [i];
+    meanpos[i].objID          = objID[i];
+    meanpos[i].catID          = catID[i];
+  }
+  fprintf (stderr, "loaded data for %lld objects (* filters)\n", (long long) Nrow);
+
+  free (R    );
+  free (D    );
+  free (objID);
+  free (catID);
+
+  *nmeanpos = Nrow;
+  return meanpos;
+}
+
+// STATUS is value expected for success
+# define CHECK_STATUS(STATUS,MSG,...)					\
+  if (!(STATUS)) {							\
+    fprintf (stderr, MSG, __VA_ARGS__);					\
+    return FALSE;							\
+  }
+
+int MeanPosSave(char *filename, MeanPos *meanpos, off_t Nmeanpos) {
+
+  int i;
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  gfits_init_header (&header);
+  header.extend = TRUE;
+  gfits_create_header (&header);
+  gfits_create_matrix (&header, &matrix);
+
+  gfits_create_table_header (&theader, "BINTABLE", "MEANPOS");
+
+  gfits_define_bintable_column (&theader, "D", "RA",        "mean position, ra",  "degrees", 1.0, 0.0);
+  gfits_define_bintable_column (&theader, "D", "DEC",       "mean position, dec", "degrees", 1.0, 0.0);
+  gfits_define_bintable_column (&theader, "J", "OBJ_ID",    "object ID",          NULL,      1.0, 1.0*0x8000);
+  gfits_define_bintable_column (&theader, "J", "CAT_ID",    "catalog ID",         NULL,      1.0, 1.0*0x8000);
+
+  // generate the output array that carries the data
+  gfits_create_table (&theader, &ftable);
+
+  double *R, *D;
+  unsigned int *objID, *catID;
+
+  // create intermediate storage arrays
+  ALLOCATE (R,         double, 	       Nmeanpos);
+  ALLOCATE (D,         double, 	       Nmeanpos);
+  ALLOCATE (objID,     unsigned int,   Nmeanpos);
+  ALLOCATE (catID,     unsigned int,   Nmeanpos);
+
+  // assign the storage arrays
+  for (i = 0; i < Nmeanpos; i++) {
+    R    [i]   = meanpos[i].R    ;
+    D    [i]   = meanpos[i].D    ;
+    objID[i]   = meanpos[i].objID;
+    catID[i]   = meanpos[i].catID;
+  }
+
+  // add the columns to the output array
+  gfits_set_bintable_column (&theader, &ftable, "RA",        R,         Nmeanpos);
+  gfits_set_bintable_column (&theader, &ftable, "DEC",       D,         Nmeanpos);
+  gfits_set_bintable_column (&theader, &ftable, "OBJ_ID",    objID,     Nmeanpos);
+  gfits_set_bintable_column (&theader, &ftable, "CAT_ID",    catID,     Nmeanpos);
+
+  free (R    );
+  free (D    );
+  free (objID);
+  free (catID);
+
+  FILE *f = fopen (filename, "w");
+  if (!f) {
+    fprintf (stderr, "ERROR: cannot open meanmag file for output %s\n", filename);
+    return FALSE;
+  }
+
+  int status;
+  status = gfits_fwrite_header  (f, &header);
+  CHECK_STATUS (status, "ERROR: cannot write header for meanpos %s\n", filename);
+
+  status = gfits_fwrite_matrix  (f, &matrix);
+  CHECK_STATUS (status, "ERROR: cannot write matrix for meanpos %s\n", filename);
+
+  status = gfits_fwrite_Theader (f, &theader);
+  CHECK_STATUS (status, "ERROR: cannot write table header for meanpos %s\n", filename);
+
+  status = gfits_fwrite_table  (f, &ftable);
+  CHECK_STATUS (status, "ERROR: cannot write table data for meanpos %s\n", filename);
+
+  gfits_free_header (&header);
+  gfits_free_matrix (&matrix);
+  gfits_free_header (&theader);
+  gfits_free_table (&ftable);
+
+  int fd = fileno (f);
+
+  status = fflush (f);
+  CHECK_STATUS (!status, "ERROR: cannot flush file meanpos %s\n", filename);
+
+  status = fsync (fd);
+  CHECK_STATUS (!status, "ERROR: cannot flush file meanpos %s\n", filename);
+
+  status = fclose (f);
+  CHECK_STATUS (!status, "ERROR: problem closing meanpos file %s\n", filename);
+
+  return TRUE;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MosaicOps.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MosaicOps.c	(revision 36563)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/MosaicOps.c	(revision 36564)
@@ -11,5 +11,5 @@
 
 // list of mosaic associated with each image  
-static off_t    Nmosaic_for_images; // number of images (for off_ternal checks)
+static off_t    Nmosaic_for_images; // number of images (for internal checks)
 static off_t    *mosaic_for_images; // array of: image -> mosaic
 
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/assign_images.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/assign_images.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/assign_images.c	(revision 36564)
@@ -0,0 +1,245 @@
+# include "relastro.h"
+
+// This function generates a subset of the images based on selections.  Input db has already
+// been loaded with the raw fits table data
+int assign_images (FITS_DB *db, RegionHostTable *regionHosts) {
+
+  off_t Nimage;
+
+  INITTIME;
+
+  // convert database table to internal structure (binary to Image)
+  // 'image' points to the same memory as db->ftable->buffer
+  Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped);
+  if (!image) {
+      fprintf (stderr, "ERROR: failed to read images\n");
+      exit (2);
+  }
+  MARKTIME("convert image table to internal structure: %f sec\n", dtime);
+
+  // *** NOTE : for the moment, regions must be in the range 0 - 360, -90 - +90
+
+  // generate the chip match here so we can define the mosaic centers (if needed)
+  BuildChipMatch (image, Nimage);
+  MARKTIME("build chip match for %d images: %f sec\n", (int) Nimage, dtime);
+
+  initMosaics (image, Nimage);
+  MARKTIME("set mosaic coordinates and Mcal values: %f sec\n", dtime);
+
+  // register the image array with ImageOps.c for later getimageByID calls
+  initImages (image, NULL, Nimage);
+
+  if (VERBOSE) fprintf (stderr, "finding images\n");
+
+  // for each regionHost, select images which are contained by the region
+  // even faster would be to use a tree to get to the real regions...
+  select_images_hostregion (regionHosts, image, Nimage);
+
+  // supply the mosaics to the image table for the regionHosts : we already have the image
+  // <-> mosaic relationship, we just need to select these mosaics (once per regionHost)
+  select_mosaics_hostregion (regionHosts, image, Nimage);
+
+  return TRUE;
+}
+
+# define D_NIMAGE 1000
+
+// assign images to the region hosts; at the end, each host will have its list of images
+int select_images_hostregion (RegionHostTable *regionHosts, Image *image, off_t Nimage) {
+
+  int ecode, found;
+  off_t i, j;
+
+  // INITTIME;
+  
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    regionHosts->hosts[i].Nimage = 0;
+    regionHosts->hosts[i].NIMAGE = D_NIMAGE;
+    ALLOCATE (regionHosts->hosts[i].image, Image, regionHosts->hosts[i].NIMAGE);
+  }
+
+  for (j = 0; j < Nimage; j++) {
+    
+    /* exclude images by photcode */
+    ecode = GetPhotcodeEquivCodebyCode (image[j].photcode);
+    found = FALSE;
+    int Ns;
+    for (Ns = 0; !found && (Ns < Nphotcodes); Ns++) {
+      if (ecode == photcodes[Ns][0].code) found = TRUE;
+    }
+    if (!found) continue;
+
+    /* exclude images by time */
+    if (TimeSelect) {
+      if (image[j].tzero < TSTART) continue;
+      if (image[j].tzero > TSTOP) continue;
+    }
+    
+    // do not include DIS (PHU-level mosaics) in the output list
+    if (!strcmp(&image[j].coords.ctype[4], "-DIS")) continue;
+
+    // match the image to its corresponding ASTROMETRIC mosaic (not the same as the Mosaic above)
+    if (!FindMosaicForImage (image, Nimage, j)) {
+      if (VERBOSE2) fprintf (stderr, "cannot find mosaic for "OFF_T_FMT"\n", i);
+      continue;
+    }
+    
+    // use a reference coordinate for each image to assign to hosts
+    // define image center - note the DIS images (mosaic phu) are special
+    double Xc, Yc;
+    double Rc, Dc;
+    Xc = 0.5*image[j].NX; 
+    Yc = 0.5*image[j].NY;
+    
+    XY_to_RD (&Rc, &Dc, Xc, Yc, &image[j].coords);
+    Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0);
+
+    i = find_host_for_coords (regionHosts, Rc, Dc);
+
+    if (i == -1) continue;
+
+    RegionHostInfo *host = &regionHosts->hosts[i];
+
+    double Rmin, Rmax, Dmin, Dmax;
+    calculate_image_bounds (&image[j], &Rmin, &Rmax, &Dmin, &Dmax);
+
+    host->RminCat = MIN(Rmin, host->RminCat);
+    host->RmaxCat = MAX(Rmax, host->RmaxCat);
+    host->DminCat = MIN(Dmin, host->DminCat);
+    host->DmaxCat = MAX(Dmax, host->DmaxCat);
+
+    // regionHosts needs to have the full outer boundary
+    // (so reload_catalogs covers the correct region)
+    regionHosts->Rmin = MIN(Rmin, regionHosts->Rmin);
+    regionHosts->Rmax = MAX(Rmax, regionHosts->Rmax);
+    regionHosts->Dmin = MIN(Dmin, regionHosts->Dmin);
+    regionHosts->Dmax = MAX(Dmax, regionHosts->Dmax);
+
+    // this is a bit memory expensive : I am making a complete copy of the image table here
+    off_t Nsubset = regionHosts->hosts[i].Nimage;
+    regionHosts->hosts[i].image[Nsubset] = image[j];
+    // regionHosts->hosts[i].line_number[Nsubset] = j;
+
+    regionHosts->hosts[i].Nimage ++;
+    if (regionHosts->hosts[i].Nimage == regionHosts->hosts[i].NIMAGE) {
+      regionHosts->hosts[i].NIMAGE += D_NIMAGE;
+      REALLOCATE (regionHosts->hosts[i].image, Image, regionHosts->hosts[i].NIMAGE);
+      // REALLOCATE (regionHosts->hosts[i].line_number, off_t, regionHosts->hosts[i].NIMAGE);
+    }
+  }
+
+  return TRUE;
+}
+
+double Xf[] = {0.0, 1.0, 0.0, 1.0};
+double Yf[] = {0.0, 0.0, 1.0, 1.0};
+
+int calculate_image_bounds (Image *image, double *rmin, double *rmax, double *dmin, double *dmax) {
+
+  int n;
+
+  double Rmin = 360.0;
+  double Rmax =   0.0;
+  double Dmin = +90.0;
+  double Dmax = -90.0;
+
+  // define image corners
+  for (n = 0; n < 4; n++) {
+    double Xc, Yc, Rc, Dc;
+    Xc = Xf[n]*image->NX; 
+    Yc = Yf[n]*image->NY;
+    XY_to_RD (&Rc, &Dc, Xc, Yc, &image->coords);
+    Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0);
+      
+    Rmin = MIN (Rmin, Rc);
+    Rmax = MAX (Rmax, Rc);
+    Dmin = MIN (Dmin, Dc);
+    Dmax = MAX (Dmax, Dc);
+  }
+
+  *rmin = Rmin;
+  *rmax = Rmax;
+  *dmin = Dmin;
+  *dmax = Dmax;
+
+  return TRUE;
+}
+
+int calculate_host_image_bounds (RegionHostTable *regionHosts) {
+
+  int i, n;
+  off_t j;
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+
+    RegionHostInfo *host = &regionHosts->hosts[i];
+
+    // XXX clear the chip match?
+
+    BuildChipMatch (host->image, host->Nimage);
+
+    double Rmin = 360.0;
+    double Rmax =   0.0;
+    double Dmin = +90.0;
+    double Dmax = -90.0;
+
+    for (j = 0; j < host->Nimage; j++) {
+
+      Image *image = &host->image[j];
+
+      if (!FindMosaicForImage (host->image, host->Nimage, j)) {
+	fprintf (stderr, "missing astrometry? programming error?\n");
+	abort ();
+      }
+      
+      // define image corners
+      for (n = 0; n < 4; n++) {
+	double Xc, Yc, Rc, Dc;
+	Xc = Xf[n]*image->NX; 
+	Yc = Yf[n]*image->NY;
+	XY_to_RD (&Rc, &Dc, Xc, Yc, &image->coords);
+	Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0);
+      
+	Rmin = MIN (Rmin, Rc);
+	Dmin = MIN (Dmin, Rc);
+	
+	Rmax = MAX (Rmax, Rc);
+	Dmax = MAX (Dmax, Rc);
+      }
+
+    }
+
+    host->RminCat = Rmin;
+    host->DminCat = Dmin;
+    host->RmaxCat = Rmax;
+    host->DmaxCat = Dmax;
+
+    // regionHosts needs to have the full outer boundary
+    // (so reload_catalogs covers the correct region)
+    regionHosts->Rmin = MIN(Rmin, regionHosts->Rmin);
+    regionHosts->Rmax = MAX(Rmax, regionHosts->Rmax);
+    regionHosts->Dmin = MIN(Dmin, regionHosts->Dmin);
+    regionHosts->Dmax = MAX(Dmax, regionHosts->Dmax);
+  }
+  return TRUE;
+}
+
+
+// XXX add a search tree to speed this up?
+int find_host_for_coords (RegionHostTable *regionHosts, double Rc, double Dc) {
+
+  int i;
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+
+    RegionHostInfo *host = &regionHosts->hosts[i];
+    if (Rc <  host->Rmin) continue;
+    if (Rc >= host->Rmax) continue;
+    if (Dc <  host->Dmin) continue;
+    if (Dc >= host->Dmax) continue;
+
+    return i;
+  }
+  return -1;
+}
+
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/indexCatalogs.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/indexCatalogs.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/indexCatalogs.c	(revision 36564)
@@ -0,0 +1,64 @@
+# include "relphot.h"
+
+static int   catIDmax = 0;
+static int  *catIDseq = NULL;
+static int  *objIDmax = NULL;
+static int **objIDseq = NULL;
+
+int indexCatalogs (Catalog *catalog, int Ncatalog) {
+
+  int i;
+  off_t j;
+
+  // find the max value of catID
+  for (i = 0; i < Ncatalog; i++) {
+    catIDmax = MAX (catalog[i].catID, catIDmax);
+  }
+
+  ALLOCATE (catIDseq, int, catIDmax + 1);
+  for (i = 0; i < catIDmax + 1; i++) {
+    catIDseq[i] = -1;
+  }
+
+  for (i = 0; i < Ncatalog; i++) {
+    int catID = catalog[i].catID;
+    catIDseq[catID] = i;
+  }
+  
+  ALLOCATE (objIDmax, int,   Ncatalog);
+  ALLOCATE (objIDseq, int *, Ncatalog);
+  for (i = 0; i < Ncatalog; i++) {
+    objIDmax[i] = 0;
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      objIDmax[i] = MAX (catalog[i].averageT[j].objID, objIDmax[i]);
+    }
+
+    ALLOCATE (objIDseq[i], int, objIDmax[i] + 1);
+    for (j = 0; j < objIDmax[i] + 1; j++) {
+      objIDseq[i][j] = -1;
+    }
+
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      int objID = catalog[i].averageT[j].objID;
+      objIDseq[i][objID] = j;
+    }
+  }
+  return TRUE;
+}
+
+int catID_and_objID_to_seq (int catID, int objID, int *catSeq, off_t *objSeq) {
+
+  if (catID > catIDmax) return FALSE;
+
+  int cat = catIDseq[catID];
+  if (cat < 1) return FALSE;
+
+  if (objID > objIDmax[cat]) return FALSE;
+
+  int obj = objIDseq[cat][objID];
+  if (obj < 1) return FALSE;
+
+  *catSeq = cat;
+  *objSeq = obj;
+  return TRUE;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/launch_region_hosts.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/launch_region_hosts.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/launch_region_hosts.c	(revision 36564)
@@ -0,0 +1,97 @@
+# include "relastro.h"
+# define DEBUG 0
+
+int strextend (char *input, char *format,...) {
+
+  char tmpextra[1024], tmpline[1024];
+  va_list argp;
+
+  va_start (argp, format);
+  vsnprintf (tmpextra, 1024, format, argp);
+  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
+  strcpy (input, tmpline);
+
+  return TRUE;
+}
+
+int launch_region_hosts (RegionHostTable *regionHosts) {
+
+  int i;
+
+  // clear the I/O files
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    char *syncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.sync");
+    truncate (syncfile, 0);
+    free (syncfile);
+
+    char *fitsfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.fits");
+    truncate (fitsfile, 0);
+    free (fitsfile);
+
+    char *imsyncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.sync");
+    truncate (imsyncfile, 0);
+    free (imsyncfile);
+
+    char *imfitsfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.fits");
+    truncate (imfitsfile, 0);
+    free (imfitsfile);
+
+    char *loopsyncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "loop.sync");
+    truncate (loopsyncfile, 0);
+    free (loopsyncfile);
+  }
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+
+    RegionHostInfo *host = &regionHosts->hosts[i];
+
+    // communication files:
+    // subset images per host : CATDIR/Image.HOSTNAME.fits
+    char filename[1024];
+    snprintf (filename, 1024, "%s/Image.%d.fits", CATDIR, host->hostID);
+
+    // write the image subset for this host
+    ImageTableSave (filename, host->image, host->Nimage);
+
+    char command[1024];
+    snprintf (command, 1024, "relastro %s -parallel-images %s -region-hosts %s -region-hostID %d -D CATDIR %s -region %f %f %f %f -statmode %s -D CAMERA %s -D STAR_TOOFEW %d -minerror %f", 
+	      PhotcodeList, filename, REGION_FILE, host->hostID, CATDIR, host->RminCat, host->RmaxCat, host->DminCat, host->DmaxCat, STATMODE, CAMERA, STAR_TOOFEW, MIN_ERROR);
+
+    if (VERBOSE)       	 strextend (command, "-v");
+    if (VERBOSE2)      	 strextend (command, "-vv");
+    if (RESET)         	 strextend (command, "-reset");
+    if (RESET_ZEROPTS) 	 strextend (command, "-reset-zpts");
+    if (UPDATE)        	 strextend (command, "-update");
+    if (!KEEP_UBERCAL) 	 strextend (command, "-reset-ubercal");
+    if (PARALLEL)      	 strextend (command, "-parallel");
+    if (PARALLEL_MANUAL) strextend (command, "-parallel-manual");
+    if (PARALLEL_SERIAL) strextend (command, "-parallel-serial");
+
+    fprintf (stderr, "command: %s\n", command);
+    
+    if (PARALLEL_REGIONS_MANUAL) continue;
+
+    // launch the job on the remote machine (no handshake)
+    int errorInfo = 0;
+    int pid = rconnect ("ssh", host->hostname, command, host->stdio, &errorInfo, FALSE);
+    if (!pid) {
+      if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", host->hostname, errorInfo);
+      exit (1);
+    }
+    host->pid = pid; // save for future reference
+  }
+
+  if (PARALLEL_REGIONS_MANUAL) {
+    fprintf (stderr, "run the relastro_client commands above.  when these are done, hit return\n");
+    getchar();
+  } else {
+    RegionHostTableWaitJobsGetIO (regionHosts, __FILE__, __LINE__, VERBOSE);
+  }
+ 
+  int status = TRUE;
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    status = status && (regionHosts->hosts[i].status == 0);
+  }
+
+  return status;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro.c	(revision 36563)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro.c	(revision 36564)
@@ -41,4 +41,14 @@
       exit (0);
 
+    case TARGET_PARALLEL_REGIONS:
+      // run image updates in parallel across multiple remote machines
+      relastro_parallel_regions ();
+      exit (0);
+
+    case TARGET_PARALLEL_IMAGES:
+      // operation on the remote machines in the PARALLEL_REGION mode
+      relastro_parallel_images ();
+      exit (0);
+
     default:
       fprintf (stderr, "impossible!\n");
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_images.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_images.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_images.c	(revision 36564)
@@ -0,0 +1,129 @@
+# include "relastro.h"
+
+/* This function is essentially identical to relastro_images, except:
+
+ * load the subset images saved by the master node
+ * distinguish detections we own (touch our images) and those we don't
+ * distinguish objects we own (in region) and those we don't
+ * update the unowned detections for owned objects from neighbor regions
+ * update the unowned objects for owned detections 
+ */
+
+int relastro_parallel_images () {
+
+  int i, Ncatalog;
+  Catalog *catalog = NULL;
+
+  INITTIME;
+
+  // load the RegionTable (UserRegion should not be used at this level)
+  RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE);
+
+  // load the subset images belonging to this host
+  off_t Nimage;
+  Image *image = ImageTableLoad (IMAGE_TABLE, &Nimage);
+  if (!image) {
+    fprintf (stderr, "ERROR loading image %s\n", IMAGE_TABLE);
+    exit (2);
+  }
+
+  // once we have read this table, we should remove it for repeat runs
+  // unlink (IMAGE_TABLE); // XXX a bit risky, add some protection?
+
+  // XXX need to deal with mosaic vs image...
+  initMosaics (image, Nimage);
+
+  initImages (image, NULL, Nimage);
+
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+  SkyList *skylist = SkyListByBounds (sky, -1, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
+
+  initCoords();
+
+  /* load catalog data from region files (hostID is 0 since we are not a client */
+  catalog = load_catalogs (skylist, &Ncatalog, TRUE, 0, NULL);
+  MARKTIME("-- load catalog data: %f sec\n", dtime);
+  
+  if (Ncatalog == 0) {
+    fprintf (stderr, "ERROR: no valid data for relastro, exiting\n");
+    exit (2);
+  }
+
+  // generate tables go from catID,objID -> catSeq,objSeq
+  indexCatalogs (catalog, Ncatalog);
+
+  /* match measurements with images, mosaics */
+  initImageBins  (catalog, Ncatalog, TRUE);
+  MARKTIME("-- make image bins: %f sec\n", dtime);
+
+  findImages (catalog, Ncatalog, TRUE);
+  MARKTIME("-- set up image indexes: %f sec\n", dtime);
+
+  // set test points based on the starmap
+  createStarMap (catalog, Ncatalog);
+
+  markObjects (catalog, Ncatalog);
+
+  SAVEPLOT = FALSE;
+
+  /* major modes */
+  switch (FIT_TARGET) {
+    case TARGET_SIMPLE:
+      for (i = 0; i < NLOOP; i++) {
+	UpdateObjects (catalog, Ncatalog);
+	share_mean_pos (catalog, Ncatalog, regionHosts, i);
+	slurp_mean_pos (catalog, Ncatalog, regionHosts, i);
+	UpdateSimple (catalog, Ncatalog);
+	share_image_pos (regionHosts, i);
+	slurp_image_pos (regionHosts, i);
+      }
+      break;
+
+    case TARGET_CHIPS:
+      for (i = 0; i < NLOOP; i++) {
+	UpdateObjects (catalog, Ncatalog);
+	share_mean_pos (catalog, Ncatalog, regionHosts, i);
+	slurp_mean_pos (catalog, Ncatalog, regionHosts, i);
+	UpdateChips (catalog, Ncatalog);
+	share_image_pos (regionHosts, i);
+	slurp_image_pos (regionHosts, i);
+	MARKTIME("update chips: %f sec\n", dtime);
+      }
+      // create summary plots of the process
+      // relastroVisualSummaryChips();
+      break;
+
+    case TARGET_MOSAICS:
+      for (i = 0; i < NLOOP; i++) {
+	UpdateObjects (catalog, Ncatalog);
+	share_mean_pos (catalog, Ncatalog, regionHosts, i);
+	slurp_mean_pos (catalog, Ncatalog, regionHosts, i);
+	UpdateMosaic (catalog, Ncatalog);
+	share_image_pos (regionHosts, i);
+	slurp_image_pos (regionHosts, i);
+      }
+      break;
+
+    default:
+      fprintf (stderr, "programming error at %s:%d", __FILE__, __LINE__);
+      exit (2);
+  }
+
+  // this is a checkpoint to make sure all hosts have finished the loop above
+  int myHost = regionHosts->index[REGION_HOST_ID];
+  char *loopsyncfile = make_filename (CATDIR, regionHosts->hosts[myHost].hostname, REGION_HOST_ID, "loop.sync");
+  update_sync_file (loopsyncfile, 0);
+  free (loopsyncfile);
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue;
+    char *loopsync = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "loop.sync");
+    check_sync_file (loopsync, 0);
+    free (loopsync);
+  }    
+
+  share_image_pos (regionHosts, -1);
+
+  exit (0);
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_regions.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_regions.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/relastro_parallel_regions.c	(revision 36564)
@@ -0,0 +1,81 @@
+# include "relastro.h"
+
+int relastro_parallel_regions () {
+
+  int status;
+  FITS_DB db;
+
+  INITTIME;
+
+  // load the RegionTable (UserRegion should not be used at this level)
+  RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE);
+  if (!regionHosts) {
+    fprintf (stderr, "ERROR: problem with region host table\n");
+    exit (2);
+  }
+
+  // register database handle with shutdown procedure 
+  set_db (&db);
+  db.mode   = dvo_catalog_catmode (CATMODE);
+  db.format = dvo_catalog_catformat (CATFORMAT);
+
+  // lock and load the image db table
+  status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT));
+  if (!status && UPDATE) {
+    fprintf (stderr, "error\n");
+    Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
+  }
+
+  if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) {
+    // if the Image.dat file is missing, db.dbstate will have a value of either:
+    // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE)
+    Shutdown ("ERROR: database %s contains no image data", CATDIR);
+  }
+
+  // read data from Image.dat file
+  if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename);
+  MARKTIME("-- load image data: %f sec\n", dtime);
+
+  /* assign the images to the different region hosts */
+  if (!assign_images (&db, regionHosts)) Shutdown ("error assigning images to region hosts");
+  MARKTIME("-- assign images: %f sec\n", dtime);
+
+  /* launch processing on the parallel region hosts */
+  if (!launch_region_hosts (regionHosts)) Shutdown ("error launching region hosts");
+
+  // retrieve updated image parameters from the remote hosts (also set Image.mcal)
+  if (!slurp_image_pos (regionHosts, -1)) Shutdown ("error loading image updates");
+
+  if (!UPDATE) { 
+    dvo_image_unlock (&db); 
+    MARKTIME ("finished relastro -parallel-regions: %f sec total\n", dtime);
+    fprintf (stderr, "NOTE: UPDATE is OFF (results are not saved)\n");
+    exit (0);
+  }
+
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+  SkyList *skylist = SkyListByBounds (sky, -1, regionHosts->Rmin, regionHosts->Rmax, regionHosts->Dmin, regionHosts->Dmax);
+
+  if (PARALLEL) {
+    // save the updated image parameters
+    dvo_image_update (&db, VERBOSE);
+    dvo_image_unlock (&db); 
+  }
+
+  // iterate over catalogs to make detection coordinates consistant
+  UpdateObjectOffsets (skylist, 0, NULL);
+
+  if (!PARALLEL) {
+    // save the changes to the image parameters
+    // XXX if we have generated a vtable, we can use update; otherwise use save
+    // dvo_image_save (&db, VERBOSE);
+    dvo_image_update (&db, VERBOSE);
+    dvo_image_unlock (&db); 
+  }
+
+  MARKTIME ("finished relastro -parallel-regions: %f sec total\n", dtime);
+
+  exit (0);
+}
+
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/share_mean_pos.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/share_mean_pos.c	(revision 36564)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relastro/src/share_mean_pos.c	(revision 36564)
@@ -0,0 +1,139 @@
+# include "relastro.h"
+// we are sharing mean positions for all objects which (a) I own and (b) which have unowned detections
+
+# define D_NMEANPOS 10000
+int share_mean_pos (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) {
+
+  int i;
+  off_t j;
+
+  off_t Nmeanpos = 0;
+  off_t NMEANPOS = D_NMEANPOS;
+
+  MeanPos *meanpos = NULL;
+  ALLOCATE (meanpos, MeanPos, NMEANPOS);
+
+  int Ns;
+  int Nsecfilt = GetPhotcodeNsecfilt();
+
+  int myHost = regionHosts->index[REGION_HOST_ID];
+  double Rmin = regionHosts->hosts[myHost].Rmin;
+  double Rmax = regionHosts->hosts[myHost].Rmax;
+  double Dmin = regionHosts->hosts[myHost].Dmin;
+  double Dmax = regionHosts->hosts[myHost].Dmax;
+
+  // XXX skip some catalogs based on UserPatch?
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+
+      // do I own this object? (in region range?) --- CAREFUL HERE!!
+      if (catalog[i].averageT[j].R <  Rmin) continue;
+      if (catalog[i].averageT[j].R >= Rmax) continue;
+      if (catalog[i].averageT[j].D <  Dmin) continue;
+      if (catalog[i].averageT[j].D >= Dmax) continue;
+
+      // does this object have missing detections (does someone else need it?)
+      // XXX : sky objects without missing detections
+      // XXX watch out for detections which are not associated with an image (REF)
+      if (catalog[i].averageT[j].nOwn == catalog[i].averageT[j].Nmeasure) continue;
+
+      for (Ns = 0; Ns < Nphotcodes; Ns++) {
+	int thisCode = photcodes[Ns][0].code;
+	int Nsec = GetPhotcodeNsec(thisCode);
+	set_mean_pos (&meanpos[Nmeanpos], &catalog[i].averageT[j], &catalog[i].secfilt[Nsecfilt*j + Nsec], Nsec);
+	Nmeanpos ++;
+	CHECK_REALLOCATE (meanpos, MeanPos, NMEANPOS, Nmeanpos, D_NMEANPOS);
+      }
+    }
+  }
+
+  // write out the meanmag fits table AND write state in some file
+  char *hostname = regionHosts->hosts[myHost].hostname;
+
+  char *posfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "meanpos.fits");
+  MeanPosSave (posfile, meanpos, Nmeanpos);
+  free (posfile);
+
+  char *syncfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "meanpos.sync");
+  update_sync_file (syncfile, nloop);
+  free (syncfile);
+
+  return TRUE;
+}
+
+int slurp_mean_pos (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) {
+
+  off_t i;
+
+  int Nmeanpos = 0;
+  MeanPos *meanpos = NULL;
+  ALLOCATE (meanpos, MeanPos, 1);
+
+  fprintf (stderr, "grabbing mean object pos from other hosts...\n");
+
+  int Nsecfilt = GetPhotcodeNsecfilt();
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    // if (not_neighbor(host[i])) continue;
+    if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue;
+
+    char *syncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanpos.sync");
+    check_sync_file (syncfile, nloop);
+    free (syncfile);
+    
+    off_t Nsubset = 0;
+    char *posfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanpos.fits");
+    MeanPos *meanposSubset = MeanPosLoad (posfile, &Nsubset);
+    free (posfile);
+
+    // merge_mean_pos reallocs meanpos and frees the input meanposSubset
+    meanpos = merge_mean_pos (meanpos, &Nmeanpos, meanposSubset, Nsubset);
+  }
+
+  for (i = 0; i < Nmeanpos; i++) {
+    int objID = meanpos[i].objID;
+    int catID = meanpos[i].catID;
+
+    // set the mean mag
+    int catSeq;
+    off_t objSeq;
+    if (!catID_and_objID_to_seq (catID, objID, &catSeq, &objSeq)) {
+	// XXX what should I do if this does not match?
+	continue;
+    }
+
+    catalog[catSeq].averageT[objSeq].R = meanpos[i].R;
+    catalog[catSeq].averageT[objSeq].D = meanpos[i].D;
+  }
+
+  fprintf (stderr, "DONE grabbing mean object pos from other hosts...\n");
+
+  return TRUE;
+}
+
+int set_mean_pos (MeanPos *meanpos, AverageTiny *average, SecFilt *secfilt, int Nsec) {
+
+  meanpos->R     = average->R;
+  meanpos->D     = average->D;
+  meanpos->objID = average->objID;
+  meanpos->catID = average->catID;
+
+  return TRUE;
+}
+
+MeanPos *merge_mean_pos (MeanPos *target, int *ntarget, MeanPos *source, int Nsource) {
+
+  off_t i;
+
+  REALLOCATE (target, MeanPos, *ntarget + Nsource);
+  for (i = 0; i < Nsource; i++) {
+    off_t n = i + *ntarget;
+    target[n] = source[i];
+  }
+  
+  free (source);
+
+  *ntarget += Nsource;
+  return (target);
+}
+