Index: /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h	(revision 36516)
@@ -295,6 +295,10 @@
   int pid;		      // remote process ID
   char *hostname;
-  ImageSubset *image;
+
   int line_numbers;
+
+  off_t Nimage;
+  off_t NIMAGE;
+  Image *image;
 } RegionHostInfo;
 
Index: /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c	(revision 36516)
@@ -7,4 +7,9 @@
   int i;
   for (i = Nhosts; i < NHOSTS; i++) {
+    hosts[i].Rmin = NAN;
+    hosts[i].Rmax = NAN;
+    hosts[i].Dmin = NAN;
+    hosts[i].Dmax = NAN;
+
     hosts[i].hostname = NULL;
     hosts[i].stdio[HOST_STDIN] = -1;
@@ -13,8 +18,7 @@
     hosts[i].pid = 0;
 
-    hosts[i].Rmin = NAN;
-    hosts[i].Rmax = NAN;
-    hosts[i].Dmin = NAN;
-    hosts[i].Dmax = NAN;
+    hosts[i].image = NULL;
+    hosts[i].Nimage = 0;
+    hosts[i].NIMAGE = 0;
   }
   return;
@@ -143,3 +147,2 @@
   return table;
 }
-
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt	(revision 36516)
@@ -1,2 +1,26 @@
+
+2014.02.14
+
+ more relphot notes:
+
+ * I load the catalogs, with a  
+
+2014.02.12
+
+I am making progess on the relphot -parallel-regions implementation.  some things I need to deal with:
+
+ * assign_images: select mosaics for all images saved for a given host (need to supply them as well)
+   ** I need to decide on the Image / Mosaic split.  If I am calibrating by image, then the image center defines ownership
+   ** if I am calibrating by exposure, then the mosaic center must define ownership
+ o determine SkyList covering the images for region host
+ o tag detections and objects which I own or do not own
+   o ImageOps.c / matchImage matches detections to images and needs to handle mine / not mine cases
+   o ditto for matchMosaics
+ o load_catalogs : I need to make my requested catalogs unique (just add uniquer to load_catalogs.c:130
+ o figure out what fields I'm saving in meanmags and how to construct it
+   -- mag, objID, catID, photcode, (dmag or other stats?)
+ * from which hosts do I slurp mean mags?
+ o function to merge a new meanmag array into the existing one
+ * how to go from (objID,catID) to a given catalog[i].average[j]
 
 2014.02.06
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/include/relphot.h
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/include/relphot.h	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/include/relphot.h	(revision 36516)
@@ -110,4 +110,13 @@
   StatType kronstats;
 } SetMrelInfo;
+
+typedef struct {
+  float M;
+  float dM;
+  float Xm;
+  int Nsec;
+  unsigned int objID;
+  unsigned int catID;
+} MeanMag;
 
 typedef struct {
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageOps.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageOps.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageOps.c	(revision 36516)
@@ -244,7 +244,11 @@
   
   int Nmatch = 0;
- for (i = 0; i < Ncatalog; i++) {
+  for (i = 0; i < Ncatalog; i++) {
     for (j = 0; j < catalog[i].Nmeasure; j++) {
+      catalog[i].measureT[j].myDet = FALSE; // a detetion is not mine until proven otherwise
       ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[j].photcode);
+
+      // skip measurements which do not match one of the requested photcodes (
+      // (do we not already exclude in bcatalog -- maybe needed for reload_objects?
       found = FALSE;
       for (Ns = 0; !found && (Ns < Nphotcodes); Ns++) {
@@ -252,4 +256,6 @@
       }
       if (!found) continue; 
+
+      // if we match one of our images, myDet gets set to TRUE
       matchImage (catalog, j, i, doImageList);
       Nmatch ++;
@@ -322,4 +328,5 @@
     return;
   }
+  catalog[cat].measureT[meas].myDet = TRUE;
 
   if (USE_GRID) {
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageTable.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageTable.c	(revision 36516)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/ImageTable.c	(revision 36516)
@@ -0,0 +1,65 @@
+# include "relphot.h"
+
+Image *ImageTableLoad(char *filename, off_t *nimage) {
+
+  int i, Ncol;
+  off_t Nrow;
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  db.mode   = dvo_catalog_catmode (CATMODE);
+  db.format = dvo_catalog_catformat (CATFORMAT);
+  
+  /* lock and load the image db table */
+  status = dvo_image_lock (&db, filename, 60.0, LCK_SOFT);
+  if (!status) {
+    Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
+  }
+
+  // if the file is missing, db.dbstate will have a value of either:
+  // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE)
+  if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) {
+    Shutdown ("ERROR: database %s contains no image data", CATDIR);
+  }
+
+  // read data from Image.dat file
+  if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename);
+
+  // convert database table to internal structure (binary to Image)
+  // 'image' points to the same memory as db->ftable->buffer
+  Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped);
+  if (!image) {
+    Shutdown ("ERROR: failed to read images");
+  }
+
+  *nimage = Nimage;
+  return image;
+}
+
+int ImageTableSave (char *filename, Image *images, off_t Nimages) {
+
+  int i;
+  FITS_DB db;
+
+  // setup image table format and lock 
+  db.mode   = dvo_catalog_catmode (CATMODE);
+  db.format = dvo_catalog_catformat (CATFORMAT);
+  status    = dvo_image_lock (&db, filename, 60.0, LCK_XCLD);  // shorter timeout?
+  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
+
+  // load or create the image table 
+  if (db.dbstate != LCK_EMPTY) {
+    Shutdown ("image catalog already exists %s", db.filename);
+  }
+
+  dvo_image_create (&db, GetZeroPoint());
+
+  /* add the new images and save */
+  dvo_image_addrows (&db, images, Nimages);
+  dvo_image_update (&db, VERBOSE);
+  dvo_image_unlock (&db);
+
+  return TRUE;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MeanMagIO.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MeanMagIO.c	(revision 36516)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MeanMagIO.c	(revision 36516)
@@ -0,0 +1,194 @@
+# include "relphot.h"
+
+# define GET_COLUMN(OUT,NAME,TYPE) \
+  TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \
+  myAssert (!strcmp(type, #TYPE), "wrong column type");
+
+// this is nearly identical to the one in 'uniphot/src' used by setphot / setphot_client.
+// Here, we use a handful of different columns (if not, we could move to libdvo)
+MeanMag *MeanMagLoad(char *filename, off_t *nmeanmags) {
+
+  int i, Ncol;
+  off_t Nrow;
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  *nmeanmags = 0;
+  MeanMag *meanmags = NULL;
+
+  FILE *f = fopen (filename, "r");
+  if (!f) {
+    fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename);
+    return NULL;
+  }
+
+  /* load in PHU segment (ignore) */
+  if (!gfits_fread_header (f, &header)) {
+    if (VERBOSE) fprintf (stderr, "can't read image subset header\n");
+    fclose (f);
+    return NULL;
+  }
+  if (!gfits_fread_matrix (f, &matrix, &header)) {
+    if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n");
+    gfits_free_header (&header);
+    fclose (f);
+    return NULL;
+  }
+
+  ftable.header = &theader;
+
+  // load data for this header 
+  if (!gfits_load_header (f, &theader)) {
+    fclose (f);
+    return NULL;
+  }
+
+  // read the fits table bytes
+  if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
+    fclose (f);
+    return (NULL);
+  }
+  fclose (f);
+
+  // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values.
+  // this means I need 3 copies in memory at some point.  ugh.
+
+  char type[16];
+
+  GET_COLUMN (M,    	 "MAG",          float);
+  GET_COLUMN (dM,   	 "MAG_ERR",      float);
+  GET_COLUMN (Xm,   	 "MAG_CHISQ",    float);
+  GET_COLUMN (Nsec, 	 "NSEC",         int);
+  GET_COLUMN (objID, 	 "OBJ_ID",       int);
+  GET_COLUMN (catID, 	 "CAT_ID",       int);
+
+  ALLOCATE (meanmags, MeanMag, Nrow);
+  for (i = 0; i < Nrow; i++) {
+    meanman[i].M              = M    [i];
+    meanman[i].dM             = dM   [i];
+    meanman[i].Xm             = Xm   [i];
+    meanman[i].Nsec           = Nsec [i];
+    meanman[i].objID          = objID[i];
+    meanman[i].catID          = catID[i];
+  }
+  fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow);
+
+  free (M    );
+  free (dM   );
+  free (Xm   );
+  free (Nsec );
+  free (objID);
+  free (catID);
+
+  *nmeanmag = Nrow;
+  return meanman;
+}
+
+// STATUS is value expected for success
+# define CHECK_STATUS(STATUS,MSG,...)					\
+  if (!(STATUS)) {							\
+    fprintf (stderr, MSG, __VA_ARGS__);					\
+    return FALSE;							\
+  }
+
+int MeanMagsSave(char *filename, MeanMags *meanmags, off_t Nmeanmags) {
+
+  int i;
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  gfits_init_header (&header);
+  header.extend = TRUE;
+  gfits_create_header (&header);
+  gfits_create_matrix (&header, &matrix);
+
+  gfits_create_table_header (&theader, "BINTABLE", "MEANMAGS");
+
+  gfits_define_bintable_column (&theader, "E", "MAG",       "mean magnitude", "magnitudes", 1.0, 0.0);
+  gfits_define_bintable_column (&theader, "E", "MAG_ERR",   "mean magnitude error", "magnitudes", 1.0, 0.0);
+  gfits_define_bintable_column (&theader, "E", "MAG_CHISQ", "mean magnitude chisq", "unitless", 1.0, 0.0);
+  gfits_define_bintable_column (&theader, "J", "NSEC",      "secfilt sequence", NULL, 1.0, 1.0*0x8000);
+  gfits_define_bintable_column (&theader, "J", "OBJ_ID",    "object ID",        NULL, 1.0, 1.0*0x8000);
+  gfits_define_bintable_column (&theader, "J", "CAT_ID",    "catalog ID",       NULL, 1.0, 1.0*0x8000);
+
+  // generate the output array that carries the data
+  gfits_create_table (&theader, &ftable);
+
+  float *M, *dM, *Xm;
+  int *Nsec;
+  unsigned int *objID, *catID;
+
+  // create intermediate storage arrays
+  ALLOCATE (M,         float, 	       Nmeanmags);
+  ALLOCATE (dM,        float, 	       Nmeanmags);
+  ALLOCATE (Xm,        float, 	       Nmeanmags);
+  ALLOCATE (Nsec,      int,            Nmeanmags);
+  ALLOCATE (objID,     unsigned int,   Nmeanmags);
+  ALLOCATE (catID,     unsigned int,   Nmeanmags);
+
+  // assign the storage arrays
+  for (i = 0; i < Nmeanmags; i++) {
+    M    [i]   = meanmag[i].M    ;
+    dM   [i]   = meanmag[i].dM   ;
+    Xm   [i]   = meanmag[i].Xm   ;
+    Nsec [i]   = meanmag[i].Nsec ;
+    objID[i]   = meanmag[i].objID;
+    catID[i]   = meanmag[i].catID;
+  }
+
+  // add the columns to the output array
+  gfits_set_bintable_column (&theader, &ftable, "MAG",       M,         Nmeanmags);
+  gfits_set_bintable_column (&theader, &ftable, "MAG_ERR",   dM,        Nmeanmags);
+  gfits_set_bintable_column (&theader, &ftable, "MAG_CHISQ", Xm,        Nmeanmags);
+  gfits_set_bintable_column (&theader, &ftable, "NSEC",      Nsec,      Nmeanmags);
+  gfits_set_bintable_column (&theader, &ftable, "OBJ_ID",    objID,     Nmeanmags);
+  gfits_set_bintable_column (&theader, &ftable, "CAT_ID",    catID,     Nmeanmags);
+
+  free (M    );
+  free (dM   );
+  free (Xm   );
+  free (Nsec );
+  free (objID);
+  free (catID);
+
+  FILE *f = fopen (filename, "w");
+  if (!f) {
+    fprintf (stderr, "ERROR: cannot open meanmag file for output %s\n", filename);
+    return FALSE;
+  }
+
+  int status;
+  status = gfits_fwrite_header  (f, &header);
+  CHECK_STATUS (status, "ERROR: cannot write header for meanmags %s\n", filename);
+
+  status = gfits_fwrite_matrix  (f, &matrix);
+  CHECK_STATUS (status, "ERROR: cannot write matrix for meanmags %s\n", filename);
+
+  status = gfits_fwrite_Theader (f, &theader);
+  CHECK_STATUS (status, "ERROR: cannot write table header for meanmags %s\n", filename);
+
+  status = gfits_fwrite_table  (f, &ftable);
+  CHECK_STATUS (status, "ERROR: cannot write table data for meanmags %s\n", filename);
+
+  gfits_free_header (&header);
+  gfits_free_matrix (&matrix);
+  gfits_free_header (&theader);
+  gfits_free_table (&ftable);
+
+  int fd = fileno (f);
+
+  status = fflush (f);
+  CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename);
+
+  status = fsync (fd);
+  CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename);
+
+  status = fclose (f);
+  CHECK_STATUS (!status, "ERROR: problem closing meanmags file %s\n", filename);
+
+  return TRUE;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MosaicOps.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MosaicOps.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/MosaicOps.c	(revision 36516)
@@ -527,8 +527,11 @@
 
   if (!MOSAIC_ZEROPT) return (FALSE);
+  // if we are calibrating by mosaic, redefine myDet == on one of my mosaics
 
   int Nmatch = 0;
   for (i = 0; i < Ncatalog; i++) {
     for (j = 0; j < catalog[i].Nmeasure; j++) {
+      catalog[i].measureT[j].myDet = FALSE; // a detetion is not mine until proven otherwise
+
       if (TimeSelect) {
 	if (catalog[i].measureT[j].t < TSTART) continue;
@@ -582,4 +585,6 @@
   }
 
+  // this measurement is on one of my mosaics, mark it as mine.
+  catalog[cat].measureT[meas].myDet = TRUE;
   MeasureToMosaic[cat][meas] = mosID;
 
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/StarOps.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/StarOps.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/StarOps.c	(revision 36516)
@@ -896,4 +896,21 @@
 }
 
+int markObjects (Catalog *catalog, int Ncatalog) {
+
+  // How strongly do I own this object?
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      int nOwn = 0;
+      int m = catalog[i].average[j].measureOffset;
+      for (n = 0; n < catalog[i].average[j].Nmeasure; n++) {
+	if (!catalog[i].measureT[m+n].myDet) continue;
+	nOwn ++;
+      }
+      catalog[i].averageT[j].nOwn = nOwn;
+    }
+  }
+  return TRUE;
+}
+
 void clean_stars (Catalog *catalog, int Ncatalog) {
 
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c	(revision 36516)
@@ -343,4 +343,17 @@
   }
 
+  if ((N = get_argument (argc, argv, "-region-hosts"))) {
+    remove_argument (N, &argc, argv);
+    REGION_FILE = strcreate (argv[N])
+    remove_argument (N, &argc, argv);
+  }
+
+  REGION_HOST_ID = 0;
+  if ((N = get_argument (argc, argv, "-region-hostID"))) {
+    remove_argument (N, &argc, argv);
+    REGION_HOST_ID = atoi (argv[N])
+    remove_argument (N, &argc, argv);
+  }
+
   RelphotMode mode = MODE_ERROR;
   if ((N = get_argument (argc, argv, "-averages"))) {
@@ -355,11 +368,13 @@
     remove_argument (N, &argc, argv);
     mode = PARALLEL_IMAGES;
+    if (N >= argc) relphot_usage();
+    IMAGE_TABLE = strcreate (argv[N])
+    remove_argument (N, &argc, argv);
+    if (!REGION_FILE) relphot_usage();
   }
   if ((N = get_argument (argc, argv, "-parallel-regions"))) {
     remove_argument (N, &argc, argv);
     mode = PARALLEL_REGIONS;
-    if (N >= argc) relphot_usage();
-    REGION_FILE = strcreate (argv[N])
-    remove_argument (N, &argc, argv);
+    if (!REGION_FILE) relphot_usage();
   }
 
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c	(revision 36516)
@@ -25,6 +25,4 @@
 
   // for each regionHost, select images which are contained by the region
-  // XXX this is probably too expensive : I need to loop over Nimage * Nregions
-  // inverting the order would probably be faster (need to track NIMAGE for each host)
   // even faster would be to use a tree to get to the real regions...
   select_images_hostregion (regionHosts, image, Nimage);
@@ -85,7 +83,8 @@
     i = find_host_for_coords (regionHosts, Rc, Dc);
 
+    // this is a bit memory expensive : I am making a complete copy of the image table here
     off_t Nsubset = regionHosts->host[i].Nimage;
-    regionHosts->host[i].image[Nsubset] = image[j]; 
-    regionHosts->host[i].line_number[Nsubset] = j;
+    regionHosts->host[i].image[Nsubset] = image[j];
+    // regionHosts->host[i].line_number[Nsubset] = j;
 
     regionHosts->host[i].Nimage ++;
@@ -93,5 +92,5 @@
       regionHosts->host[i].NIMAGE += D_NIMAGE;
       REALLOCATE (regionHosts->host[i].image, Image, regionHosts->host[i].NIMAGE);
-      REALLOCATE (regionHosts->host[i].line_number, off_t, regionHosts->host[i].NIMAGE);
+      // REALLOCATE (regionHosts->host[i].line_number, off_t, regionHosts->host[i].NIMAGE);
     }
   }
@@ -99,4 +98,5 @@
 }
 
+// XXX add a search tree to speed this up?
 int find_host_for_coords (RegionHostTable *regionHosts, double R, double d) {
 
@@ -112,2 +112,3 @@
   }
 }
+
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/help.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/help.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/help.c	(revision 36516)
@@ -6,6 +6,6 @@
   fprintf (stderr, "       or:    relphot -averages -region RA RA DEC DEC\n");
   fprintf (stderr, "       or:    relphot -apply-offsets -region RA RA DEC DEC\n");
-  fprintf (stderr, "       or:    relphot -parallel-regions (RegionFile)\n");
-  fprintf (stderr, "       or:    relphot -parallel-images ()\n");
+  fprintf (stderr, "       or:    relphot -parallel-regions -region-hosts (RegionFile)\n");
+  fprintf (stderr, "       or:    relphot -parallel-images (ImageTable) -region-hosts (RegionFile)\n");
   fprintf (stderr, "  use -h for more usage information\n");
   exit (2);
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/indexCatalog.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/indexCatalog.c	(revision 36516)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/indexCatalog.c	(revision 36516)
@@ -0,0 +1,8 @@
+# include "relphot.h"
+
+int indexCatalogs (catalog, Ncatalog) {
+
+  // do I know 
+  
+  
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/launch_region_hosts.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/launch_region_hosts.c	(revision 36516)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/launch_region_hosts.c	(revision 36516)
@@ -0,0 +1,62 @@
+# include "relphot.h"
+
+// strextend (command, "-v");
+int strextend (char *input, char *format,..) {
+
+  char tmpfmt[1024], tmpline[1024];
+  va_list argp;
+
+  va_start (argp, format);
+  
+  snprintf (tmpfmt, 1024, "%%s %s", format);
+  vsnprintf (tmpline, 1024, tmpfmt, input, argp);
+  strcpy (input, tmpline);
+
+  return TRUE;
+}
+
+int launch_region_hosts (RegionHostTable *regionHosts) {
+
+  int i;
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+
+    RegionHostInfo *host = &regionHosts->hosts[i];
+
+    // communication files:
+    // subset images per host : CATDIR/Image.HOSTNAME.fits
+    sprintf (filename, "%s/Image.%s.fits", CATDIR, host->hostname);
+
+    // write the image subset for this host
+    ImageTableSave (filename, host->images, host->Nimages);
+
+    char command[1024];
+    snprintf (command, 1024, "relphot -parallel-images %s -region-hosts %s -region-hostID %d -D CATDIR %s -region %f %f %f %f -statmode %s -D CAMERA %s -D STAR_TOOFEW %d -minerror %f", 
+	      filename, REGION_FILE, host->hostID, CATDIR, host->Rmin, host->Rmax, host->Dmin, host->Dmax, STATMODE, CAMERA, STAR_TOOFEW, MIN_ERROR);
+
+    if (VERBOSE)       strextend (command, "-v");
+    if (VERBOSE2)      strextend (command, "-vv");
+    if (RESET)         strextend (command, "-reset");
+    if (RESET_ZEROPTS) strextend (command, "-reset-zpts");
+    if (UPDATE)        strextend (command, "-update");
+    if (!KEEP_UBERCAL) strextend (command, "-reset-ubercal");
+
+    // XXX deprecate this if we are happy with the new version
+    if (SET_MREL_VERSION != 1) strextend (command, "-set-mrel-version %d", SET_MREL_VERSION);
+
+    fprintf (stderr, "command: %s\n", command);
+    
+    // launch the job on the remote machine (no handshake)
+    int errorInfo = 0;
+    int pid = rconnect ("ssh", host->hostname, command, host->stdio, &errorInfo, FALSE);
+    if (!pid) {
+      if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", host->hostname, errorInfo);
+      exit (1);
+    }
+    host->pid = pid; // save for future reference
+  }
+
+  // XXX probably need an alternate function here...
+  HostTableWaitJobs (table, __FILE__, __LINE__, VERBOSE);
+
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/load_catalogs.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/load_catalogs.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/load_catalogs.c	(revision 36516)
@@ -108,4 +108,9 @@
 Catalog *load_catalogs_parallel (SkyList *sky, int *Ncatalog) {
 
+  char uniquer[12];
+  int TIME = time(NULL);
+  int PID = getpid();
+  snprintf (uniquer, 12, "%05d.%05d", PID, TIME % 100000);
+
   int Nsecfilt  = GetPhotcodeNsecfilt ();               // set the desired number in case we need to create the catalog
 
@@ -128,5 +133,5 @@
 
     ALLOCATE (table->hosts[i].results, char, 1024);
-    snprintf (table->hosts[i].results, 1024, "%s/relphot.catalog.subset.dat", table->hosts[i].pathname);
+    snprintf (table->hosts[i].results, 1024, "%s/relphot.catalog.%s.dat", table->hosts[i].pathname, uniquer);
 
     // options / arguments that can affect relphot_client -load:
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c	(revision 36516)
@@ -19,13 +19,14 @@
   INITTIME;
 
+  // load the RegionTable (UserRegion should not be used at this level)
+  RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE);
+
   // load the subset images belonging to this host
-  // XXX need to determine the file name for each host (here or in master)
   off_t Nimage;
-  ImageSubset *image = ImageSubsetLoad (IMAGES, &Nimage);
+  Image *image = ImageTableLoad (IMAGE_TABLE, &Nimage);
   if (!image) {
-    fprintf (stderr, "ERROR loading image subset %s\n", CATDIR);
+    fprintf (stderr, "ERROR loading image %s\n", IMAGE_TABLE);
     exit (2);
   }
-  client_logger_message ("loaded image subset data\n");
 
   // load the flat correction table (if defined)
@@ -34,6 +35,7 @@
   FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrfile, VERBOSE);
 
-  // XXX need to define the skylist for this region
-  skylist = function(region);
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+  SkyList *skylist = SkyListByBounds (sky, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
 
   /* load catalog data from region files (hostID is 0 since we are not a client */
@@ -41,4 +43,7 @@
   MARKTIME("-- load catalog data: %f sec\n", dtime);
   
+  // generate tables go from catID,objID -> catSeq,objSeq
+  indexCatalogs (catalog, Ncatalog);
+
   /* match measurements with images, mosaics */
   initImageBins  (catalog, Ncatalog, TRUE);
@@ -46,5 +51,4 @@
 
   initMosaicBins (catalog, Ncatalog, TRUE);
-  initGridBins   (catalog, Ncatalog);
   initMrel (catalog, Ncatalog);
 
@@ -54,4 +58,6 @@
   findMosaics (catalog, Ncatalog, TRUE);  /* also sets Grid values */
   MARKTIME("-- set up mosaic indexes: %f sec\n", dtime);
+
+  markObjects (catalog, Ncatalog);
 
   SAVEPLOT = FALSE;
@@ -76,8 +82,8 @@
 
     // share mean mags for objects at the boundary (number of unowned meas > 0)
-    share_mean_mags (catalog, Ncatalog);
+    share_mean_mags (catalog, Ncatalog, regionHosts, i);
 
     // load mean mags from other region hosts
-    slurp_mean_mags (catalog, Ncatalog);
+    slurp_mean_mags (catalog, Ncatalog, regionHosts, i);
 
     // set the image (Mcal) and mosaic (Mmos) zero point offsets given the mean mags and measurements
@@ -87,15 +93,7 @@
     
     // share image mags for images with non-zero unowned detections
-    share_image_mags ();
+    share_image_mags (regionHosts, i);
 
-    slurp_image_mags ();
-
-    if (PLOTSTUFF) {
-      plot_scatter (catalog, Ncatalog, flatcorr); 
-      plot_mosaics ();
-      plot_images ();
-      plot_stars (catalog, Ncatalog);
-      plot_chisq (catalog, Ncatalog);
-    }
+    slurp_image_mags (regionHosts, i);
 
     if ((i > 8) && (i % 8 == 2)) clean_measures (catalog, Ncatalog, FALSE, flatcorr); 
@@ -108,13 +106,4 @@
   }
 
-  if (PLOTSTUFF) {
-    plot_scatter (catalog, Ncatalog, flatcorr); 
-    plot_grid (catalog, flatcorr); 
-    plot_mosaics ();
-    plot_images ();
-    plot_stars (catalog, Ncatalog);
-    plot_chisq (catalog, Ncatalog);
-  }
-  
   /* set Mcal & Mmos for bad images */
   setMcal  (catalog, TRUE, flatcorr);
@@ -122,19 +111,8 @@
   MARKTIME("-- finalize Mcal values: %f sec\n", dtime);
 
-  setMcalFinal (); // copy per-mosaic calibrations to the images
+  share_image_mags (regionHosts, i);
 
-  /* at this point, we have correct cal coeffs in the image/mosaic structures */
-  for (i = 0; i < Ncatalog; i++) {
-    // these tiny values are set by BrightCatalogSplit from load_catalogs
-    free_tiny_values (&catalog[i]);
-    dvo_catalog_free (&catalog[i]);
-  }
-  freeImageBins (Ncatalog, TRUE);
-  freeMosaicBins (Ncatalog, TRUE);
-  freeGridBins (Ncatalog);
-
-  save_image_subset ();
+  // save_image_subset ();
 
   exit (0);
-
 }
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_regions.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_regions.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_regions.c	(revision 36516)
@@ -42,8 +42,8 @@
 
   /* launch processing on the parallel region hosts */
-  launch_region_hosts (regionHosts, XXX);
+  launch_region_hosts (regionHosts);
 
-  /* retrieve updated image parameters from the remote hosts */
-  retrieve_images (regionHosts, XXX);
+  // retrieve updated image parameters from the remote hosts (also set Image.mcal)
+  slurp_image_mags (regionHosts, XXX);
 
   /* update catalogs (in parallel) */
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_image_mags.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_image_mags.c	(revision 36516)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_image_mags.c	(revision 36516)
@@ -0,0 +1,116 @@
+# include "relphot.h"
+
+// we are sharing image calibrations mags for all images which (a) I own and (b) which have unowned detections
+
+// XXX Image vs Mosaic...
+int share_image_mags (RegionHostTable *regionHosts, int nloop) {
+
+  Image *images = get_images (&Nimages);
+
+  ImageMag *image_mags = NULL;
+  ALLOCATE (image_mags, ImageMag, NIMAGE_MAGS);
+
+  for (i = 0; i < Nimages; i++) {
+    
+    // XXX do I need to test the center coordinates?
+
+    // does this object have missing detections (does someone else need it?)
+    // XXX : sky objects without missing detections
+    if (imageExtra[i].Nmiss == 0) continue;
+    
+    image_mags[Nimage_mags] = something(image[i]);
+    Nimage_mags ++;
+
+    // check realloc..
+  }
+
+  // write out the image_mag fits table AND write state in some file
+  int myHost = regionHosts->index[REGION_HOST_ID];
+  char *hostname = regionHosts->hosts[myHost].hostname;
+
+  char filename[1024];
+  snprintf (filename, 1024, "%s/%s.imagemags.fits", CATDIR, hostname);
+  ImageMagsSave (filename, image_mags, Nimage_mags);
+
+  update_imsync_file (hostname, nloop);
+
+  return TRUE;
+}
+
+int slurp_image_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) {
+
+  ImageMag *image_mags = NULL;
+  ALLOCATE (image_mags, ImageMag, NIMAGE_MAGS);
+
+  for (i = 0; i < Nhost; i++) {
+    if (not_neighbor(host[i])) continue;
+    // XXX skip regions->hosts[i].hostID == REGION_HOST_ID
+
+    check_imsync_file (regionHosts->hosts[i].hostname, nloop);
+    
+    char filename[1024];
+    snprintf (filename, 1024, "%s/%s.imagemags.fits", CATDIR, regionHosts->hosts[i].hostname);
+    image_mags_subset = LoadImageMags (host[i], &Nsubset);
+
+    image_mags = merge_image_mags (image_mags, &Nimage_mags, image_mags_subset, Nsubset);
+  }
+
+  for (i = 0; i < Nimage_mags; i++) {
+    seq = Image_ID_to_seq (image_mags[i].ID);
+    image[seq].mag = image_mags[i].mag;
+  }
+}
+
+# define MSG_LENGTH 11
+
+int check_imsync_file (char *hostname, int nloop) {
+
+  char filename[1024];
+  sprintf (filename, "%s/%s.image.sync", CATDIR, hostname);
+
+  // XXX I need a rule to generate the filename for each host
+  FILE *f = NULL; 
+
+  while (retry) {
+
+    f = fopen (filename);
+    if (!f) {
+      usleep (2000000);
+      continue;
+    }
+
+    int Nread = fread (message, MSG_LENGTH, 1, f);
+    if (Nread < MSG_LENGTH) {
+      fclose (f);
+      usleep (2000000);
+      continue;
+    }
+    fclose (f);
+
+    // message is of the form: NLOOP: %03d
+    sscanf (message, "%s %d", &loop);
+    if (loop != nloop) {
+      usleep (2000000);
+      continue;
+    }
+    return TRUE;
+  }
+}
+
+int update_imsync_file (char *hostname, int nloop) {
+
+  char filename[1024];
+  sprintf (filename, "%s/%s.image.sync", CATDIR, hostname);
+
+  FILE *f = fopen (filename, "w");
+  snprintf (message, 11, "NLOOP: %03d\n", nloop);
+  
+  int Nwrite = fwrite (message, MSG_LENGTH, 1, f);
+  if (Nwrite != MSG_LENGTH) {
+    fprintf (stderr, "failure to write sync message\n");
+    exit (3);
+  }
+
+  fclose (f);
+  return TRUE;
+}
Index: /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c	(revision 36516)
@@ -3,46 +3,74 @@
 // we are sharing mean mags for all objects which (a) I own and (b) which have unowned detections
 
-int share_mean_mags () {
+# define D_NMEANMAGS 10000
+int share_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) {
 
-  SomeType *meanmags = NULL;
-  ALLOCATE (meanmags, SomeType, NMEANMAGS);
+  off_t Nmeanmags = 0;
+  off_t NMEANMAGS = D_NMEANMAGS;
 
-  // XXX skip some internal catalogs?
+  MeanMag *meanmags = NULL;
+  ALLOCATE (meanmags, MeanMag, NMEANMAGS);
+
+  int Ns;
+  int Nsecfilt = GetPhotcodeNsecfilt();
+
+  // XXX skip some catalogs based on UserPatch?
   for (i = 0; i < Ncatalog; i++) {
     for (j = 0; j < catalog[i].Naverage; j++) {
 
       // do I own this object? (in region range?)
-      if (catalog[i].average[j].R <  Rmin) continue;
-      if (catalog[i].average[j].R >= Rmax) continue;
-      if (catalog[i].average[j].D <  Dmin) continue;
-      if (catalog[i].average[j].D >= Dmax) continue;
+      if (catalog[i].average[j].R <  UserPatch.Rmin) continue;
+      if (catalog[i].average[j].R >= UserPatch.Rmax) continue;
+      if (catalog[i].average[j].D <  UserPatch.Dmin) continue;
+      if (catalog[i].average[j].D >= UserPatch.Dmax) continue;
 
       // does this object have missing detections (does someone else need it?)
-      if (catalog[i].averageExtra[j].Nmiss == 0) continue;
+      // XXX : sky objects without missing detections
+      // XXX watch out for detections which are not associated with an image (REF)
+      if (catalog[i].averageT[j].nOwn != catalog[i].averageT[j].Nmeasure) continue;
 
-      meanmags[Nmeanmags] = something(catalog[i].average[j]);
-      Nmeanmags ++;
-
-      // check realloc..
+      for (Ns = 0; Ns < Nphotcodes; Ns++) {
+	int thisCode = photcodes[Ns][0].code;
+	int Nsec = GetPhotcodeNsec(thisCode);
+	set_mean_mags (&meanmags[Nmeanmags], &catalog[i].average[j], &catalog[i].secfilt[Nsecfilt*j + Nsec], Nsec);
+	Nmeanmags ++;
+	CHECK_REALLOCATE (meanmags, MeanMag, NMEANMAGS, Nmeanmags, D_NMEANMAGS);
+      }
     }
   }
 
   // write out the meanmag fits table AND write state in some file
-  SaveMeanMags (meanmags, Nmeanmags, where);
+  int myHost = regionHosts->index[REGION_HOST_ID];
+  char *hostname = regionHosts->hosts[myHost].hostname;
+
+  char filename[1024];
+  snprintf (filename, 1024, "%s/%s.meanmags.fits", CATDIR, hostname);
+  MeanMagSave (filename, meanmags, Nmeanmags);
+
+  update_sync_file (hostname, nloop);
+
+  return TRUE;
 }
 
-int slurp_mean_mags () {
+int slurp_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) {
 
-  SomeType *meanmags = NULL;
-  ALLOCATE (meanmags, SomeType, NMEANMAGS);
+  int Nmeanmags = 0;
+  MeanMag *meanmags = NULL;
+  ALLOCATE (meanmags, MeanMag, 1);
 
   for (i = 0; i < Nhost; i++) {
     if (not_neighbor(host[i])) continue;
+    // XXX skip regions->hosts[i].hostID == REGION_HOST_ID
 
-    check_sync_file (host[i]);
+    check_sync_file (regionHosts->hosts[i].hostname, nloop);
     
-    meanmagsSubset = LoadMeanMags (host[i], &Nsubset);
+    char filename[1024];
+    snprintf (filename, 1024, "%s/%s.meanmags.fits", CATDIR, regionHosts->hosts[i].hostname);
 
-    merge_mean_mags ();
+    int Nsubset = 0;
+    MeanMag *meanmagsSubset = MeanMagLoad (host[i], &Nsubset);
+
+    // merge_mean_mags reallocs meanmags and frees the input meanmagsSubset
+    meanmags = merge_mean_mags (meanmags, &Nmeanmags, meanmagsSubset, Nsubset);
   }
 
@@ -51,10 +79,93 @@
     catID = meanmags[i].catID;
 
+    // set the mean mag
+    // XXX I need to figure out how to go from catID,objID to catSeq,objSeq
     catSeq = catalog_ID_to_seq (catID);
-    objSeq = object_ID_to_seq (objID);
+    objSeq = object_ID_to_seq (catID, objID);
 
-    // set the mean mag
     Nsecfit = photcode_to_secfilt (meanmags[i].photcode);
     catalog[catSeq].secfilt[objSeq*Nsec + Nsecfilt].mag = meanmags[i].mag;
   }
 }
+
+# define MSG_LENGTH 11
+
+int check_sync_file (char *hostname, int nloop) {
+
+  char filename[1024];
+  sprintf (filename, "%s/%s.meanmags.sync", CATDIR, hostname);
+
+  // XXX I need a rule to generate the filename for each host
+  FILE *f = NULL; 
+
+  while (retry) {
+
+    f = fopen (filename);
+    if (!f) {
+      usleep (2000000);
+      continue;
+    }
+
+    int Nread = fread (message, MSG_LENGTH, 1, f);
+    if (Nread < MSG_LENGTH) {
+      fclose (f);
+      usleep (2000000);
+      continue;
+    }
+    fclose (f);
+
+    // message is of the form: NLOOP: %03d
+    sscanf (message, "%s %d", &loop);
+    if (loop != nloop) {
+      usleep (2000000);
+      continue;
+    }
+    return TRUE;
+  }
+}
+
+int update_sync_file (char *hostname, int nloop) {
+
+  char filename[1024];
+  sprintf (filename, "%s/%s.meanmags.sync", CATDIR, hostname);
+
+  FILE *f = fopen (filename, "w");
+  snprintf (message, 11, "NLOOP: %03d\n", nloop);
+  
+  int Nwrite = fwrite (message, MSG_LENGTH, 1, f);
+  if (Nwrite != MSG_LENGTH) {
+    fprintf (stderr, "failure to write sync message\n");
+    exit (3);
+  }
+
+  fclose (f);
+  return TRUE;
+}
+
+int set_mean_mags (MeanMag *meanmags, Average *average, SecFilt *secfilt, int Nsec) {
+
+  meanmags->M  = secfilt->M;
+  meanmags->dM = secfilt->dM;
+  meanmags->Xm = secfilt->Xm;
+  meanmags->Nsec = Nsec; // key to secfilt entry
+
+  meanmags->objID = average->objID;
+  meanmags->catID = average->catID;
+
+  return TRUE;
+}
+
+MeanMag *merge_mean_mags (MeanMag *target, int *ntarget, MeanMag *source, int Nsource) {
+
+  REALLOCATE (target, MeanMag, *ntarget + Nsource);
+  for (i = 0; i < Nsource; i++) {
+    int n = i + *ntarget;
+    target[n] = source[i];
+  }
+  
+  free (source);
+
+  *ntarget += Nsource;
+  return (target);
+}
+
Index: /branches/eam_branches/ipp-20140206/Ohana/src/uniphot/src/update_dvo_setphot.c
===================================================================
--- /branches/eam_branches/ipp-20140206/Ohana/src/uniphot/src/update_dvo_setphot.c	(revision 36515)
+++ /branches/eam_branches/ipp-20140206/Ohana/src/uniphot/src/update_dvo_setphot.c	(revision 36516)
@@ -2,4 +2,5 @@
 int update_dvo_setphot_parallel (SkyTable *sky, Image *image, off_t Nimage);
 
+// XXX convert this to use the ImageSubset to save space
 int update_dvo_setphot (Image *image, off_t Nimage, FlatCorrectionTable *flatcorr) {