Index: branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/libdvo/include/dvo.h	(revision 36503)
@@ -284,4 +284,24 @@
   short *index;
 } HostTable;
+
+// A RegionHost processes data for some region in parallel with other regions
+typedef struct {
+  double Rmin;
+  double Rmax;
+  double Dmin;
+  double Dmax;
+  int hostID;		      // remove machine ID in SkyTable
+  int stdio[3]; 	      // fd's for communication with the remote host
+  int pid;		      // remote process ID
+  char *hostname;
+  ImageSubset *image;
+  int line_numbers;
+} RegionHostInfo;
+
+typedef struct {
+  int Nhosts;
+  RegionHostInfo *hosts;
+  short *index;
+} RegionHostTable;
 
 // special-case function:
Index: branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c	(revision 36503)
+++ branches/eam_branches/ipp-20140206/Ohana/src/libdvo/src/RegionHostTable.c	(revision 36503)
@@ -0,0 +1,145 @@
+# include "dvo.h"
+# include <sys/types.h>
+# include <sys/wait.h>
+
+void InitRegionHosts (RegionHostInfo *hosts, int Nhosts, int NHOSTS) {
+
+  int i;
+  for (i = Nhosts; i < NHOSTS; i++) {
+    hosts[i].hostname = NULL;
+    hosts[i].stdio[HOST_STDIN] = -1;
+    hosts[i].stdio[HOST_STDOUT] = -1;
+    hosts[i].stdio[HOST_STDERR] = -1;
+    hosts[i].pid = 0;
+
+    hosts[i].Rmin = NAN;
+    hosts[i].Rmax = NAN;
+    hosts[i].Dmin = NAN;
+    hosts[i].Dmax = NAN;
+  }
+  return;
+}
+
+void FreeRegionHosts (RegionHostInfo *hosts, int Nhosts) {
+
+  int i;
+  for (i = 0; i < Nhosts; i++) {
+    free (hosts[i].hostname);
+  }
+  free (hosts);
+  return;
+}
+
+void FreeRegionHostTable (RegionHostTable *table) {
+
+  if (!table) return;
+  if (table->hosts) {
+    FreeRegionHosts (table->hosts, table->Nhosts);
+  }
+  if (table->index) free (table->index);
+  free (table);
+  return;
+}
+
+RegionHostTable *RegionHostTableLoad (char *catdir, char *rootname) {
+
+  int i, Nline;
+
+  char *filename = NULL;
+
+  ALLOCATE (filename, char, strlen(catdir) + strlen(rootname) + 2); // one slash and one EOL
+  sprintf (filename, "%s/%s", catdir, rootname);
+
+  FILE *f = fopen (filename, "r");
+  if (!f) {
+    fprintf (stderr, "failed to open host table %s\n", filename);
+    free (filename);
+    return NULL;
+  }
+
+  // simple format: ID hostname pathname
+  
+  int NHOSTS = 16;
+  int Nhosts = 0;
+  RegionHostInfo *hosts = NULL;
+  ALLOCATE (hosts, RegionHostInfo, NHOSTS);
+  InitRegionHosts (hosts, Nhosts, NHOSTS);
+
+  int maxID = 0;
+
+  for (Nline = 0; TRUE; Nline ++) {
+    int ID;
+    char tmphost[1024];
+    char tmppath[1024];
+    char line[1024];
+
+    // XXXX use this for safety: int status = scan_line_maxlen (f, line, 1024);
+    int status = scan_line (f, line);
+    if (status == EOF) break;
+
+    // find first non-whitespace char & skip commented lines
+    for (i = 0; OHANA_WHITESPACE (line[i]); i++);
+    if (line[i] == '#') continue;
+    if (line[i] == 0) continue;
+
+    status = sscanf (line, "%d %1023s %lf %lf %lf %lf", &ID, tmphost, &Rmin, &Rmax, &Dmin, &Dmax);
+    if (status != 6) {
+      fprintf (stderr, "error reading line %d of region host table %s\n", Nline, filename);
+      FreeRegionHosts (hosts, Nhosts);
+      free (filename);
+      fclose (f);
+      return NULL;
+    }
+
+    // check the validity of ID (0 < ID < MAX_SHORT)
+
+    if (ID < 1) {
+      fprintf (stderr, "invalid host ID %d\n", ID);
+      exit (1);
+    }
+    if (ID > 255) {
+      fprintf (stderr, "invalid host ID %d\n", ID);
+      exit (1);
+    }
+    maxID = MAX(maxID, ID);
+
+    hosts[Nhosts].hostID = ID;
+    hosts[Nhosts].hostname = strcreate(tmphost);
+
+    hosts[Nhosts].Rmin = Rmin;
+    hosts[Nhosts].Rmax = Rmax;
+    hosts[Nhosts].Dmin = Dmin;
+    hosts[Nhosts].Dmax = Dmax;
+    
+    // InitIOBuffer (&hosts[Nhosts].stdout, 1000);
+    // InitIOBuffer (&hosts[Nhosts].stderr, 1000);
+
+    Nhosts ++;
+    if (Nhosts >= NHOSTS) {
+      NHOSTS += 16;
+      REALLOCATE (hosts, RegionHostInfo, NHOSTS);
+      InitRegionHosts (hosts, Nhosts, NHOSTS);
+    }
+  }    
+
+  RegionHostTable *table = NULL;
+  ALLOCATE (table, RegionHostTable, 1);
+  table->Nhosts = Nhosts;
+  table->hosts = hosts;
+
+  ALLOCATE (table->index, short, maxID + 1);
+  for (i = 0; i <= maxID; i++) table->index[i] = -1;
+
+  for (i = 0; i < table->Nhosts; i++) {
+    if (table->index[table->hosts[i].hostID] != -1) {
+      fprintf (stderr, "error: duplicate hostID %d\n", table->hosts[i].hostID);
+      exit (1);
+    }
+    table->index[table->hosts[i].hostID] = i;
+  }
+
+  free (filename);
+  fclose (f);
+  return table;
+}
+
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/doc/parallel.txt	(revision 36503)
@@ -1,2 +1,40 @@
+
+2014.02.06
+
+** a single region-level host owns images for which the center lands
+   in its region.
+
+** it owns detections which come from images which it owns
+   
+** it owns objects which land in its region
+
+Extending parallel relphot processing to split the sky (and images)
+into regions, each of which runs in parallel at the same time:
+
+Top Level (relphot -parallel-images):
+ * define regions of the sky -> hosts
+ * load images, assign to hosts
+ * launch region-level jobs on remote hosts
+
+ Region Level (relphot -parallel-images-region)
+  * load my image subset table
+  * request objects and detections for my skyregion
+  * match images & objects, etc
+
+  * update my image parameters
+  * update my detections
+  * write out detections 
+  * load detections from my border hosts
+  * match to my objects
+  * update my objects
+  * write out objects
+  * load objects from my border hosts
+  * update images
+    (iterate N times)
+  * write out image parameters
+
+Top Level
+  * read image parameters, update
+  * update objects
 
 2012.02.13
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/args.c	(revision 36503)
@@ -354,4 +354,10 @@
   }
 
+  ParallelImages = FALSE;
+  if ((N = get_argument (argc, argv, "-parallel-images"))) {
+    remove_argument (N, &argc, argv);
+    ParallelImages = TRUE;
+  }
+
   if (UpdateAverages && (argc == 1)) return TRUE;
   if (argc != 2) relphot_usage ();
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c	(revision 36503)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/assign_images.c	(revision 36503)
@@ -0,0 +1,108 @@
+# include "relphot.h"
+
+// This function generates a subset of the images based on selections.  Input db has already
+// been loaded with the raw fits table data
+SkyList *assign_images (FITS_DB *db, RegionHostTable *regionHosts) {
+
+  off_t Nimage;
+
+  INITTIME;
+
+  // convert database table to internal structure (binary to Image)
+  // 'image' points to the same memory as db->ftable->buffer
+  Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped);
+  if (!image) {
+      fprintf (stderr, "ERROR: failed to read images\n");
+      exit (2);
+  }
+  MARKTIME("convert image table to internal structure: %f sec\n", dtime);
+
+  // *** NOTE : for the moment, regions must be in the range 0 - 360, -90 - +90
+
+  if (VERBOSE) fprintf (stderr, "finding images\n");
+  BuildChipMatch (image, Nimage);
+  MARKTIME("build chip match for %d images: %f sec\n", (int) Nimage, dtime);
+
+  // for each regionHost, select images which are contained by the region
+  // XXX this is probably too expensive : I need to loop over Nimage * Nregions
+  // inverting the order would probably be faster (need to track NIMAGE for each host)
+  // even faster would be to use a tree to get to the real regions...
+  select_images_hostregion (regionHosts, image, Nimage);
+
+}
+
+# define D_NIMAGE 1000
+
+int select_images_hostregion (RegionHostTable *hosts, Image *image, off_t Nimage) {
+
+  INITTIME;
+  
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+    regionHosts->host[i].Nimage = 0;
+    regionHosts->host[i].NIMAGE = D_NIMAGE;
+    ALLOCATE (regionHosts->host[i].image, Image, regionHosts->host[i].NIMAGE);
+  }
+
+  for (j = 0; j < Nimage; j++) {
+    
+    /* exclude images by photcode */
+    ecode = GetPhotcodeEquivCodebyCode (image[j].photcode);
+    found = FALSE;
+    int Ns;
+    for (Ns = 0; !found && (Ns < Nphotcodes); Ns++) {
+      if (ecode == photcodes[Ns][0].code) found = TRUE;
+    }
+    if (!found) continue;
+
+    /* exclude images by time */
+    if (TimeSelect) {
+      if (image[j].tzero < TSTART) continue;
+      if (image[j].tzero > TSTOP) continue;
+    }
+    
+    // this adds 1.3 sec for 3M images
+    if (!FindMosaicForImage (image, Nimage, i)) {
+      if (VERBOSE2) fprintf (stderr, "cannot find mosaic for "OFF_T_FMT"\n", i);
+      continue;
+    }
+
+    /* define image center - note the DIS images (mosaic phu) are special */
+    if (!strcmp(&image[j].coords.ctype[4], "-DIS")) {
+      Xc = 0.0; Yc = 0.0;
+    } else {
+      Xc = 0.5*image[j].NX; 
+      Yc = 0.5*image[j].NY;
+    }
+
+    XY_to_RD (&Rc, &Dc, Xc, Yc, &image[j].coords);
+    Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0);
+
+    i = find_host_for_coords (regionHosts, Rc, Dc);
+
+    off_t Nsubset = regionHosts->host[i].Nimage;
+    regionHosts->host[i].image[Nsubset] = image[j]; 
+    regionHosts->host[i].line_number[Nsubset] = j;
+
+    regionHosts->host[i].Nimage ++;
+    if (regionHosts->host[i].Nimage == regionHosts->host[i].NIMAGE) {
+      regionHosts->host[i].NIMAGE += D_NIMAGE;
+      REALLOCATE (regionHosts->host[i].image, Image, regionHosts->host[i].NIMAGE);
+      REALLOCATE (regionHosts->host[i].line_number, off_t, regionHosts->host[i].NIMAGE);
+    }
+  }
+  return TRUE;
+}
+
+int find_host_for_coords (RegionHostTable *regionHosts, double R, double d) {
+
+  for (i = 0; i < regionHosts->Nhosts; i++) {
+
+    RegionHostInfo *host = regionHosts->hosts[i];
+    if (Rc <  host->Rmin) continue;
+    if (Rc >= host->Rmax) continue;
+    if (Dc <  host->Dmin) continue;
+    if (Dc >= host->Dmax) continue;
+
+    return i;
+  }
+}
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot.c	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot.c	(revision 36503)
@@ -22,4 +22,16 @@
   }
 
+  /* the object analysis is a separate process iterating over catalogs */
+  if (ParallelImages) {
+    relphot_parallel_images ();
+    exit (0);
+  }
+
+  /* the object analysis is a separate process iterating over catalogs */
+  if (ParallelRegion) {
+    relphot_parallel_region ();
+    exit (0);
+  }
+
   relphot_images ();
 
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_images.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_images.c	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_images.c	(revision 36503)
@@ -244,5 +244,7 @@
   if (!UPDATE) exit (0);
   
-  /* load catalog data from region files, update Mrel include all data */
+  /* Load catalog data from region files, update Mrel include all data.  In a parallel
+     context, this function writes the image parameters as a subset table for the remote
+     clients */
   reload_catalogs (skylist, flatcorr, 0, NULL);
   MARKTIME("-- updated all catalogs: %f sec\n", dtime);
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c	(revision 36503)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_images.c	(revision 36503)
@@ -0,0 +1,60 @@
+# include "relphot.h"
+
+// measure.M contains: -2.5*log(DN) + 2.5*log(exptime) + 25.0 
+
+int relphot_parallel_images () {
+
+  int i, status, Ncatalog;
+  Catalog *catalog = NULL;
+  FITS_DB db;
+  SkyList *skylist = NULL;
+
+  INITTIME;
+
+  // load the RegionTable (UserRegion should not be used at this level)
+  RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, "RegionHostTable.dat");
+
+  /* register database handle with shutdown procedure */
+  set_db (&db);
+  db.mode   = dvo_catalog_catmode (CATMODE);
+  db.format = dvo_catalog_catformat (CATFORMAT);
+
+  /* lock and load the image db table */
+  status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT));
+  if (!status && UPDATE) {
+    fprintf (stderr, "error\n");
+    Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
+  }
+
+  if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) {
+    // if the Image.dat file is missing, db.dbstate will have a value of either:
+    // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE)
+    Shutdown ("ERROR: database %s contains no image data", CATDIR);
+  }
+
+  // read data from Image.dat file
+  if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename);
+  MARKTIME("-- load image data: %f sec\n", dtime);
+
+  /* assign the images to the different region hosts */
+  XXX = assign_images (&db, regionHosts);
+  MARKTIME("-- assign images: %f sec\n", dtime);
+
+  /* launch processing on the parallel region hosts */
+  launch_region_hosts (regionHosts, XXX);
+
+  /* retrieve updated image parameters from the remote hosts */
+  retrieve_images (regionHosts, XXX);
+
+  /* update catalogs (in parallel) */
+  reload_catalogs ();
+
+  // do not save changes if we did not make changes. 
+  dvo_image_update (&db, VERBOSE);
+
+  if (!UPDATE) dvo_image_unlock (&db); 
+  dvo_image_unlock (&db); 
+
+  exit (0);
+}
+
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_region.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_region.c	(revision 36503)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/relphot_parallel_region.c	(revision 36503)
@@ -0,0 +1,140 @@
+# include "relphot.h"
+
+/* This function is essentially identical to relphot_images, except:
+
+ * load the subset images saved by the master node
+ * distinguish detections we own (touch our images) and those we don't
+ * distinguish objects we own (in region) and those we don't
+ * update the unowned detections for owned objects from neighbor regions
+ * update the unowned objects for owned detections 
+ */
+
+int relphot_parallel_images () {
+
+  int i, status, Ncatalog;
+  Catalog *catalog = NULL;
+  FITS_DB db;
+  SkyList *skylist = NULL;
+
+  INITTIME;
+
+  // load the subset images belonging to this host
+  // XXX need to determine the file name for each host (here or in master)
+  off_t Nimage;
+  ImageSubset *image = ImageSubsetLoad (IMAGES, &Nimage);
+  if (!image) {
+    fprintf (stderr, "ERROR loading image subset %s\n", CATDIR);
+    exit (2);
+  }
+  client_logger_message ("loaded image subset data\n");
+
+  // load the flat correction table (if defined)
+  char flatcorrfile[256];
+  sprintf (flatcorrfile, "%s/flatcorr.fits", CATDIR);
+  FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrfile, VERBOSE);
+
+  // XXX need to define the skylist for this region
+  skylist = function(region);
+
+  /* load catalog data from region files (hostID is 0 since we are not a client */
+  catalog = load_catalogs (skylist, &Ncatalog, 0, NULL);
+  MARKTIME("-- load catalog data: %f sec\n", dtime);
+  
+  /* match measurements with images, mosaics */
+  initImageBins  (catalog, Ncatalog, TRUE);
+  MARKTIME("-- make image bins: %f sec\n", dtime);
+
+  initMosaicBins (catalog, Ncatalog, TRUE);
+  initGridBins   (catalog, Ncatalog);
+  initMrel (catalog, Ncatalog);
+
+  findImages (catalog, Ncatalog, TRUE);
+  MARKTIME("-- set up image indexes: %f sec\n", dtime);
+
+  findMosaics (catalog, Ncatalog, TRUE);  /* also sets Grid values */
+  MARKTIME("-- set up mosaic indexes: %f sec\n", dtime);
+
+  SAVEPLOT = FALSE;
+
+  setExclusions (catalog, Ncatalog, TRUE);
+
+  global_stats (catalog, Ncatalog, flatcorr);
+
+  if (PLOTSTUFF) {
+    plot_star_coords (catalog, Ncatalog);
+    // plot_mosaic_fields (catalog);
+  }
+
+  // XXX : add this in the loop at various points
+  // if (PLOTSTUFF) plot_scatter (catalog, Ncatalog, flatcorr); 
+
+  /* determine fit values */
+  for (i = 0; i < NLOOP; i++) {
+
+    // set the mean stellar mags given the measurements and the image calibrations
+    setMrel  (catalog, Ncatalog, flatcorr);
+
+    // share mean mags for objects at the boundary (number of unowned meas > 0)
+    share_mean_mags (catalog, Ncatalog);
+
+    // load mean mags from other region hosts
+    slurp_mean_mags (catalog, Ncatalog);
+
+    // set the image (Mcal) and mosaic (Mmos) zero point offsets given the mean mags and measurements
+    setMcal  (catalog, FALSE, flatcorr);
+    setMmos  (catalog, FALSE, flatcorr);
+    MARKTIME("-- set Mrel, Mcal, Mmos, Mgrid : %f sec\n", dtime);
+    
+    // share image mags for images with non-zero unowned detections
+    share_image_mags ();
+
+    slurp_image_mags ();
+
+    if (PLOTSTUFF) {
+      plot_scatter (catalog, Ncatalog, flatcorr); 
+      plot_mosaics ();
+      plot_images ();
+      plot_stars (catalog, Ncatalog);
+      plot_chisq (catalog, Ncatalog);
+    }
+
+    if ((i > 8) && (i % 8 == 2)) clean_measures (catalog, Ncatalog, FALSE, flatcorr); 
+    if ((i > 8) && (i % 8 == 3)) clean_stars (catalog, Ncatalog);
+    if ((i > 8) && (i % 8 == 5)) clean_mosaics ();
+    if ((i > 8) && (i % 8 == 5)) clean_images ();
+
+    if (i % 3 == 2) global_stats (catalog, Ncatalog, flatcorr);
+    MARKTIME("-- finished loop %d: %f sec\n", i, dtime);
+  }
+
+  if (PLOTSTUFF) {
+    plot_scatter (catalog, Ncatalog, flatcorr); 
+    plot_grid (catalog, flatcorr); 
+    plot_mosaics ();
+    plot_images ();
+    plot_stars (catalog, Ncatalog);
+    plot_chisq (catalog, Ncatalog);
+  }
+  
+  /* set Mcal & Mmos for bad images */
+  setMcal  (catalog, TRUE, flatcorr);
+  setMmos  (catalog, TRUE, flatcorr);
+  MARKTIME("-- finalize Mcal values: %f sec\n", dtime);
+
+  setMcalFinal (); // copy per-mosaic calibrations to the images
+
+  /* at this point, we have correct cal coeffs in the image/mosaic structures */
+  for (i = 0; i < Ncatalog; i++) {
+    // these tiny values are set by BrightCatalogSplit from load_catalogs
+    free_tiny_values (&catalog[i]);
+    dvo_catalog_free (&catalog[i]);
+  }
+  freeImageBins (Ncatalog, TRUE);
+  freeMosaicBins (Ncatalog, TRUE);
+  freeGridBins (Ncatalog);
+
+  save_image_subset ();
+
+  exit (0);
+
+}
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/setMrelCatalog.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/setMrelCatalog.c	(revision 36494)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/setMrelCatalog.c	(revision 36503)
@@ -259,4 +259,9 @@
       list[N]  = Msys - Mcal - Mmos - Mgrid;
 
+      // NOTE: 
+      // Msys is measure[i].M + zp corrections
+      // Mcal is image[j].Mcal
+      // Mmos and Mgrid are offsets for mosaic and grid
+
       // up-weight the ubercal values (or convergence can take a long time...)
       if (measureT[k].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
Index: branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c
===================================================================
--- branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c	(revision 36503)
+++ branches/eam_branches/ipp-20140206/Ohana/src/relphot/src/share_mean_mags.c	(revision 36503)
@@ -0,0 +1,61 @@
+# include "relphot.h"
+
+// we are sharing mean mags for all objects which (a) I own and (b) which have unowned detections
+
+int share_mean_mags () {
+
+  SomeType *meanmags = NULL;
+  ALLOCATE (meanmags, SomeType, NMEANMAGS);
+
+  // XXX skip some internal catalogs?
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+
+      // do I own this object? (in region range?)
+      if (catalog[i].average[j].R <  Rmin) continue;
+      if (catalog[i].average[j].R >= Rmax) continue;
+      if (catalog[i].average[j].D <  Dmin) continue;
+      if (catalog[i].average[j].D >= Dmax) continue;
+
+      // does this object have missing detections (does someone else need it?)
+      if (catalog[i].averageExtra[j].Nmiss == 0) continue;
+
+      meanmags[Nmeanmags] = something(catalog[i].average[j]);
+      Nmeanmags ++;
+
+      // check realloc..
+    }
+  }
+
+  // write out the meanmag fits table AND write state in some file
+  SaveMeanMags (meanmags, Nmeanmags, where);
+}
+
+int slurp_mean_mags () {
+
+  SomeType *meanmags = NULL;
+  ALLOCATE (meanmags, SomeType, NMEANMAGS);
+
+  for (i = 0; i < Nhost; i++) {
+    if (not_neighbor(host[i])) continue;
+
+    check_sync_file (host[i]);
+    
+    meanmagsSubset = LoadMeanMags (host[i], &Nsubset);
+
+    merge_mean_mags ();
+  }
+
+  for (i = 0; i < Nmeanmags; i++) {
+    objID = meanmags[i].objID;
+    catID = meanmags[i].catID;
+
+    catSeq = catalog_ID_to_seq (catID);
+    objSeq = object_ID_to_seq (objID);
+
+    // set the mean mag
+    Nsecfit = photcode_to_secfilt (meanmags[i].photcode);
+    catalog[catSeq].secfilt[objSeq*Nsec + Nsecfilt].mag = meanmags[i].mag;
+  }
+  SaveMeanMags (meanmags, Nmeanmags, where);
+}