Index: /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded.c
===================================================================
--- /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded.c	(revision 33894)
+++ /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded.c	(revision 33895)
@@ -1,3 +1,11 @@
 # include "addstar.h"
+
+/* upgrade to parallel mode:
+   
+   setup up skylist and call resort_unthreaded_catalogs ()
+   
+   resort_unthreaded_catalogs has a parallel and serial version, etc.
+
+ */
 
 int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky) {
@@ -44,56 +52,5 @@
   }
 
-  // XXX ALLOCATE catalog[Nthreads]
-
-  /* match stars to existing catalog data (or otherwise manipulate catalog data) */
-  Naverage = Nmeasure = 0;
-  for (i = 0; i < skylist[0].Nregions; i++) {
-
-    // XXX BLOCK here for an empty thread
-
-    // set the parameters which guide catalog open/load/create
-    catalog.filename  = skylist[0].filename[i];
-    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS;
-    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
-
-    // an error exit status here is a significant error (disk I/O or file access)
-    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
-      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename);
-      exit (2);
-    }
-
-    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
-    if (catalog.Naves_disk == 0) {
-      dvo_catalog_unlock (&catalog);
-      dvo_catalog_free (&catalog);
-      continue;
-    }
-
-    // this is an overloaded value to mean 'force sort'
-    if (options->nosort == 3) catalog.sorted = FALSE;
-
-    // XXX send these to a free thread
-
-    if (OLD_RESORT) { 
-      resort_catalog_old (&catalog);
-    } else {
-      resort_catalog (&catalog);
-    }
-
-    // XXX wait for completed threads
-
-    /* report total updated values */
-    Naverage += catalog.Naverage;
-    Nmeasure += catalog.Nmeasure;
-
-    // write out catalog, if appropriate
-    SetProtect (TRUE);
-    dvo_catalog_save (&catalog, VERBOSE);
-    SetProtect (FALSE);
-    dvo_catalog_unlock (&catalog);
-    dvo_catalog_free (&catalog);
-  }
+  resort_unthreaded_catalogs ()
 
   gettimeofday (&stop, NULL);
Index: /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded_catalogs.c
===================================================================
--- /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded_catalogs.c	(revision 33895)
+++ /branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded_catalogs.c	(revision 33895)
@@ -0,0 +1,146 @@
+# include "addstar.h"
+
+int resort_unthreaded_catalogs (int hostID, char *hostpath) {
+
+  if (PARALLEL && !hostID) {
+    resort_unthreaded_parallel ();
+    return TRUE;
+  }
+
+  /* match stars to existing catalog data (or otherwise manipulate catalog data) */
+  Naverage = Nmeasure = 0;
+  for (i = 0; i < skylist[0].Nregions; i++) {
+
+    // does this host ID match the desired location for the table?
+    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
+
+    // XXX BLOCK here for an empty thread
+
+    // set the parameters which guide catalog open/load/create
+    catalog.filename  = skylist[0].filename[i];
+
+    char hostfile[1024];
+    snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
+    catalog.filename  = hostID ? hostfile : skylist[0].filename[i];
+
+    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
+    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
+    catalog.catflags  = LOAD_AVES | LOAD_MEAS;
+    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+
+    // an error exit status here is a significant error (disk I/O or file access)
+    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
+      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename);
+      exit (2);
+    }
+
+    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
+    if (catalog.Naves_disk == 0) {
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+
+    // this is an overloaded value to mean 'force sort'
+    if (options->nosort == 3) catalog.sorted = FALSE;
+
+    // XXX send these to a free thread
+
+    if (OLD_RESORT) { 
+      resort_catalog_old (&catalog);
+    } else {
+      resort_catalog (&catalog);
+    }
+
+    // XXX wait for completed threads
+
+    /* report total updated values */
+    Naverage += catalog.Naverage;
+    Nmeasure += catalog.Nmeasure;
+
+    // write out catalog, if appropriate
+    SetProtect (TRUE);
+    dvo_catalog_save (&catalog, VERBOSE);
+    SetProtect (FALSE);
+    dvo_catalog_unlock (&catalog);
+    dvo_catalog_free (&catalog);
+  }
+}
+
+// CATDIR is supplied globally
+# define DEBUG 1
+int resort_unthreaded_parallel (SkyList *sky) {
+
+  // launch the setphot_client jobs to the parallel hosts
+
+  // load the list of hosts
+  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+  if (!table) {
+    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+    exit (1);
+  }    
+
+  int i;
+  for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    // options / arguments that can affect relphot_client -update-objects:
+    // VERBOSE, VERBOSE2
+    // KEEP_UBERCAL
+    // RESET (-reset)
+    // TimeSelect -time
+    // DophotSelect
+    // (note that psfQual is applied rigidly at 0.85, as is the galaxy test)
+    // MAG_LIM
+    // SIGMA_LIM
+    // ImagSelect, ImagMin, ImagMax
+    // MaxDensityUse, MaxDensityValue
+
+    char command[1024];
+    snprintf (command, 1024, "relphot_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -D CAMERA %s -D STAR_TOOFEW %d -minerror %f", 
+	      table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, CAMERA, STAR_TOOFEW, MIN_ERROR);
+
+    char tmpline[1024];
+    if (VERBOSE)       { snprintf (tmpline, 1024, "%s -v",              command);                    strcpy (command, tmpline); }
+    if (VERBOSE2)      { snprintf (tmpline, 1024, "%s -vv",             command); 		     strcpy (command, tmpline); }
+    if (RESET)         { snprintf (tmpline, 1024, "%s -reset",          command); 		     strcpy (command, tmpline); }
+    if (UPDATE)        { snprintf (tmpline, 1024, "%s -update",         command); 		     strcpy (command, tmpline); }
+    if (!KEEP_UBERCAL) { snprintf (tmpline, 1024, "%s -reset-ubercal",  command); 		     strcpy (command, tmpline); }
+
+    fprintf (stderr, "command: %s\n", command);
+
+    if (PARALLEL_MANUAL) continue;
+
+    if (PARALLEL_SERIAL) {
+      int status = system (command);
+      if (status) {
+	fprintf (stderr, "ERROR running relphot_client\n");
+	exit (2);
+      }
+    } else {
+      // launch the job on the remote machine (no handshake)
+      int errorInfo = 0;
+      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
+      if (!pid) {
+	if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+	exit (1);
+      }
+      table->hosts[i].pid = pid; // save for future reference
+    }
+  }
+
+  if (PARALLEL_MANUAL) {
+    fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
+    getchar();
+  }
+  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+  }
+
+  return TRUE;
+}      
+