Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/Makefile
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/Makefile	(revision 38656)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/Makefile	(revision 38657)
@@ -1,5 +1,5 @@
-default: photdbc photdbc_client dvodist
+default: photdbc photdbc_client dvocompress dvocompress_client dvodist
 help:
-	@echo "make options: photdbc photdbc_client dvodist"
+	@echo "make options: photdbc photdbc_client dvocompress dvocompress_client dvodist"
 
 include ../../Makefile.System
@@ -20,7 +20,11 @@
 photdbc_client: $(BIN)/photdbc_client.$(ARCH)
 
+dvocompress: $(BIN)/dvocompress.$(ARCH)
+dvocompress_client: $(BIN)/dvocompress_client.$(ARCH)
+
+
 dvodist: $(BIN)/dvodist.$(ARCH)
 
-install: $(DESTBIN)/photdbc $(DESTBIN)/photdbc_client $(DESTBIN)/dvodist
+install: $(DESTBIN)/photdbc $(DESTBIN)/photdbc_client $(DESTBIN)/dvocompress $(DESTBIN)/dvocompress_client $(DESTBIN)/dvodist
 
 PHOTDBC = \
@@ -47,4 +51,18 @@
 $(SRC)/join_stars.$(ARCH).o        \
 $(SRC)/make_subcatalog.$(ARCH).o        
+
+DVOCOMPRESS =                         \
+$(SRC)/dvocompress.$(ARCH).o	      \
+$(SRC)/dvocompress_catalogs.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o 	      \
+$(SRC)/args_dvocompress.$(ARCH).o     \
+$(SRC)/Shutdown.$(ARCH).o 
+
+DVOCOMPRESS_CLIENT = 	              \
+$(SRC)/dvocompress_client.$(ARCH).o   \
+$(SRC)/dvocompress_catalogs.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o 	      \
+$(SRC)/args_dvocompress.$(ARCH).o     \
+$(SRC)/Shutdown.$(ARCH).o
 
 DVODIST = \
@@ -91,4 +109,10 @@
 $(BIN)/photdbc_client.$(ARCH): $(PHOTDBC_CLIENT)
 
+$(DVOCOMPRESS): $(INC)/dvocompress.h
+$(BIN)/dvocompress.$(ARCH): $(DVOCOMPRESS)
+
+$(DVOCOMPRESS_CLIENT): $(INC)/dvocompress.h
+$(BIN)/dvocompress_client.$(ARCH): $(DVOCOMPRESS_CLIENT)
+
 $(DVODIST): $(INC)/dvodist.h
 $(BIN)/dvodist.$(ARCH): $(DVODIST)
Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/include/dvocompress.h
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/include/dvocompress.h	(revision 38657)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/include/dvocompress.h	(revision 38657)
@@ -0,0 +1,32 @@
+# include <ohana.h>
+# include <dvo.h>
+# include <signal.h>
+
+int    PARALLEL;
+int    PARALLEL_MANUAL;
+int    PARALLEL_SERIAL;
+
+int          HOST_ID;
+char        *HOSTDIR;
+
+# define DVO_MAX_PATH 1024
+
+/* global variables */
+int    VERBOSE;
+char  *UPDATE_CATFORMAT;  /* internal, elixir, loneos, panstarrs */
+char  *UPDATE_CATCOMPRESS;  /* ?? */
+
+SkyRegion REGION;
+
+int           args (int *argc, char **argv); 
+int           args_client (int *argc, char **argv); 
+
+int Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2);
+void check_permissions (char *basefile);
+void TrapSignal (int sig);
+void SetProtect (int mode);
+int SetSignals (void);
+void usage();
+
+int dvocompress_catalogs (char *catdir, SkyList *skylist, int hostID);
+int dvocompress_parallel (char *catdir, SkyList *skylist);
Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/args_dvocompress.c
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/args_dvocompress.c	(revision 38657)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/args_dvocompress.c	(revision 38657)
@@ -0,0 +1,208 @@
+# include "dvocompress.h"
+
+int args (int *argc, char **argv) {
+
+  int N;
+
+  if (get_argument (*argc, argv, "-h")) usage();
+  if (get_argument (*argc, argv, "--h")) usage();
+  if (get_argument (*argc, argv, "-help")) usage();
+  if (get_argument (*argc, argv, "--help")) usage();
+
+  VERBOSE = FALSE;
+  if ((N = get_argument (*argc, argv, "-v"))) {
+    VERBOSE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  // override input catalog format (PS1_V1, PS1_REF, etc)
+  UPDATE_CATFORMAT = NULL;
+  if ((N = get_argument (*argc, argv, "-set-format"))) {
+    remove_argument (N, argc, argv);
+    UPDATE_CATFORMAT = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+
+  // override input catalog format (PS1_V1, PS1_REF, etc)
+  UPDATE_CATCOMPRESS = NULL;
+  if ((N = get_argument (*argc, argv, "-set-compress"))) {
+    remove_argument (N, argc, argv);
+    UPDATE_CATCOMPRESS = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+
+  if (!UPDATE_CATFORMAT && !UPDATE_CATCOMPRESS) {
+    fprintf (stderr, "neither -set-compress nor -set-format are specified : this is a NOP\n");
+    exit (2);
+  }
+
+  /* specify portion of the sky */
+  REGION.Rmin = 0;
+  REGION.Rmax = 360;
+  REGION.Dmin = -90;
+  REGION.Dmax = +90;
+  if ((N = get_argument (*argc, argv, "-region"))) {
+    remove_argument (N, argc, argv);
+    REGION.Rmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Rmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Dmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Dmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+
+    if (REGION.Rmin == REGION.Rmax) {
+      fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n");
+      exit (2);
+    }
+    if (REGION.Dmin == REGION.Dmax) {
+      fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n");
+      exit (2);
+    }
+  }
+
+  // XXX for the moment, make this selection manual.  it needs to be automatic 
+  // based on the state of the SkyTable
+  PARALLEL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel"))) {
+    PARALLEL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the remote jobs and waiting for completion,
+  // relphot will simply list the remote command and wait for the user to signal completion
+  PARALLEL_MANUAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
+    PARALLEL = TRUE; // -parallel-manual implies -parallel
+    PARALLEL_MANUAL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the relphot_client jobs remotely, they are 
+  // run in serial via 'system'
+  PARALLEL_SERIAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
+    if (PARALLEL_MANUAL) {
+      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
+      exit (1);
+    }
+    PARALLEL = TRUE; // -parallel-serial implies -parallel
+    PARALLEL_SERIAL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  if (*argc != 2) usage();
+
+  return (TRUE);
+}
+
+int args_client (int *argc, char **argv) {
+
+  int N;
+
+  PARALLEL = FALSE;
+  PARALLEL_MANUAL = FALSE;
+  PARALLEL_SERIAL = FALSE;
+
+  if (get_argument (*argc, argv, "-h")) usage();
+  if (get_argument (*argc, argv, "--h")) usage();
+  if (get_argument (*argc, argv, "-help")) usage();
+  if (get_argument (*argc, argv, "--help")) usage();
+
+  HOST_ID = 0;
+  if ((N = get_argument (*argc, argv, "-hostID"))) {
+    remove_argument (N, argc, argv);
+    HOST_ID = atoi (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOST_ID) usage();
+
+  HOSTDIR = NULL;
+  if ((N = get_argument (*argc, argv, "-hostdir"))) {
+    remove_argument (N, argc, argv);
+    HOSTDIR = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOSTDIR) usage();
+
+  VERBOSE = FALSE;
+  if ((N = get_argument (*argc, argv, "-v"))) {
+    VERBOSE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  if ((N = get_argument (*argc, argv, "-vv"))) {
+    VERBOSE = 2;
+    remove_argument (N, argc, argv);
+  }
+
+  // override input catalog format (PS1_V1, PS1_REF, etc)
+  UPDATE_CATFORMAT = NULL;
+  if ((N = get_argument (*argc, argv, "-set-format"))) {
+    remove_argument (N, argc, argv);
+    UPDATE_CATFORMAT = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+
+  // override input catalog format (PS1_V1, PS1_REF, etc)
+  UPDATE_CATCOMPRESS = NULL;
+  if ((N = get_argument (*argc, argv, "-set-compress"))) {
+    remove_argument (N, argc, argv);
+    UPDATE_CATCOMPRESS = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+
+  if (!UPDATE_CATFORMAT && !UPDATE_CATCOMPRESS) {
+    fprintf (stderr, "neither -set-compress nor -set-format are specified : this is a NOP\n");
+    exit (2);
+  }
+
+  /* specify portion of the sky */
+  REGION.Rmin = 0;
+  REGION.Rmax = 360;
+  REGION.Dmin = -90;
+  REGION.Dmax = +90;
+  if ((N = get_argument (*argc, argv, "-region"))) {
+    remove_argument (N, argc, argv);
+    REGION.Rmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Rmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Dmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    REGION.Dmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+
+    if (REGION.Rmin == REGION.Rmax) {
+      fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n");
+      exit (2);
+    }
+    if (REGION.Dmin == REGION.Dmax) {
+      fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n");
+      exit (2);
+    }
+  }
+
+  if (*argc != 2) usage();
+
+  return (TRUE);
+}
+
+void usage() {
+
+  fprintf (stderr, "USAGE: dvocompress (catdir) [-set-compress name] [-set-format name]\n\n");
+  fprintf (stderr, " this program takes an existing DVO database and re-writes the tables in the specified compression / formation\n");
+
+  fprintf (stderr, " dvocompress (catdir)\n\n");
+
+  fprintf (stderr, " parallel options:\n");
+  fprintf (stderr, "  -parallel:\n");
+  fprintf (stderr, "  -parallel-manual\n");
+  fprintf (stderr, "  -parallel-serial\n\n");
+  fprintf (stderr, " -region Rmin Rmax Dmin Dmax : limit operation to the specified region \n");
+  
+  fprintf (stderr, " option options:\n");
+  fprintf (stderr, " -v : verbose mode\n");
+  fprintf (stderr, " -set-compress (catcompress) : NONE, GZIP_1, GZIP_2, RICE_1, AUTO\n");
+  fprintf (stderr, " -set-format (catmode)\n");
+
+  exit (2);
+}
Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress.c
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress.c	(revision 38657)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress.c	(revision 38657)
@@ -0,0 +1,27 @@
+# include "dvocompress.h"
+
+int main (int argc, char **argv) {
+
+  SetSignals ();
+  args (&argc, argv);
+
+  char *CATDIR = argv[1];
+
+  // the output catalog needs to inherit the SKY_DEPTH of the input catalog
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, 0, VERBOSE);
+  if (!sky) {
+    fprintf (stderr, "ERROR loading sky table from %s\n", CATDIR);
+    exit (2);
+  }
+
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+  SkyList *skylist = SkyListByPatch (sky, -1, &REGION);
+  if (!skylist) {
+    fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR);
+    exit (2);
+  }
+
+  // hostID is 0 for master program
+  dvocompress_catalogs (CATDIR, skylist, 0);
+  exit (0);
+}
Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_catalogs.c
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_catalogs.c	(revision 38657)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_catalogs.c	(revision 38657)
@@ -0,0 +1,146 @@
+# include "dvocompress.h"
+# define DEBUG 1
+
+int dvocompress_catalogs (char *catdir, SkyList *skylist, int hostID) {
+
+  int i;
+  Catalog catalog;
+
+  if (PARALLEL && !hostID) {
+    dvocompress_parallel (catdir, skylist);
+    return FALSE;
+  }
+
+  for (i = 0; i < skylist[0].Nregions; i++) {
+
+    // does this host ID match the desired location for the table?
+    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
+
+    // set the parameters which guide catalog open/load/create
+    char hostfile[DVO_MAX_PATH];
+    snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+
+    dvo_catalog_init (&catalog, TRUE);
+    catalog.filename  = hostID ? hostfile : skylist[0].filename[i];
+
+    // read all catalog types
+    catalog.catflags = 
+      DVO_LOAD_AVERAGE | 
+      DVO_LOAD_MEASURE | 
+      DVO_LOAD_SECFILT | 
+      DVO_LOAD_MISSING | 
+      DVO_LOAD_LENSING | 
+      DVO_LOAD_LENSOBJ | 
+      DVO_LOAD_STARPAR | 
+      DVO_LOAD_GALPHOT;
+
+    // an error exit status here is a significant error
+    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], (VERBOSE > 1), "w")) {
+      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename);
+      exit (2);
+    }
+
+    // skip empty input catalogs
+    if (!catalog.Naverage_disk) {
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+
+    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
+
+    if (UPDATE_CATFORMAT) {
+      catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
+    } 
+    if (UPDATE_CATCOMPRESS) {
+      catalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS);
+    } 
+
+    if (!dvo_catalog_backup (&catalog, TRUE)) {
+      fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", catalog.filename);
+      exit (1);
+    }
+
+    SetProtect (TRUE);
+    if (!dvo_catalog_save (&catalog, (VERBOSE > 1))) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }
+    if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); }
+    SetProtect (FALSE);
+
+    if (!dvo_catalog_unlink_backup (&catalog, TRUE)) {
+      fprintf (stderr, "WARNING: failed to remove backup for catalog %s\n", catalog.filename);
+    }
+
+    dvo_catalog_free (&catalog);
+  }
+
+  return TRUE;
+}
+
+int dvocompress_parallel (char *catdir, SkyList *skylist) {
+
+  // launch the photdbo_client jobs to the parallel hosts
+
+  // ensure that the paths are absolute path names
+  char *catdir_abs = abspath (catdir, DVO_MAX_PATH);
+
+  // load the list of hosts
+  HostTable *table = HostTableLoad (catdir, skylist->hosts);
+  if (!table) {
+    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", skylist->hosts, catdir);
+    exit (1);
+  }    
+
+  int i;
+  for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    // options / arguments that can affect relastro_client -update-objects:
+    char *command = NULL;
+    strextend (&command, "dvocompress_client %s -hostID %d -hostdir %s -region %f %f %f %f", 
+	       catdir_abs, table->hosts[i].hostID, table->hosts[i].pathname, 
+	       REGION.Rmin, REGION.Rmax, REGION.Dmin, REGION.Dmax
+      );
+
+    if (VERBOSE)            { strextend (&command, "-v"); }
+    if (VERBOSE > 1)        { strextend (&command, "-vv"); }
+    if (UPDATE_CATCOMPRESS) { strextend (&command, "-set-compress %s", UPDATE_CATCOMPRESS); }
+    if (UPDATE_CATFORMAT)   { strextend (&command, "-set-format %s", UPDATE_CATFORMAT); }
+
+    fprintf (stderr, "command: %s\n", command);
+
+    if (PARALLEL_MANUAL) continue;
+
+    if (PARALLEL_SERIAL) {
+      int status = system (command);
+      if (status) {
+	fprintf (stderr, "ERROR running dvocompress_client\n");
+	exit (2);
+      }
+    } else {
+      // launch the job on the remote machine (no handshake)
+      int errorInfo = 0;
+      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
+      if (!pid) {
+	if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+	exit (1);
+      }
+      table->hosts[i].pid = pid; // save for future reference
+    }
+  }
+
+  if (PARALLEL_MANUAL) {
+    fprintf (stderr, "run the dvocompress_client commands above.  when these are done, hit return\n");
+    getchar();
+  }
+  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, (VERBOSE > 1));
+  }
+
+  free (catdir_abs);
+
+  return TRUE;
+}      
Index: /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_client.c
===================================================================
--- /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_client.c	(revision 38657)
+++ /branches/eam_branches/ipp-20150625/Ohana/src/photdbc/src/dvocompress_client.c	(revision 38657)
@@ -0,0 +1,27 @@
+# include "dvocompress.h"
+
+int main (int argc, char **argv) {
+
+  SetSignals ();
+  args_client (&argc, argv);
+
+  char *CATDIR = argv[1];
+
+  // load the current sky table (layout of all SkyRegions) 
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE);
+  if (!sky) {
+    fprintf (stderr, "ERROR loading sky table from %s\n", CATDIR);
+    exit (2);
+  }
+
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+  SkyList *skylist = SkyListByPatch (sky, -1, &REGION);
+  if (!skylist) {
+    fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR);
+    exit (2);
+  }
+  
+  // hostID is 0 for master program
+  dvocompress_catalogs (CATDIR, skylist, HOST_ID);
+  exit (0);
+}