Index: /trunk/Ohana/src/dvosplit/include/dvosplit.h
===================================================================
--- /trunk/Ohana/src/dvosplit/include/dvosplit.h	(revision 40560)
+++ /trunk/Ohana/src/dvosplit/include/dvosplit.h	(revision 40561)
@@ -40,5 +40,5 @@
 
 Catalog   *open_output_catalogs   PROTO((SkyList *outlist, int catformat, int catmode));
-AveLinks  *split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
+int        split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
 int        split_measures         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks));
 void GetConfig (char *config, char *field, char *format, int N, void *ptr);
Index: /trunk/Ohana/src/dvosplit/src/dvosplit.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/dvosplit.c	(revision 40560)
+++ /trunk/Ohana/src/dvosplit/src/dvosplit.c	(revision 40561)
@@ -2,6 +2,5 @@
 
 // dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
-// dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax]
-// (inherites the input catalog's format and mode, unless -set-format or -set-mode are used)
+// (inherits the input catalog's format and mode, unless -set-format or -set-mode are used)
 int main (int argc, char **argv) {
 
@@ -10,5 +9,4 @@
   SkyList *skylist, *outlist;
   Catalog incatalog, *outcatalogs;
-  AveLinks *avelinks;
   char *filename, *CATDIR;
 
@@ -16,12 +14,4 @@
   ConfigInit (&argc, argv);
   args (argc, argv);
-
-  /*
-  fprintf (stderr, 
-  "this program has the ability to split dvo catalogs one file (cpt, cpm, etc) at a time.
-  however, new updates to the cpm format requires the average RA,DEC coords to load the
-  measures.  update the code to handle that first");
-  exit (2);
-  */
 
   CATDIR = strcreate (argv[1]);
@@ -65,4 +55,5 @@
   for (i = 0; i < skylist[0].Nregions; i++) {
     if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
+    fprintf (stderr, "reading from %s\n", skylist[0].regions[i][0].name);
 
     // if !LocalCopy, always copy the file
@@ -81,5 +72,6 @@
     incatalog.filename = skylist[0].filename[i];
     incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
-    incatalog.catflags = DVO_LOAD_NONE;
+    incatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
+    // load all of the tables at once
 
     // an error exit status here is a significant error
@@ -97,5 +89,8 @@
 
     // change sky.regions[i].depth for these regions
-    outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
+    // outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
+    outlist = SkyListChildrenByBounds (sky, skylist[0].regions[i][0].index, OUT_DEPTH,
+				       skylist[0].regions[i][0].Rmin + 0.01, skylist[0].regions[i][0].Rmax - 0.01,
+				       skylist[0].regions[i][0].Dmin + 0.01, skylist[0].regions[i][0].Dmax - 0.01 );
 
     // Modify the outlist filenames to match the output directory.  Note that the
@@ -106,17 +101,12 @@
     outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode);
 
-    avelinks = split_averages (&incatalog, outlist, outcatalogs); 
-
-    split_measures (&incatalog, outlist, outcatalogs, avelinks); 
-
-    // XXX missing entries have to be reconstructed if they are desired
-    // split_missings (&incatalog, outlist, outcatalogs, avelinks); 
-
-    free (avelinks[0].outref);
-    free (avelinks[0].outcat);
+    split_averages (&incatalog, outlist, outcatalogs); 
 
     dvo_catalog_unlock (&incatalog);
 
     for (j = 0; j < outlist[0].Nregions; j++) {
+      outcatalogs[j].sorted = TRUE; // split_averages generates a sorted database
+      fprintf (stderr, "save: %s\n", outcatalogs[j].filename);
+      dvo_catalog_save (&outcatalogs[j], VERBOSE);
       dvo_catalog_unlock (&outcatalogs[j]);
     }
@@ -146,5 +136,5 @@
 
   if (!LocalCopy) {
-    // copy the images table
+    // copy the images table (there are others we should copy as well)
     char line[2048];
     snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR);
Index: /trunk/Ohana/src/dvosplit/src/dvosplit.subset.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/dvosplit.subset.c	(revision 40561)
+++ /trunk/Ohana/src/dvosplit/src/dvosplit.subset.c	(revision 40561)
@@ -0,0 +1,150 @@
+# include "dvosplit.h"
+
+// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
+// (inherits the input catalog's format and mode, unless -set-format or -set-mode are used)
+int main (int argc, char **argv) {
+
+  int i, j, OUT_DEPTH;
+  SkyTable *sky;
+  SkyList *skylist, *outlist;
+  Catalog incatalog, *outcatalogs;
+  AveLinks *avelinks;
+  char *filename, *CATDIR;
+
+  SetSignals ();
+  ConfigInit (&argc, argv);
+  args (argc, argv);
+
+  CATDIR = strcreate (argv[1]);
+  OUT_DEPTH = atoi (argv[2]);
+  if (!OUTDIR) {
+    OUTDIR = strcreate (CATDIR);
+  }
+
+  // load the photcode table (for Nsecfilt and related)
+  char photcodeFile[1024];
+  snprintf (photcodeFile, 1024, "%s/Photcodes.dat", CATDIR);
+  if (!LoadPhotcodes (photcodeFile, NULL, FALSE)) {
+    fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
+    exit (1);
+  }
+
+  // dvosplit can be run locally (no -outdir specified); otherwise, copy Photcode.dat,
+  // Images.dat, SkyTable.fits
+  int LocalCopy = !strcmp (CATDIR, OUTDIR);
+
+  // if not local, save the photcode table (for Nsecfilt and related)
+  if (!LocalCopy) {
+    if (!check_dir_access (OUTDIR, VERBOSE)) {
+      fprintf (stderr, "failed to create output directory %s\n", OUTDIR);
+      exit (1);
+    }
+    snprintf (photcodeFile, 1024, "%s/Photcodes.dat", OUTDIR);
+    if (!SavePhotcodesFITS (photcodeFile)) {
+      fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
+      exit (1);
+    }
+  }
+
+  // load the sky table for the existing database
+  sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE);
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+
+  // get the list of populated regions
+  skylist  = SkyListByPatch (sky, -1, &UserPatch);
+  
+  for (i = 0; i < skylist[0].Nregions; i++) {
+    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
+
+    // if !LocalCopy, always copy the file
+    // if (current level >  out level) skip: cannot currently merge catalogs
+    // if (current level == out level) skip: no action is needed
+
+    if (skylist[0].regions[i][0].depth > OUT_DEPTH) {
+      if (VERBOSE) fprintf (stderr, "WARNING, cannot merge deeper catalog %s (%d vs %d)\n", skylist[0].regions[i][0].name, skylist[0].regions[i][0].depth, OUT_DEPTH);
+      continue;
+    }
+
+    if (LocalCopy && (skylist[0].regions[i][0].depth == OUT_DEPTH)) continue;
+
+    // set the parameters which guide catalog open/load/create
+    dvo_catalog_init (&incatalog, TRUE);
+    incatalog.filename = skylist[0].filename[i];
+    incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
+    incatalog.catflags = DVO_LOAD_NONE;
+
+    // an error exit status here is a significant error
+    if (!dvo_catalog_open (&incatalog, skylist[0].regions[i], VERBOSE, "r")) {
+      fprintf (stderr, "ERROR: failure to open catalog file %s\n", incatalog.filename);
+      exit (2);
+    }
+
+    // skip empty input catalogs
+    if (!incatalog.Naverage_disk) {
+      dvo_catalog_unlock (&incatalog);
+      dvo_catalog_free (&incatalog);
+      continue;
+    }
+
+    // change sky.regions[i].depth for these regions
+    outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
+
+    // Modify the outlist filenames to match the output directory.  Note that the
+    // filenames are now owned by the list (and are not freed by SkyListFree)
+    SkyListSetFilenames (outlist, OUTDIR, "cpt");
+
+    // inherit the input catalog's format and mode (unless overridden)
+    outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode);
+
+    avelinks = split_averages (&incatalog, outlist, outcatalogs); 
+
+    split_measures (&incatalog, outlist, outcatalogs, avelinks); 
+
+    // XXX missing entries have to be reconstructed if they are desired
+    // split_missings (&incatalog, outlist, outcatalogs, avelinks); 
+
+    free (avelinks[0].outref);
+    free (avelinks[0].outcat);
+
+    dvo_catalog_unlock (&incatalog);
+
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      dvo_catalog_unlock (&outcatalogs[j]);
+    }
+
+    // adjust depth
+    skylist[0].regions[i][0].table = FALSE;
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      outlist[0].regions[j][0].table = TRUE;
+    }
+
+    // free the newly allocated filenames
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      free (outlist[0].filename[j]);
+    }
+
+    // free the rest of the list
+    SkyListFree (outlist);
+  }
+
+  // save sky table copy (Local or not Local : the depth has changed)
+  filename = SkyTableFilename (OUTDIR);
+  check_file_access (filename, TRUE, TRUE, VERBOSE);
+  if (!SkyTableSave (sky, filename)) {
+    fprintf (stderr, "ERROR: failed to save sky table for %s\n", OUTDIR);
+    exit (1);
+  }
+
+  if (!LocalCopy) {
+    // copy the images table
+    char line[2048];
+    snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR);
+    int status = system (line);
+    if (status) {
+      fprintf (stderr, "ERROR: failed to copy Images.dat\n");
+      exit (1);
+    }
+  }
+
+  exit (0);
+}
Index: /trunk/Ohana/src/dvosplit/src/open_output_catalogs.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/open_output_catalogs.c	(revision 40560)
+++ /trunk/Ohana/src/dvosplit/src/open_output_catalogs.c	(revision 40561)
@@ -15,5 +15,5 @@
     outcatalogs[i].filename  = outlist[0].filename[i];
     outcatalogs[i].Nsecfilt  = GetPhotcodeNsecfilt ();
-    outcatalogs[i].catflags  = DVO_LOAD_NONE;
+    outcatalogs[i].catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; // for a subset, use: DVO_LOAD_NONE;
     outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat;  // set the default catformat from config data
     outcatalogs[i].catmode   = CATMODE   ? dvo_catalog_catmode (CATMODE)     : catmode;    // set the default catmode from config data
Index: /trunk/Ohana/src/dvosplit/src/split_averages.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/split_averages.c	(revision 40560)
+++ /trunk/Ohana/src/dvosplit/src/split_averages.c	(revision 40561)
@@ -3,138 +3,163 @@
 # define DNOUT 1000
 
-AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
+/* used in find_matches, find_matches_refstars */
+# define IN_REGION(REG,R,D) (			     \
+((D) >= REG[0].Dmin) && ((D) < REG[0].Dmax) && \
+((R) >= REG[0].Rmin) && ((R) < REG[0].Rmax))
 
-  double inR, inD;
-  int n, block, ave, cat, averef, Nblocks, Ncat, Nout, Nsecfilt;
-  int *outref, *outcat, *outmem;
-  AveLinks *avelinks;
+// incatalog has already been loaded; we now need to split out the entries to the subcatalogs
+// outcatalog[] have already been opened (and are starting at empty)
 
-  ALLOCATE (outref, int, incatalog[0].Naverage_disk);
-  ALLOCATE (outcat, int, incatalog[0].Naverage_disk);
-  ALLOCATE (outmem, int, outlist[0].Nregions);
+// this version requires the entire catalog in memory at once
+int split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
 
-  Nsecfilt = GetPhotcodeNsecfilt ();
+  off_t Nm; // used to track number of values for a given object in a table
 
-  // allocate enough space for these output buffers: use Nsecfilt + 1 incase the file
-  // contains primary photcodes, which will increase Nsecfilt by one.
-  for (cat = 0; cat < outlist[0].Nregions; cat++) {
-    outmem[cat] = DNOUT;
-    REALLOCATE (outcatalogs[cat].average, Average, outmem[cat]);
-    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, outmem[cat]*(Nsecfilt + 1));
+  // we need a list of currently-allocated elements for each data type:
+  ALLOCATE_PTR (NAVERAGE, int, outlist[0].Nregions);
+  ALLOCATE_PTR (NMEASURE, int, outlist[0].Nregions);
+  ALLOCATE_PTR (NLENSING, int, outlist[0].Nregions);
+  ALLOCATE_PTR (NSTARPAR, int, outlist[0].Nregions);
+  ALLOCATE_PTR (NGALPHOT, int, outlist[0].Nregions);
+
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+
+  for (int cat = 0; cat < outlist[0].Nregions; cat++) {
+    NAVERAGE[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].average, Average, NAVERAGE[cat]);
+    NMEASURE[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].measure, Measure, NMEASURE[cat]);
+    NLENSING[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].lensing, Lensing, NLENSING[cat]);
+    NSTARPAR[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].starpar, StarPar, NSTARPAR[cat]);
+    NGALPHOT[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].galphot, GalPhot, NGALPHOT[cat]);
+
+    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, NAVERAGE[cat]*Nsecfilt);
   }
 
-  // split out the average & secfilt entries:
-  incatalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
+  // distribute data to the output catalogs
+  for (off_t ave = 0; ave < incatalog[0].Naverage; ave++) {
+    double inR = incatalog[0].average[ave].R;
+    double inD = incatalog[0].average[ave].D;
 
-  if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
-    Nblocks = incatalog[0].Naverage_disk / NROWS;
-    if (incatalog[0].Naverage_disk % NROWS) Nblocks ++;
-  } else {
-    Nblocks = 1;
+    // which of the outcatalogs contains this coordinate?
+    int Ncat = -1;
+    for (int cat = 0; cat < outlist[0].Nregions; cat++) {
+      if (!IN_REGION(outlist[0].regions[cat], inR, inD)) continue;
+      Ncat = cat;
+      break;
+    }
+
+    // NO outcatalogs contains this coordinate?
+    if (Ncat == -1) {
+      fprintf (stderr, "WARNING: missed "OFF_T_FMT" (%f, %f)\n", ave, inR, inD);
+      continue;
+    }
+    
+    // these values are the current (next) entry for the table
+    off_t averageOut = outcatalogs[Ncat].Naverage;
+    off_t measureOut = outcatalogs[Ncat].Nmeasure;
+    off_t lensingOut = outcatalogs[Ncat].Nlensing;
+    off_t starparOut = outcatalogs[Ncat].Nstarpar;
+    off_t galphotOut = outcatalogs[Ncat].Ngalphot;
+
+    // assign the value to the next element of the output catalog
+    outcatalogs[Ncat].average[averageOut] = incatalog[0].average[ave];
+
+    // fprintf (stderr, "catalog %s: aveOut: %d m
+
+    // these values track the start of the values for the table for this object
+    outcatalogs[Ncat].average[averageOut].measureOffset = measureOut;
+    outcatalogs[Ncat].average[averageOut].lensingOffset = lensingOut;
+    outcatalogs[Ncat].average[averageOut].starparOffset = starparOut;
+    outcatalogs[Ncat].average[averageOut].galphotOffset = galphotOut;
+    
+    // update secfilt at the same time
+    for (int n = 0; n < Nsecfilt; n++) {
+      outcatalogs[Ncat].secfilt[averageOut*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n];
+    }
+    outcatalogs[Ncat].Naverage ++;
+    
+    if (outcatalogs[Ncat].Naverage >= NAVERAGE[Ncat]) {
+      NAVERAGE[Ncat] += DNOUT;
+      REALLOCATE (outcatalogs[Ncat].average, Average, NAVERAGE[Ncat]);
+      REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, NAVERAGE[Ncat]*Nsecfilt);
+    }
+
+    // assign the Measure values for this object to the output catalog
+    Nm = 0;
+    for (int j = 0; j < incatalog[0].average[ave].Nmeasure; j++) {
+      off_t offset = incatalog[0].average[ave].measureOffset + j;
+
+      outcatalogs[Ncat].measure[measureOut] = incatalog[0].measure[offset];
+      outcatalogs[Ncat].measure[measureOut].averef = averageOut;
+
+      measureOut ++;
+      Nm ++;
+      if (measureOut >= NMEASURE[Ncat]) {
+	NMEASURE[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].measure, Measure, NMEASURE[Ncat]);
+      }
+    }
+    outcatalogs[Ncat].average[averageOut].Nmeasure = Nm;
+
+    Nm = 0;
+    for (int j = 0; j < incatalog[0].average[ave].Nlensing; j++) {
+      off_t offset = incatalog[0].average[ave].lensingOffset + j;
+
+      outcatalogs[Ncat].lensing[lensingOut] = incatalog[0].lensing[offset];
+      outcatalogs[Ncat].lensing[lensingOut].averef = averageOut;
+
+      lensingOut ++;
+      Nm ++;
+      if (lensingOut >= NLENSING[Ncat]) {
+	NLENSING[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].lensing, Lensing, NLENSING[Ncat]);
+      }
+    }
+    outcatalogs[Ncat].average[averageOut].Nlensing = Nm;
+
+    Nm = 0;
+    for (int j = 0; j < incatalog[0].average[ave].Nstarpar; j++) {
+      off_t offset = incatalog[0].average[ave].starparOffset + j;
+
+      outcatalogs[Ncat].starpar[starparOut] = incatalog[0].starpar[offset];
+      outcatalogs[Ncat].starpar[starparOut].averef = averageOut;
+
+      starparOut ++;
+      Nm ++;
+      if (starparOut == NSTARPAR[Ncat]) {
+	NSTARPAR[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].starpar, StarPar, NSTARPAR[Ncat]);
+      }
+    }
+    outcatalogs[Ncat].average[averageOut].Nstarpar = Nm;
+
+    Nm = 0;
+    for (int j = 0; j < incatalog[0].average[ave].Ngalphot; j++) {
+      off_t offset = incatalog[0].average[ave].galphotOffset + j;
+
+      outcatalogs[Ncat].galphot[galphotOut] = incatalog[0].galphot[offset];
+      outcatalogs[Ncat].galphot[galphotOut].averef = averageOut;
+
+      galphotOut ++;
+      Nm ++;
+      if (galphotOut == NGALPHOT[Ncat]) {
+	NGALPHOT[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].galphot, GalPhot, NGALPHOT[Ncat]);
+      }
+    }
+    outcatalogs[Ncat].average[averageOut].Ngalphot = Nm;
+
+    outcatalogs[Ncat].Nmeasure = measureOut;
+    outcatalogs[Ncat].Nlensing = lensingOut;
+    outcatalogs[Ncat].Nstarpar = starparOut;
+    outcatalogs[Ncat].Ngalphot = galphotOut;
+
   }
 
-  for (block = 0; block < Nblocks; block++) {
+  free (NAVERAGE);
+  free (NMEASURE);
+  free (NLENSING);
+  free (NSTARPAR);
+  free (NGALPHOT);
 
-    if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
-      // read up to NROWS at a time
-      dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
-      fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
-      assert (block*NROWS == incatalog[0].Naverage_off);
-    } else {
-      dvo_catalog_load (incatalog, VERBOSE);
-      fprintf (stderr, "splitting %s (averages)\n", incatalog[0].filename);
-    }
-
-    // distribute data to the output catalogs
-    for (ave = 0; ave < incatalog[0].Naverage; ave++) {
-      averef = ave + incatalog[0].Naverage_off;
-	
-      inR = incatalog[0].average[ave].R;
-      inD = incatalog[0].average[ave].D;
-
-      if (incatalog[0].average[ave].Nmeasure == 0) {
-	fprintf (stderr, "WARNING: object with no measurements, skipping %d (%f, %f)\n", averef, inR, inD);
-	continue;
-      }
-      // which of the outcatalogs contains this coordinate?
-
-      Ncat = -1;
-      for (cat = 0; cat < outlist[0].Nregions; cat++) {
-	if (inR < outlist[0].regions[cat][0].Rmin) continue;
-	if (inR > outlist[0].regions[cat][0].Rmax) continue;
-	if (inD < outlist[0].regions[cat][0].Dmin) continue;
-	if (inD > outlist[0].regions[cat][0].Dmax) continue;
-	Ncat = cat;
-	break;
-      }
-
-      if (Ncat == -1) {
-	fprintf (stderr, "WARNING: missed %d (%f, %f)\n", averef, inR, inD);
-	continue;
-      }
-
-      Nout = outcatalogs[Ncat].Naverage;
-      outref[averef] = Nout + outcatalogs[Ncat].Naverage_off;
-      outcat[averef] = Ncat;
-
-      // assign the value to the next element of the output catalog
-      outcatalogs[Ncat].average[Nout] = incatalog[0].average[ave];
-      outcatalogs[Ncat].Naverage ++;
-
-      // update secfilt at the same time
-      for (n = 0; n < Nsecfilt; n++) {
-	outcatalogs[Ncat].secfilt[Nout*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n];
-	outcatalogs[Ncat].Nsecfilt_mem++;
-      }
-
-      if (outcatalogs[Ncat].Naverage >= outmem[Ncat]) {
-	outmem[Ncat] += DNOUT;
-	REALLOCATE (outcatalogs[Ncat].average, Average, outmem[Ncat]);
-	REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, outmem[Ncat]*(Nsecfilt + 1));
-      }
-    }
-    dvo_catalog_free_data (incatalog);
-
-    // double check the values of Naverage, Nsecfilt_mem?
-
-    // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below
-
-    // write out the new values
-    if (!FULL_TABLE) {
-      for (cat = 0; cat < outlist[0].Nregions; cat++) {
-	outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
-
-	SetProtect (TRUE);
-	dvo_catalog_save (&outcatalogs[cat], VERBOSE);
-	SetProtect (FALSE);
-	// fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecfilt_mem, outcatalogs[cat].Nsecfilt_disk, outcatalogs[cat].Nsecfilt_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
-
-	// advance the pointers and free the current data
-	// XXX these should be done within save segment:
-	outcatalogs[cat].Naverage_disk += outcatalogs[cat].Naverage;
-	outcatalogs[cat].Naverage_off  += outcatalogs[cat].Naverage;
-	outcatalogs[cat].Nsecfilt_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
-	outcatalogs[cat].Nsecfilt_off  += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
-	outcatalogs[cat].Nsecfilt    = Nsecfilt;
-
-	outcatalogs[cat].Naverage    = 0;
-	outcatalogs[cat].Nsecfilt_mem   = 0;
-      }
-    }
-  }
-
-  if (!FULL_TABLE) {
-    for (cat = 0; cat < outlist[0].Nregions; cat++) {
-      dvo_catalog_free_data (&outcatalogs[cat]);
-    }
-  }
-
-  free (outmem);
-
-  ALLOCATE (avelinks, AveLinks, 1);
-  avelinks[0].outref = outref;
-  avelinks[0].outcat = outcat;
-
-
-  return (avelinks);
+  return TRUE;
 }
Index: /trunk/Ohana/src/dvosplit/src/split_averages.subset.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/split_averages.subset.c	(revision 40561)
+++ /trunk/Ohana/src/dvosplit/src/split_averages.subset.c	(revision 40561)
@@ -0,0 +1,144 @@
+# include "dvosplit.h"
+# define NROWS 1000000 /* ~10MB per block for measures */
+# define DNOUT 1000
+
+/* used in find_matches, find_matches_refstars */
+# define IN_REGION(REG,R,D) (			     \
+((D) >= REG[0].Dmin) && ((D) < REG[0].Dmax) && \
+((R) >= REG[0].Rmin) && ((R) < REG[0].Rmax))
+
+
+AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
+
+  double inR, inD;
+  int n, block, ave, cat, averef, Nblocks, Ncat, Nout, Nsecfilt;
+  int *outref, *outcat, *outmem;
+  AveLinks *avelinks;
+
+  ALLOCATE (outref, int, incatalog[0].Naverage_disk);
+  ALLOCATE (outcat, int, incatalog[0].Naverage_disk);
+  ALLOCATE (outmem, int, outlist[0].Nregions);
+
+  Nsecfilt = GetPhotcodeNsecfilt ();
+
+  // allocate enough space for these output buffers: use Nsecfilt + 1 incase the file
+  // contains primary photcodes, which will increase Nsecfilt by one.
+  for (cat = 0; cat < outlist[0].Nregions; cat++) {
+    outmem[cat] = DNOUT;
+    REALLOCATE (outcatalogs[cat].average, Average, outmem[cat]);
+    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, outmem[cat]*(Nsecfilt + 1));
+  }
+
+  // split out the average & secfilt entries:
+  incatalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
+
+  if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
+    Nblocks = incatalog[0].Naverage_disk / NROWS;
+    if (incatalog[0].Naverage_disk % NROWS) Nblocks ++;
+  } else {
+    Nblocks = 1;
+  }
+
+  for (block = 0; block < Nblocks; block++) {
+
+    if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
+      // read up to NROWS at a time
+      dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
+      fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
+      assert (block*NROWS == incatalog[0].Naverage_off);
+    } else {
+      dvo_catalog_load (incatalog, VERBOSE);
+      fprintf (stderr, "splitting %s (averages)\n", incatalog[0].filename);
+    }
+
+    // distribute data to the output catalogs
+    for (ave = 0; ave < incatalog[0].Naverage; ave++) {
+      averef = ave + incatalog[0].Naverage_off;
+	
+      inR = incatalog[0].average[ave].R;
+      inD = incatalog[0].average[ave].D;
+
+      // XXX do not skip : galphot dvo has Nmeasure == 0
+      if (incatalog[0].average[ave].Nmeasure == 0) {
+	fprintf (stderr, "WARNING: object with no measurements, skipping %d (%f, %f)\n", averef, inR, inD);
+      }
+
+      // which of the outcatalogs contains this coordinate?
+      Ncat = -1;
+      for (cat = 0; cat < outlist[0].Nregions; cat++) {
+	if (!IN_REGION(outlist[0].regions[cat], inR, inD)) continue;
+	Ncat = cat;
+	break;
+      }
+
+      // NO outcatalogs contains this coordinate?
+      if (Ncat == -1) {
+	fprintf (stderr, "WARNING: missed %d (%f, %f)\n", averef, inR, inD);
+	continue;
+      }
+
+      Nout = outcatalogs[Ncat].Naverage;
+      outref[averef] = Nout + outcatalogs[Ncat].Naverage_off;
+      outcat[averef] = Ncat;
+
+      // assign the value to the next element of the output catalog
+      outcatalogs[Ncat].average[Nout] = incatalog[0].average[ave];
+      outcatalogs[Ncat].Naverage ++;
+
+      // update secfilt at the same time
+      for (n = 0; n < Nsecfilt; n++) {
+	outcatalogs[Ncat].secfilt[Nout*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n];
+	outcatalogs[Ncat].Nsecfilt_mem++;
+      }
+
+      if (outcatalogs[Ncat].Naverage >= outmem[Ncat]) {
+	outmem[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].average, Average, outmem[Ncat]);
+	REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, outmem[Ncat]*(Nsecfilt + 1));
+      }
+    }
+    dvo_catalog_free_data (incatalog);
+
+    // double check the values of Naverage, Nsecfilt_mem?
+
+    // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below
+
+    // write out the new values
+    if (!FULL_TABLE) {
+      for (cat = 0; cat < outlist[0].Nregions; cat++) {
+	outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
+
+	SetProtect (TRUE);
+	dvo_catalog_save (&outcatalogs[cat], VERBOSE);
+	SetProtect (FALSE);
+	// fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecfilt_mem, outcatalogs[cat].Nsecfilt_disk, outcatalogs[cat].Nsecfilt_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
+
+	// advance the pointers and free the current data
+	// XXX these should be done within save segment:
+	outcatalogs[cat].Naverage_disk += outcatalogs[cat].Naverage;
+	outcatalogs[cat].Naverage_off  += outcatalogs[cat].Naverage;
+	outcatalogs[cat].Nsecfilt_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
+	outcatalogs[cat].Nsecfilt_off  += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
+	outcatalogs[cat].Nsecfilt    = Nsecfilt;
+
+	outcatalogs[cat].Naverage    = 0;
+	outcatalogs[cat].Nsecfilt_mem   = 0;
+      }
+    }
+  }
+
+  if (!FULL_TABLE) {
+    for (cat = 0; cat < outlist[0].Nregions; cat++) {
+      dvo_catalog_free_data (&outcatalogs[cat]);
+    }
+  }
+
+  free (outmem);
+
+  ALLOCATE (avelinks, AveLinks, 1);
+  avelinks[0].outref = outref;
+  avelinks[0].outcat = outcat;
+
+
+  return (avelinks);
+}
Index: /trunk/Ohana/src/dvosplit/src/split_measures.subset.c
===================================================================
--- /trunk/Ohana/src/dvosplit/src/split_measures.subset.c	(revision 40561)
+++ /trunk/Ohana/src/dvosplit/src/split_measures.subset.c	(revision 40561)
@@ -0,0 +1,101 @@
+# include "dvosplit.h"
+# define NROWS 1000000 /* ~10MB per row for measures */
+# define DNOUT 1000
+
+int split_measures (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks) {
+
+  int block, meas, cat, Nblocks, Ncat, Nout, averef;
+  int *outref, *outcat, *outmem;
+
+  outref = avelinks->outref;
+  outcat = avelinks->outcat;
+  ALLOCATE (outmem, int, outlist[0].Nregions);
+
+  // allocate enough space for the output buffer
+  for (cat = 0; cat < outlist[0].Nregions; cat++) {
+    outmem[cat] = DNOUT;
+    REALLOCATE (outcatalogs[cat].measure, Measure, outmem[cat]);
+  }
+
+  // split out the measure entries:
+  incatalog[0].catflags = DVO_LOAD_MEASURE | DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
+
+  // if ((incatalog[0].catformat == DVO_FORMAT_ELIXIR) || (incatalog[0].catformat == DVO_FORMAT_LONEOS)) {
+  //   // for these two formats, we need the average and secfilt values around until we do the measures...
+  //   // XXX I am loading these 2x -- perhaps I can make the API smarter about reloading?
+  //   incatalog[0].catflags |= DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; 
+  // }
+
+  if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
+    Nblocks = incatalog[0].Nmeasure_disk / NROWS;
+    if (incatalog[0].Nmeasure_disk % NROWS) Nblocks ++;
+  } else {
+    Nblocks = 1;
+  }
+
+  for (block = 0; block < Nblocks; block++) {
+
+    // read up to NROWS at a time
+    if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
+      dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
+      fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
+      assert (block*NROWS == incatalog[0].Nmeasure_off);
+    } else {
+      dvo_catalog_load (incatalog, VERBOSE);
+      fprintf (stderr, "splitting %s (measures)\n", incatalog[0].filename);
+    }
+
+    for (meas = 0; meas < incatalog[0].Nmeasure; meas++) {
+
+      averef = incatalog[0].measure[meas].averef;
+      Ncat = outcat[averef];
+
+      int averef_out = outref[averef];
+      if (averef_out >= outcatalogs[Ncat].Naverage) {
+	fprintf (stderr, "mismatch 1\n");
+	abort();
+      }
+
+      Nout = outcatalogs[Ncat].Nmeasure;
+      outcatalogs[Ncat].measure[Nout] = incatalog[0].measure[meas];
+      outcatalogs[Ncat].measure[Nout].averef = outref[averef];
+
+      outcatalogs[Ncat].Nmeasure++;
+
+      if (outcatalogs[Ncat].Nmeasure >= outmem[Ncat]) {
+	outmem[Ncat] += DNOUT;
+	REALLOCATE (outcatalogs[Ncat].measure, Measure, outmem[Ncat]);
+      }
+    }
+    dvo_catalog_free_data (incatalog);
+
+    if (!FULL_TABLE) {
+      for (cat = 0; cat < outlist[0].Nregions; cat++) {
+	outcatalogs[cat].catflags = DVO_LOAD_MEASURE;
+
+	SetProtect (TRUE);
+	dvo_catalog_save (&outcatalogs[cat], VERBOSE);
+	SetProtect (FALSE);
+
+	outcatalogs[cat].Nmeasure_disk += outcatalogs[cat].Nmeasure;
+	outcatalogs[cat].Nmeasure_off  += outcatalogs[cat].Nmeasure;
+	outcatalogs[cat].Nmeasure    = 0;
+      }
+    }
+  }
+
+  if (FULL_TABLE) {
+    for (cat = 0; cat < outlist[0].Nregions; cat++) {
+      outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE;
+      dvo_catalog_save (&outcatalogs[cat], VERBOSE);
+    }
+  }
+
+  for (cat = 0; cat < outlist[0].Nregions; cat++) {
+    dvo_catalog_free_data (&outcatalogs[cat]);
+  }
+
+  free (outmem);
+
+  return (TRUE);
+}