Changeset 38351

Timestamp:

Jun 2, 2015, 12:35:49 PM (11 years ago)

Author:

eugene

Message:

macro-ize repeated code blocks in table i/o; add compress/uncompress options to all dvo tables; rename some internal structures to enable macro-ization

Location:

branches/eam_branches/ohana.20150429/src/libdvo

Files:

• include/dvo.h (modified) (3 diffs)
• src/dvo_catalog.c (modified) (9 diffs)
• src/dvo_catalog_mef.c (modified) (21 diffs)
• src/dvo_catalog_raw.c (modified) (7 diffs)
• src/dvo_catalog_split.c (modified) (78 diffs)
• src/dvo_convert.c (modified) (6 diffs)
• src/dvo_util.c (modified) (1 diff)

branches/eam_branches/ohana.20150429/src/libdvo/include/dvo.h

@@ -39 +39 @@
 /* catalog values to be loaded */
 typedef enum {
-  LOAD_NONE     = 0x0000,
-  LOAD_AVES     = 0x0001,
-  LOAD_MEAS     = 0x0002,
-  LOAD_MISS     = 0x0004,
-  LOAD_SECF     = 0x0008, 
-  SKIP_AVES     = 0x0010,
-  SKIP_MEAS     = 0x0020,
-  SKIP_MISS     = 0x0040,
-  SKIP_SECF     = 0x0080,
-  LOAD_LENSING  = 0x0100,
-  LOAD_LENSOBJ  = 0x0200,
-  SKIP_LENSING  = 0x0400,
-  SKIP_LENSOBJ  = 0x0800,
-  LOAD_STARPAR  = 0x1000,
-  SKIP_STARPAR  = 0x2000,
-  LOAD_GALSHAPE = 0x4000,
-  SKIP_GALSHAPE = 0x8000,
+  DVO_LOAD_NONE     = 0x0000,
+  DVO_LOAD_AVERAGE  = 0x0001,
+  DVO_LOAD_MEASURE  = 0x0002,
+  DVO_LOAD_MISSING  = 0x0004,
+  DVO_LOAD_SECFILT  = 0x0008, 
+  DVO_SKIP_AVERAGE  = 0x0010,
+  DVO_SKIP_MEASURE  = 0x0020,
+  DVO_SKIP_MISSING  = 0x0040,
+  DVO_SKIP_SECFILT  = 0x0080,
+  DVO_LOAD_LENSING  = 0x0100,
+  DVO_LOAD_LENSOBJ  = 0x0200,
+  DVO_SKIP_LENSING  = 0x0400,
+  DVO_SKIP_LENSOBJ  = 0x0800,
+  DVO_LOAD_STARPAR  = 0x1000,
+  DVO_SKIP_STARPAR  = 0x2000,
+  DVO_LOAD_GALSHAPE = 0x4000,
+  DVO_SKIP_GALSHAPE = 0x8000,
 } DVOCatFlags;
 
@@ -758 +758 @@
 
   int Nsecfilt;  /* number of secfilt entries for each average entry */
-  off_t Naverage,   Nmeasure,   Nmissing,   Nlensing,      Nlensobj,      Nstarpar,      Ngalshape,      Nsecf_mem;  /* current number of each component in memory */
-  off_t Naves_disk, Nmeas_disk, Nmiss_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalshape_disk, Nsecf_disk; /* current number of each component on disk */
-  off_t Naves_off,  Nmeas_off,  Nmiss_off,  Nlensing_off,  Nlensobj_off,  Nstarpar_off,  Ngalshape_off,  Nsecf_off;  /* index of first loaded data value */
+  off_t Naverage,      Nmeasure,      Nmissing,      Nlensing,      Nlensobj,      Nstarpar,      Ngalshape,      Nsecfilt_mem;  /* current number of each component in memory */
+  off_t Naverage_disk, Nmeasure_disk, Nmissing_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalshape_disk, Nsecfilt_disk; /* current number of each component on disk */
+  off_t Naverage_off,  Nmeasure_off,  Nmissing_off,  Nlensing_off,  Nlensobj_off,  Nstarpar_off,  Ngalshape_off,  Nsecfilt_off;  /* index of first loaded data value */
 
   // note that we use these for the full-sky relphot analysis
@@ -974 +974 @@
 DVOCatFormat FtableGetFormat (FTable *ftable);
 
-Measure *FtableToMeasure   (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeOrder);
 Average *FtableToAverage   (FTable *ftable, off_t *Naverage,  DVOCatFormat *format, SecFilt **primary, char nativeOrder);
-SecFilt *FtableToSecFilt   (FTable *ftable, off_t *Nsecfilt,  DVOCatFormat *format, char nativeOrder);
-Lensing *FtableToLensing   (FTable *ftable, off_t *Nlensing,  DVOCatFormat *format, char nativeOrder);
-Lensobj *FtableToLensobj   (FTable *ftable, off_t *Nlensobj,  DVOCatFormat *format, char nativeOrder);
-StarPar *FtableToStarPar   (FTable *ftable, off_t *Nstarpar,  DVOCatFormat *format, char nativeOrder);
-GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format, char nativeOrder);
+
+Measure *FtableToMeasure   (FTable *ftable, Average *average, off_t *Nmeasure,  DVOCatFormat *format, char nativeOrder);
+Missing *FtableToMissing   (FTable *ftable, Average *average, off_t *Nmissing,  DVOCatFormat *format, char nativeBytes);
+SecFilt *FtableToSecFilt   (FTable *ftable, Average *average, off_t *Nsecfilt,  DVOCatFormat *format, char nativeOrder);
+Lensing *FtableToLensing   (FTable *ftable, Average *average, off_t *Nlensing,  DVOCatFormat *format, char nativeOrder);
+Lensobj *FtableToLensobj   (FTable *ftable, Average *average, off_t *Nlensobj,  DVOCatFormat *format, char nativeOrder);
+StarPar *FtableToStarPar   (FTable *ftable, Average *average, off_t *Nstarpar,  DVOCatFormat *format, char nativeOrder);
+GalShape *FtableToGalShape (FTable *ftable, Average *average, off_t *Ngalshape, DVOCatFormat *format, char nativeOrder);
 
 int      FtableToImage   (FTable *ftable, Header *theader, DVOCatFormat *format);

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog.c

@@ -527 +527 @@
     catalog[0].catformat   = DVO_FORMAT_UNDEF;
     catalog[0].catcompress = DVO_COMPRESS_NONE;
-    catalog[0].catflags    = LOAD_NONE;
+    catalog[0].catflags    = DVO_LOAD_NONE;
     catalog[0].Nsecfilt    = 0;
   }
@@ -554 +554 @@
   catalog[0].Nmeasure = 0;
   catalog[0].Nmissing = 0;
-  catalog[0].Nsecf_mem = 0;
+  catalog[0].Nsecfilt_mem = 0;
 
   catalog[0].Nlensing = 0;
@@ -561 +561 @@
   catalog[0].Nstarpar = 0;
 
-  catalog[0].Naves_disk = 0;
-  catalog[0].Nmeas_disk = 0;
-  catalog[0].Nmiss_disk = 0;
-  catalog[0].Nsecf_disk = 0;
+  catalog[0].Naverage_disk = 0;
+  catalog[0].Nmeasure_disk = 0;
+  catalog[0].Nmissing_disk = 0;
+  catalog[0].Nsecfilt_disk = 0;
 
   catalog[0].Nlensing_disk = 0;
@@ -571 +571 @@
   catalog[0].Nstarpar_disk = 0;
 
-  catalog[0].Naves_off  = 0;
-  catalog[0].Nmeas_off  = 0;
-  catalog[0].Nmiss_off  = 0;
-  catalog[0].Nsecf_off  = 0;
+  catalog[0].Naverage_off  = 0;
+  catalog[0].Nmeasure_off  = 0;
+  catalog[0].Nmissing_off  = 0;
+  catalog[0].Nsecfilt_off  = 0;
 
   catalog[0].Nlensing_off  = 0;
@@ -900 +900 @@
     catalog[0].secfilt = outsec;
     catalog[0].Nsecfilt = Nsecfilt;
-    catalog[0].Nsecf_mem = Nsecfilt * catalog[0].Naverage;
+    catalog[0].Nsecfilt_mem = Nsecfilt * catalog[0].Naverage;
   }
   return (TRUE);
@@ -965 +965 @@
   if (catalog[0].secfilt) {
     free (catalog[0].secfilt); 
-    catalog[0].Nsecf_mem = 0;
+    catalog[0].Nsecfilt_mem = 0;
     catalog[0].secfilt = NULL;
   }
@@ -989 +989 @@
 
 /*
-  mode   : items to read (LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF)
+  mode   : items to read (DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT)
   format : what table structure on disk (INTERNAL, LONEOS, etc, )
   style  : raw, mef, split, mysql
@@ -1022 +1022 @@
 
   // skip empty cpt files
-  if (primary && !catalog->Naves_disk) {
+  if (primary && !catalog->Naverage_disk) {
     return TRUE;
   }
@@ -1111 +1111 @@
 int dvo_catalog_unlink_backup (Catalog *catalog, int primary) {
 
-  if (primary && !catalog->Naves_disk) {
+  if (primary && !catalog->Naverage_disk) {
     // skip empty files (empty when read, but output may not be empty)
     return TRUE;

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_mef.c

@@ -26 +26 @@
   }
 
-  // NSTARS, average, Naves_disk
+  // NSTARS, average, Naverage_disk
 
 # define GET_TABLE_SIZES(HFIELD, CFIELD, DFIELD, REQUIRED)              \
@@ -41 +41 @@
   
   /* get the components and sizes from the header */
-  GET_TABLE_SIZES ("NSTARS",    average,  Naves_disk,     TRUE);
-  GET_TABLE_SIZES ("NMEAS",     measure,  Nmeas_disk,     TRUE);
-  GET_TABLE_SIZES ("NMISS",     missing,  Nmiss_disk,     FALSE);
+  GET_TABLE_SIZES ("NSTARS",    average,  Naverage_disk,     TRUE);
+  GET_TABLE_SIZES ("NMEAS",     measure,  Nmeasure_disk,     TRUE);
+  GET_TABLE_SIZES ("NMISS",     missing,  Nmissing_disk,     FALSE);
   GET_TABLE_SIZES ("NLENSING",  lensing,  Nlensing_disk,  FALSE);
   GET_TABLE_SIZES ("NLENSOBJ",  lensobj,  Nlensobj_disk,  FALSE);
@@ -54 +54 @@
   if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0;
   catalog[0].Nsecfilt = Nsecfilt;
-  catalog[0].Nsecf_disk = Naverage * Nsecfilt;
+  catalog[0].Nsecfilt_disk = Naverage * Nsecfilt;
   catalog[0].secfilt = NULL;
 
@@ -71 +71 @@
 
   /* read Average table data (or skip) */
-  if (catalog[0].catflags & LOAD_AVES) {
+  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table average data");
@@ -78 +78 @@
     /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
     catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE);
-    if (Naverage != catalog[0].Naves_disk) {
-      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naves_disk);
-    }
-    catalog[0].Naverage = catalog[0].Naves_disk;
-    catalog[0].Naves_off = 0;
+    if (Naverage != catalog[0].Naverage_disk) {
+      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naverage_disk);
+    }
+    catalog[0].Naverage = catalog[0].Naverage_disk;
+    catalog[0].Naverage_off = 0;
   } else {
     catalog[0].catformat = FtableGetFormat (&ftable);
@@ -89 +89 @@
     ALLOCATE (catalog[0].average, Average, 1);
     catalog[0].Naverage = 0;
-    catalog[0].Naves_off = catalog[0].Naves_disk;
+    catalog[0].Naverage_off = catalog[0].Naverage_disk;
   }
   gfits_free_header (&header);
@@ -100 +100 @@
   }
   /* read Measure table data */
-  if (catalog[0].catflags & LOAD_MEAS) {
+  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table measure data");
@@ -106 +106 @@
     }
     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat, FALSE);
-    if (Nmeasure != catalog[0].Nmeas_disk) {
-      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
-    }
-    catalog[0].Nmeasure = catalog[0].Nmeas_disk;
-    catalog[0].Nmeas_off = 0;
+    if (Nmeasure != catalog[0].Nmeasure_disk) {
+      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeasure_disk);
+    }
+    catalog[0].Nmeasure = catalog[0].Nmeasure_disk;
+    catalog[0].Nmeasure_off = 0;
   } else {
     Nbytes = gfits_data_size (&header);
@@ -116 +116 @@
     ALLOCATE (catalog[0].measure, Measure, 1);
     catalog[0].Nmeasure = 0;
-    catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
+    catalog[0].Nmeasure_off = catalog[0].Nmeasure_disk;
   }
 
@@ -125 +125 @@
   }
   /* read Missing table data */
-  if (catalog[0].catflags & LOAD_MISS) {
+  if (catalog[0].catflags & DVO_LOAD_MISSING) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table missing data");
@@ -136 +136 @@
       exit (2);
     }
-    if (Nmissing != catalog[0].Nmiss_disk) {
-      fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmiss_disk);
-    }
-    catalog[0].Nmissing = catalog[0].Nmiss_disk;
-    catalog[0].Nmiss_off = 0;
+    if (Nmissing != catalog[0].Nmissing_disk) {
+      fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmissing_disk);
+    }
+    catalog[0].Nmissing = catalog[0].Nmissing_disk;
+    catalog[0].Nmissing_off = 0;
   } else {
     Nbytes = gfits_data_size (&header);
@@ -146 +146 @@
     ALLOCATE (catalog[0].missing, Missing, 1);
     catalog[0].Nmissing = 0;
-    catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
+    catalog[0].Nmissing_off = catalog[0].Nmissing_disk;
   }
 
@@ -155 +155 @@
   }
   /* read secfilt table data */
-  if (catalog[0].catflags & LOAD_SECF) {
+  if (catalog[0].catflags & DVO_LOAD_SECFILT) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table secfilt data");
@@ -163 +163 @@
     /* how many entries do we expect from the secfilt table? */
     Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage;
-    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, FALSE);
+    catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE);
     if (Nexpect != Nitems) {
       fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nexpect,  Nitems);
@@ -176 +176 @@
       Ntmpfilt = catalog[0].Nsecfilt;
       Nsecfilt = catalog[0].Nsecfilt + 1;
-      Ntotal = Nsecfilt * catalog[0].Naves_disk;
+      Ntotal = Nsecfilt * catalog[0].Naverage_disk;
       ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
-      for (i = 0; i < catalog[0].Naves_disk; i++) {
+      for (i = 0; i < catalog[0].Naverage_disk; i++) {
         catalog[0].secfilt[i*Nsecfilt + 0] = primary[i];
         for (j = 0; j < Ntmpfilt; j++) {
@@ -185 +185 @@
       }         
       catalog[0].Nsecfilt = Nsecfilt;
-      catalog[0].Nsecf_disk = Ntotal;
+      catalog[0].Nsecfilt_disk = Ntotal;
       free (tmpfilt);
       free (primary);
     } 
-    catalog[0].Nsecf_mem = catalog[0].Nsecf_disk;
-    catalog[0].Nsecf_off = 0;
+    catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt_disk;
+    catalog[0].Nsecfilt_off = 0;
   } else {
     /* no real need to skip the data array here... */
@@ -198 +198 @@
       free (primary);
       catalog[0].Nsecfilt ++;
-      catalog[0].Nsecf_disk =  catalog[0].Nsecfilt * catalog[0].Naves_disk;
+      catalog[0].Nsecfilt_disk =  catalog[0].Nsecfilt * catalog[0].Naverage_disk;
     }
     ALLOCATE (catalog[0].secfilt, SecFilt, 1);
-    catalog[0].Nsecf_mem = 0;
-    catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
+    catalog[0].Nsecfilt_mem = 0;
+    catalog[0].Nsecfilt_off = catalog[0].Nsecfilt_disk;
   }
 
@@ -211 +211 @@
   }
   /* read Lensing table data */
-  if (catalog[0].catflags & LOAD_LENSING) {
+  if (catalog[0].catflags & DVO_LOAD_LENSING) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table lensing data");
       return (FALSE);
     }
-    catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat, FALSE);
+    catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &catalog[0].Nlensing, &catalog[0].catformat, FALSE);
     if (Nlensing != catalog[0].Nlensing_disk) {
       fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
@@ -236 +236 @@
   }
   /* read Lensobj table data */
-  if (catalog[0].catflags & LOAD_LENSOBJ) {
+  if (catalog[0].catflags & DVO_LOAD_LENSOBJ) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table lensobj data");
       return (FALSE);
     }
-    catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE);
+    catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE);
     if (Nlensobj != catalog[0].Nlensobj_disk) {
       fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
@@ -261 +261 @@
   }
   /* read StarPar table data */
-  if (catalog[0].catflags & LOAD_STARPAR) {
+  if (catalog[0].catflags & DVO_LOAD_STARPAR) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table starpar data");
       return (FALSE);
     }
-    catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE);
+    catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE);
     if (Nstarpar != catalog[0].Nstarpar_disk) {
       fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
@@ -286 +286 @@
   }
   /* read GalShape table data */
-  if (catalog[0].catflags & LOAD_GALSHAPE) {
+  if (catalog[0].catflags & DVO_LOAD_GALSHAPE) {
     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table galshape data");
       return (FALSE);
     }
-    catalog[0].galshape = FtableToGalShape (&ftable, &catalog[0].Ngalshape, &catalog[0].catformat, FALSE);
+    catalog[0].galshape = FtableToGalShape (&ftable, catalog[0].average, &catalog[0].Ngalshape, &catalog[0].catformat, FALSE);
     if (Ngalshape != catalog[0].Ngalshape_disk) {
       fprintf (stderr, "Warning: mismatch between Ngalshape in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalshape,  catalog[0].Ngalshape_disk);

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_raw.c

@@ -136 +136 @@
   /* read and convert the averages (use a macro to clean this up?) */
   /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
-  if (catalog[0].catflags & LOAD_AVES) {
+  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
     catalog[0].average = ReadRawAverage (catalog[0].f, catalog[0].Naverage, catalog[0].catformat, &primary);
   } else {
@@ -145 +145 @@
   
   /* read and convert the measures (use a macro to clean this up?) */
-  if (catalog[0].catflags & LOAD_MEAS) {
+  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
     catalog[0].measure = ReadRawMeasure (catalog[0].f, catalog[0].average, catalog[0].Nmeasure, catalog[0].catformat);
   } else {
@@ -154 +154 @@
 
   /* read and convert missing */
-  if (catalog[0].catflags & LOAD_MISS) {
+  if (catalog[0].catflags & DVO_LOAD_MISSING) {
     ALLOCATE (catalog[0].missing, Missing, MAX (catalog[0].Nmissing, 1));
     Nitems = catalog[0].Nmissing;
@@ -170 +170 @@
   
   /* read and convert secfilt */
-  if (catalog[0].catflags & LOAD_SECF) {
+  if (catalog[0].catflags & DVO_LOAD_SECFILT) {
     Nitems = catalog[0].Naverage * catalog[0].Nsecfilt;
     catalog[0].secfilt = ReadRawSecFilt (catalog[0].f, Nitems, catalog[0].catformat);
@@ -191 +191 @@
       }         
       catalog[0].Nsecfilt = Nsecfilt;
-      catalog[0].Nsecf_mem = Ntotal;
+      catalog[0].Nsecfilt_mem = Ntotal;
       free (primary);
     } 
@@ -210 +210 @@
 
   /* check data integrity */
-  if (catalog[0].catflags & LOAD_AVES) {
+  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
     for (i = Nmeas = Nmiss = 0; i < catalog[0].Naverage; i++) {
       Nmeas += catalog[0].average[i].Nmeasure; 
@@ -226 +226 @@
 
   /* save the current number so we can do partial updates */
-  catalog[0].Naves_disk = catalog[0].Naverage;
-  catalog[0].Nmeas_disk = catalog[0].Nmeasure;
-  catalog[0].Nmiss_disk = catalog[0].Nmissing;
+  catalog[0].Naverage_disk = catalog[0].Naverage;
+  catalog[0].Nmeasure_disk = catalog[0].Nmeasure;
+  catalog[0].Nmissing_disk = catalog[0].Nmissing;
 
   return (TRUE);

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c

@@ -27 +27 @@
 
   // XXX this translation only works if we have loaded / created a matched average/secfilt set
-  assert (catalog[0].Nsecf_mem == catalog[0].Nsecfilt*catalog[0].Naverage);
+  assert (catalog[0].Nsecfilt_mem == catalog[0].Nsecfilt*catalog[0].Naverage);
 
   Nallfilt = catalog[0].Nsecfilt;
@@ -42 +42 @@
   }             
   catalog[0].Nsecfilt --;
-  catalog[0].Nsecf_mem = catalog[0].Naverage*catalog[0].Nsecfilt;
+  catalog[0].Nsecfilt_mem = catalog[0].Naverage*catalog[0].Nsecfilt;
 
   *myPrimary = primary;
@@ -58 +58 @@
   Ntmpfilt = catalog[0].Nsecfilt;
 
-  // we do NOT modify Nsecf_disk; this operation only modifies in in-memory values
+  // we do NOT modify Nsecfilt_disk; this operation only modifies in in-memory values
 
   catalog[0].Nsecfilt ++;
@@ -64 +64 @@
   Ntotal = Nsecfilt * Naves;
 
-  catalog[0].Nsecf_mem = Ntotal;
+  catalog[0].Nsecfilt_mem = Ntotal;
 
   ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
@@ -84 +84 @@
   // ohana_memcheck_func (TRUE);
   
-
-  if (0) {
-    // XXX a TEST : is the data corrupted or the file?
-    char name[1024];
-    snprintf (name, 1024, "%s.phu.v2", catalog->filename);
-    FILE *f = fopen (name, "w");
-    fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
-    // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
-    // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", 
-    //       catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], 
-    //       catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
-    fclose (f);
-  }
-  if (0) {
-    // XXX a TEST : is the data corrupted or the file?
-    char name[1024];
-    snprintf (name, 1024, "%s.phu", catalog->filename);
-    FILE *f = fopen (name, "w");
-    fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
-    // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
-    // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", 
-    //       catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], 
-    //       catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
-    fclose (f);
-  }
-# if (0)
-  {
-    // XXX a TEST : is the data corrupted or the file?
-    char name[1024];
-    snprintf (name, 1024, "%s.phu.v2", catalog->filename);
-    FILE *f = fopen (name, "w");
-    fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
-    fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
-    fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", 
-             catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], 
-             catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
-    fclose (f);
-  }
-  {
-    // XXX a TEST : is the data corrupted or the file?
-    char name[1024];
-    snprintf (name, 1024, "%s.phu.v3", catalog->filename);
-    FILE *f = fopen (name, "w");
-    fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
-    fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
-    fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", 
-             catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], 
-             catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
-    fclose (f);
-  }
-# endif
   /* rewind file pointers and truncate (file is still open) */
   if (fseeko (catalog->f, 0LL, SEEK_SET)) {
@@ -278 +227 @@
 
 // ftable must already exist and have a valid, loaded header
-int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder) {
+int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder, char VERBOSE) {
 
   // fread_ftable_data requires the header
@@ -310 +259 @@
     // copies values but does not allocate new memory
     gfits_copy_header_ptr (&rawheader, ftable->header);
-    gfits_copy_table_ptr (&rawtable, ftable);
+    gfits_copy_ftable_ptr (&rawtable, ftable);
     *nativeOrder = TRUE;
   }
   return TRUE;
 }
+
+// LOAD_SUBCAT(measure,MEASURE,Measure)
+
+# define LOAD_SUBCAT(FIELD, NAME, STRUCT)                               \
+  {                                                                     \
+    Header header;                                                      \
+    FTable ftable;                                                      \
+    off_t Nitems;                                                       \
+    gfits_init_table  (&ftable);                                        \
+    gfits_init_header (&header);                                        \
+    ftable.header = &header;                                            \
+    int status = DVO_CAT_OPEN_EMPTY;                                    \
+    if (!(catalog[0].catflags & DVO_SKIP_##NAME)) {                     \
+      status = dvo_catalog_open_subcat (catalog, &catalog[0].FIELD##_catalog, ftable.header, #NAME, VERBOSE); \
+      if (status == DVO_CAT_OPEN_FAIL) {                                \
+        return (FALSE);                                                 \
+      }                                                                 \
+      if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].N##FIELD##_disk > 0)) { \
+        return (FALSE);                                                 \
+      }                                                                 \
+    }                                                                   \
+    if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & DVO_LOAD_##NAME)) { \
+      char nativeOrder = FALSE;                                         \
+      /* read table data */                                             \
+      if (!gfits_fread_uncompressed (catalog[0].FIELD##_catalog, &ftable, &nativeOrder, VERBOSE)) { \
+        if (VERBOSE) fprintf (stderr, "can't read table %s data\n", #FIELD); \
+        gfits_free_header (&header);                                    \
+        return (FALSE);                                                 \
+      }                                                                 \
+      /* convert data format to internal : returns number of row read in Nvalues */ \
+      catalog[0].FIELD = FtableTo##STRUCT (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, nativeOrder); \
+        if (Nitems != catalog[0].N##FIELD##_disk) {                     \
+          fprintf (stderr, "Warning: mismatch between N%s in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", #FIELD, Nitems, catalog[0].N##FIELD##_disk); \
+        }                                                               \
+        catalog[0].N##FIELD = catalog[0].N##FIELD##_disk;                       \
+          catalog[0].N##FIELD##_off = 0;                                        \
+    } else {                                                            \
+      if (catalog[0].FIELD##_catalog) {                                 \
+        gfits_free_header (&catalog[0].FIELD##_catalog[0].header);      \
+      } else {                                                          \
+        ALLOCATE (catalog[0].FIELD##_catalog, Catalog, 1);              \
+        dvo_catalog_init (catalog[0].FIELD##_catalog, TRUE);            \
+      }                                                                 \
+      gfits_create_header (&catalog[0].FIELD##_catalog[0].header);      \
+      ALLOCATE (catalog[0].FIELD, STRUCT, 1);                           \
+      catalog[0].N##FIELD = 0;                                          \
+        catalog[0].N##FIELD##_off = catalog[0].N##FIELD##_disk;         \
+    }                                                                   \
+    gfits_free_header (&header);                                        \
+  }
 
 int dvo_catalog_load_split (Catalog *catalog, int VERBOSE) {
@@ -326 +325 @@
   off_t Nstarpar;
   off_t Ngalshape;
-  off_t Nitems;
-  int status, Nsecfilt;
+  int Nsecfilt;
   SecFilt *primary;
 
   primary = NULL;
-  status = FALSE;
 
   /* get the components from the header - these duplicate information in the split files (NAXIS2) */
@@ -357 +354 @@
 
   /* save the current number so we can do partial updates */
-  catalog[0].Naves_disk = Naverage;
-  catalog[0].Nmeas_disk = Nmeasure;
-  catalog[0].Nmiss_disk = Nmissing;
-  catalog[0].Nsecf_disk = Naverage * Nsecfilt;
+  catalog[0].Naverage_disk = Naverage;
+  catalog[0].Nmeasure_disk = Nmeasure;
+  catalog[0].Nmissing_disk = Nmissing;
+  catalog[0].Nsecfilt_disk = Naverage * Nsecfilt;
   catalog[0].Nlensing_disk  = Nlensing;
   catalog[0].Nlensobj_disk  = Nlensobj;
   catalog[0].Nstarpar_disk  = Nstarpar;
   catalog[0].Ngalshape_disk = Ngalshape;
-
-  /**  Nsecfilt is unusual: it does not list the number of data items in the table
-       instead, the number of items is Nsecfilt * Naverage.  **/
-  catalog[0].Nsecfilt  = Nsecfilt;
 
   /* default values, but we will assign these a valid value before we exit (even if empty) */
@@ -388 +381 @@
 
   /* ftable header storage for below */
+  char nativeOrder;
   Header header;
   FTable ftable;
@@ -399 +393 @@
     return (FALSE);
   }
-  if (catalog[0].catflags & LOAD_AVES) {
-    char nativeOrder = FALSE;
-    if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder)) {
+  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
+    if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder, VERBOSE)) {
       if (VERBOSE) fprintf (stderr, "can't read Average table\n");
       gfits_free_header (&header);
@@ -407 +400 @@
     }
 
-    /* convert the saved version of the table to the internal version.  Old versions of DVO stored
+    /* convert the disk version of the table to the internal version.  Old versions of DVO stored
      * one of the average magnitudes in Average.  We save this in case it is needed below.  NOTE:
      * primary is only used if we read in the secfilt table, otherwise it should be freed */
     catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, nativeOrder);
-    if (Naverage != catalog[0].Naves_disk) {
-      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naves_disk);
-    }
-    catalog[0].Naverage = catalog[0].Naves_disk;
-    catalog[0].Naves_off = 0;
+    if (Naverage != catalog[0].Naverage_disk) {
+      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naverage_disk);
+    }
+    catalog[0].Naverage = catalog[0].Naverage_disk;
+    catalog[0].Naverage_off = 0;
   } else {
     catalog[0].catformat = FtableGetFormat (&ftable);
     ALLOCATE (catalog[0].average, Average, 1);
     catalog[0].Naverage = 0;
-    catalog[0].Naves_off = catalog[0].Naves_disk;
+    catalog[0].Naverage_off = catalog[0].Naverage_disk;
   }
   gfits_free_header (&header);
 
   /*** Measure Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  if (!(catalog[0].catflags & SKIP_MEAS)) {
-    // unless we specify 'skip', we still need to load the 
-    status = dvo_catalog_open_subcat (catalog, &catalog[0].measure_catalog, ftable.header, "MEASURE", VERBOSE);
-    if (status == DVO_CAT_OPEN_FAIL) {
-      return (FALSE);
-    }
-    if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmeas_disk > 0)) {
-      return (FALSE);
-    }
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) {
-    char nativeOrder = FALSE;
-    /* read Measure table data */
-    if (!gfits_fread_uncompressed (catalog[0].measure_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table measure data\n");
-      gfits_free_header (&header);
-      return (FALSE);
-    }
-    /* convert data format to internal : returns number of row read in Nmeasure */
-    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, nativeOrder);
-    if (Nmeasure != catalog[0].Nmeas_disk) {
-      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
-    }
-    catalog[0].Nmeasure = catalog[0].Nmeas_disk;
-    catalog[0].Nmeas_off = 0;
-  } else {
-    // XXX is it necessary to generate a template header here?
-    // XXX this is a memory leak, right?
-    if (catalog[0].measure_catalog) {
-      gfits_free_header (&catalog[0].measure_catalog[0].header);
+  
+  LOAD_SUBCAT(measure,MEASURE,Measure);
+  LOAD_SUBCAT(missing,MISSING,Missing);
+  LOAD_SUBCAT(secfilt,SECFILT,SecFilt);
+  LOAD_SUBCAT(lensing,LENSING,Lensing);
+  LOAD_SUBCAT(lensobj,LENSOBJ,Lensobj);
+  LOAD_SUBCAT(starpar,STARPAR,StarPar);
+  LOAD_SUBCAT(galshape,GALSHAPE,GalShape);
+
+  /**  catalog->Nsecfilt is unusual: it does not list the number of data items in the
+       table instead, the number of items is Nsecfilt * Naverage, and is stored in
+       Nsecfilt_mem. fix these below **/
+
+  catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt;
+  catalog[0].Nsecfilt  = Nsecfilt;
+
+  /** some old formats stored one of the secfilt values in the average table. 
+      repair this below **/
+
+  if (primary) {
+    if (catalog[0].Nsecfilt_mem) {
+      dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naverage_disk);
     } else {
-      ALLOCATE (catalog[0].measure_catalog, Catalog, 1);
-      dvo_catalog_init (catalog[0].measure_catalog, TRUE);
-    }
-    gfits_create_header (&catalog[0].measure_catalog[0].header);
-    ALLOCATE (catalog[0].measure, Measure, 1);
-    catalog[0].Nmeasure = 0;
-    catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
-  }
-  gfits_free_header (&header);
-
-  /*** Missing Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE);
-  if (status == DVO_CAT_OPEN_FAIL) {
-    return (FALSE);
-  }
-  if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmiss_disk > 0)) {
-    return (FALSE);
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) {
-    char nativeOrder = FALSE;
-    /* read Missing table data */
-    if (!gfits_fread_uncompressed (catalog[0].missing_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table missing data\n");
-      return (FALSE);
-    }
-    /* no conversions currently defined : this just does the byte swap */
-    catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, &nativeOrder);
-    if (!catalog[0].missing) {
-      fprintf (stderr, "ERROR: failed to read missing\n");
-      exit (2);
-    }
-    if (Nmissing != catalog[0].Nmiss_disk) {
-      fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmiss_disk);
-    }
-    catalog[0].Nmissing = catalog[0].Nmiss_disk;
-    catalog[0].Nmiss_off = 0;
-  } else {
-    // XXX is it necessary to generate a template header here?
-    gfits_create_header (&catalog[0].missing_catalog[0].header);
-    ALLOCATE (catalog[0].missing, Missing, 1);
-    catalog[0].Nmissing = 0;
-    catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
-  }
-  gfits_free_header (ftable.header);
-
-  /*** Secfilt Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE);
-  // fprintf (stderr, "secfilt header %s : %zx\n", catalog[0].secfilt_catalog[0].filename, (size_t) catalog[0].secfilt_catalog[0].header.buffer);
-  if (status == DVO_CAT_OPEN_FAIL) {
-    return (FALSE);
-  }
-  if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nsecf_disk > 0)) {
-    return (FALSE);
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) {
-    char nativeOrder = FALSE;
-    /* read secfilt table data */
-    if (!gfits_fread_uncompressed (catalog[0].secfilt_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n");
-      return (FALSE);
-    }
-    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, nativeOrder);
-    if (Nitems != catalog[0].Nsecf_disk) {
-      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  catalog[0].Nsecf_disk);
-    }
-    catalog[0].Nsecf_mem = catalog[0].Nsecf_disk;
-    catalog[0].Nsecf_off = 0;
-
-    /* if primary is defined, we were supplied with one additional average magnitude from Average
-       we need to interleave these magnitudes with the secfilt entries just loaded */
-    if (primary != NULL) {
-      // this modifies catalog.Nsecf_mem,Nsecfilt
-      dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naves_disk);
-    } 
-  } else {
-    if (primary != NULL) {
       free (primary);
       catalog[0].Nsecfilt ++;
     }
-    gfits_create_header (&catalog[0].secfilt_catalog[0].header);
-    ALLOCATE (catalog[0].secfilt, SecFilt, 1);
-    catalog[0].Nsecf_mem = 0;
-    catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
-  }
-  gfits_free_header (ftable.header);
-
-  /*** Lensing Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  if (!(catalog[0].catflags & SKIP_LENSING)) {
-    // unless we specify 'skip', we still need to load the Lensing table
-    status = dvo_catalog_open_subcat (catalog, &catalog[0].lensing_catalog, ftable.header, "LENSING", VERBOSE);
-    if (status == DVO_CAT_OPEN_FAIL) {
-      return (FALSE);
-    }
-    // the Lensing file need not exist, unless we expect data to exist
-    if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensing_disk > 0)) {
-      return (FALSE);
-    }
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSING)) {
-    char nativeOrder = FALSE;
-    // only read the Lensing table if the file actually exists
-    /* read Lensing table data */
-    if (!gfits_fread_uncompressed (catalog[0].lensing_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table lensing data\n");
-      return (FALSE);
-    }
-    /* convert data format to internal : returns number of row read in Nlensing */
-    catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat, nativeOrder);
-    if (Nlensing != catalog[0].Nlensing_disk) {
-      fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
-    }
-    catalog[0].Nlensing = catalog[0].Nlensing_disk;
-    catalog[0].Nlensing_off = 0;
-  } else {
-    if (catalog[0].lensing_catalog) {
-      gfits_free_header (&catalog[0].lensing_catalog[0].header);
-    } else {
-      ALLOCATE (catalog[0].lensing_catalog, Catalog, 1);
-      dvo_catalog_init (catalog[0].lensing_catalog, TRUE);
-    }
-    gfits_create_header (&catalog[0].lensing_catalog[0].header);
-    ALLOCATE (catalog[0].lensing, Lensing, 1);
-    catalog[0].Nlensing = 0;
-    catalog[0].Nlensing_off = catalog[0].Nlensing_disk;
-  }
-  gfits_free_header (&header);
-
-  /*** Lensobj Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  if (!(catalog[0].catflags & SKIP_LENSOBJ)) {
-    // unless we specify 'skip', we still need to load the 
-    status = dvo_catalog_open_subcat (catalog, &catalog[0].lensobj_catalog, ftable.header, "LENSOBJ", VERBOSE);
-    if (status == DVO_CAT_OPEN_FAIL) {
-      return (FALSE);
-    }
-    // the Lensobj file need not exist, unless we expect data to exist
-    if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensobj_disk > 0)) {
-      return (FALSE);
-    }
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSOBJ)) {
-    char nativeOrder = FALSE;
-    // only read the Lensobj table if the file actually exists
-    /* read Lensobj table data */
-    if (!gfits_fread_uncompressed (catalog[0].lensobj_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table lensobj data\n");
-      return (FALSE);
-    }
-    /* convert data format to internal : returns number of row read in Nlensobj */
-    catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat, FALSE);
-    if (Nlensobj != catalog[0].Nlensobj_disk) {
-      fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
-    }
-    catalog[0].Nlensobj = catalog[0].Nlensobj_disk;
-    catalog[0].Nlensobj_off = 0;
-  } else {
-    if (catalog[0].lensobj_catalog) {
-      gfits_free_header (&catalog[0].lensobj_catalog[0].header);
-    } else {
-      ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1);
-      dvo_catalog_init (catalog[0].lensobj_catalog, TRUE);
-    }
-    gfits_create_header (&catalog[0].lensobj_catalog[0].header);
-    ALLOCATE (catalog[0].lensobj, Lensobj, 1);
-    catalog[0].Nlensobj = 0;
-    catalog[0].Nlensobj_off = catalog[0].Nlensobj_disk;
-  }
-  gfits_free_header (&header);
-
-  /*** StarPar Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  if (!(catalog[0].catflags & SKIP_STARPAR)) {
-    // unless we specify 'skip', we still need to load the 
-    status = dvo_catalog_open_subcat (catalog, &catalog[0].starpar_catalog, ftable.header, "STARPAR", VERBOSE);
-    if (status == DVO_CAT_OPEN_FAIL) {
-      return (FALSE);
-    }
-    // the StarPar file need not exist, unless we expect data to exist
-    if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nstarpar_disk > 0)) {
-      return (FALSE);
-    }
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_STARPAR)) {
-    char nativeOrder = FALSE;
-    // only read the StarPar table if the file actually exists
-    /* read StarPar table data */
-    if (!gfits_fread_uncompressed (catalog[0].starpar_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table starpar data\n");
-      return (FALSE);
-    }
-    /* convert data format to internal : returns number of row read in Nstarpar */
-    catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat, FALSE);
-    if (Nstarpar != catalog[0].Nstarpar_disk) {
-      fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
-    }
-    catalog[0].Nstarpar = catalog[0].Nstarpar_disk;
-    catalog[0].Nstarpar_off = 0;
-  } else {
-    if (catalog[0].starpar_catalog) {
-      gfits_free_header (&catalog[0].starpar_catalog[0].header);
-    } else {
-      ALLOCATE (catalog[0].starpar_catalog, Catalog, 1);
-      dvo_catalog_init (catalog[0].starpar_catalog, TRUE);
-    }
-    gfits_create_header (&catalog[0].starpar_catalog[0].header);
-    ALLOCATE (catalog[0].starpar, StarPar, 1);
-    catalog[0].Nstarpar = 0;
-    catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk;
-  }
-  gfits_free_header (&header);
-
-  /*** GalShape Table ***/
-  gfits_init_table  (&ftable);
-  gfits_init_header (&header);
-  ftable.header = &header;
-  if (!(catalog[0].catflags & SKIP_GALSHAPE)) {
-    // unless we specify 'skip', we still need to load the GalShape table
-    status = dvo_catalog_open_subcat (catalog, &catalog[0].galshape_catalog, ftable.header, "GALSHAPE", VERBOSE);
-    if (status == DVO_CAT_OPEN_FAIL) {
-      return (FALSE);
-    }
-    // the GalShape file need not exist, unless we expect data to exist
-    if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Ngalshape_disk > 0)) {
-      return (FALSE);
-    }
-  }
-  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_GALSHAPE)) {
-    char nativeOrder = FALSE;
-    // only read the GalShape table if the file actually exists
-    /* read GalShape table data */
-    if (!gfits_fread_uncompressed (catalog[0].galshape_catalog, &ftable, &nativeOrder)) {
-      if (VERBOSE) fprintf (stderr, "can't read table galshape data\n");
-      return (FALSE);
-    }
-    /* convert data format to internal : returns number of row read in Ngalshape */
-    catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat, FALSE);
-    if (Ngalshape != catalog[0].Ngalshape_disk) {
-      fprintf (stderr, "Warning: mismatch between Ngalshape in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalshape,  catalog[0].Ngalshape_disk);
-    }
-    catalog[0].Ngalshape = catalog[0].Ngalshape_disk;
-    catalog[0].Ngalshape_off = 0;
-  } else {
-    if (catalog[0].galshape_catalog) {
-      gfits_free_header (&catalog[0].galshape_catalog[0].header);
-    } else {
-      ALLOCATE (catalog[0].galshape_catalog, Catalog, 1);
-      dvo_catalog_init (catalog[0].galshape_catalog, TRUE);
-    }
-    gfits_create_header (&catalog[0].galshape_catalog[0].header);
-    ALLOCATE (catalog[0].galshape, GalShape, 1);
-    catalog[0].Ngalshape = 0;
-    catalog[0].Ngalshape_off = catalog[0].Ngalshape_disk;
-  }
-  gfits_free_header (&header);
+  }
 
   return (TRUE);
@@ -747 +466 @@
 
   /*** Average (& SecFilt) Table ***/
-  if (catalog[0].catflags & LOAD_AVES) {
+  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
 
     /*** load the Average data ***/
@@ -774 +493 @@
     gfits_free_header (&header);
     catalog[0].Naverage = Naverage;
-    catalog[0].Naves_off = start;
+    catalog[0].Naverage_off = start;
 
     /*** load the secfilt data ***/
@@ -795 +514 @@
 
     Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt;
-    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, FALSE);
+    catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE);
     if (Nitems != Nexpect) {
       fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  Nexpect);
     }
-    catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt;
-    catalog[0].Nsecf_off = start               * catalog[0].Nsecfilt;
+    catalog[0].Nsecfilt_mem = catalog[0].Naverage * catalog[0].Nsecfilt;
+    catalog[0].Nsecfilt_off = start               * catalog[0].Nsecfilt;
 
     /* if primary is defined, we were supplied with one additional average magnitude from Average
@@ -811 +530 @@
 
   // XXX check the open status of the catalog
-  if (catalog[0].catflags & LOAD_MEAS) {
+  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
 
     Catalog *subcat = catalog[0].measure_catalog;
@@ -838 +557 @@
     gfits_free_header (&header);
     catalog[0].Nmeasure = Nmeasure;
-    catalog[0].Nmeas_off = start;
+    catalog[0].Nmeasure_off = start;
   }
 
   // XXX check the open status of the catalog?
-  if (catalog[0].catflags & LOAD_MISS) {
+  if (catalog[0].catflags & DVO_LOAD_MISSING) {
 
     Catalog *subcat = catalog[0].missing_catalog;
@@ -872 +591 @@
     gfits_free_header (&header);
     catalog[0].Nmissing = Nmissing;
-    catalog[0].Nmiss_off = start;
+    catalog[0].Nmissing_off = start;
   }
 
   // XXX check the open status of the catalog
-  if (catalog[0].catflags & LOAD_LENSING) {
+  if (catalog[0].catflags & DVO_LOAD_LENSING) {
 
     Catalog *subcat = catalog[0].lensing_catalog;
@@ -896 +615 @@
 
     /* convert data format to internal : returns number of row read in Nlensing */
-    catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat, FALSE);
+    catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &Nlensing, &catalog[0].catformat, FALSE);
     if (Nlensing != Nrows) {
       // XXX this condition denotes the eof has been reached; not an error or a warning
@@ -907 +626 @@
 
   // XXX check the open status of the catalog
-  if (catalog[0].catflags & LOAD_LENSOBJ) {
+  if (catalog[0].catflags & DVO_LOAD_LENSOBJ) {
 
     Catalog *subcat = catalog[0].lensobj_catalog;
@@ -927 +646 @@
 
     /* convert data format to internal : returns number of row read in Nlensobj */
-    catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat, FALSE);
+    catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &Nlensobj, &catalog[0].catformat, FALSE);
     if (Nlensobj != Nrows) {
       // XXX this condition denotes the eof has been reached; not an error or a warning
@@ -938 +657 @@
 
   // XXX check the open status of the catalog
-  if (catalog[0].catflags & LOAD_STARPAR) {
+  if (catalog[0].catflags & DVO_LOAD_STARPAR) {
 
     Catalog *subcat = catalog[0].starpar_catalog;
@@ -958 +677 @@
 
     /* convert data format to internal : returns number of row read in Nstarpar */
-    catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat, FALSE);
+    catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &Nstarpar, &catalog[0].catformat, FALSE);
     if (Nstarpar != Nrows) {
       // XXX this condition denotes the eof has been reached; not an error or a warning
@@ -969 +688 @@
 
   // XXX check the open status of the catalog
-  if (catalog[0].catflags & LOAD_GALSHAPE) {
+  if (catalog[0].catflags & DVO_LOAD_GALSHAPE) {
 
     Catalog *subcat = catalog[0].galshape_catalog;
@@ -989 +708 @@
 
     /* convert data format to internal : returns number of row read in Ngalshape */
-    catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat, FALSE);
+    catalog[0].galshape = FtableToGalShape (&ftable, catalog[0].average, &Ngalshape, &catalog[0].catformat, FALSE);
     if (Ngalshape != Nrows) {
       // XXX this condition denotes the eof has been reached; not an error or a warning
@@ -1010 +729 @@
   SecFilt *primary, *secfilt;
   int Nsecfilt;
-  off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
+  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
   off_t first, start, Nrows;
 
@@ -1019 +738 @@
   
   // skip empty catalogs: it is illegal to have Measures without corresponding Averages
-  Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off);
-  if (Naves_disk_new == 0) {
+  Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off);
+  if (Naverage_disk_new == 0) {
     if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n");
     return (TRUE);
@@ -1038 +757 @@
   }
 
-  Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off);
-  Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off);
-  Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off);
+  Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off);
+  Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off);
+  Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off);
   Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off);
   Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off);
@@ -1047 +766 @@
 
   /* make sure header is consistent with data */
-  gfits_modify (&catalog[0].header, "NSTARS",    OFF_T_FMT, 1,  Naves_disk_new);
-  gfits_modify (&catalog[0].header, "NMEAS",     OFF_T_FMT, 1,  Nmeas_disk_new);
-  gfits_modify (&catalog[0].header, "NMISS",     OFF_T_FMT, 1,  Nmiss_disk_new);
+  gfits_modify (&catalog[0].header, "NSTARS",    OFF_T_FMT, 1,  Naverage_disk_new);
+  gfits_modify (&catalog[0].header, "NMEAS",     OFF_T_FMT, 1,  Nmeasure_disk_new);
+  gfits_modify (&catalog[0].header, "NMISS",     OFF_T_FMT, 1,  Nmissing_disk_new);
   gfits_modify (&catalog[0].header, "NSECFILT",  "%d",      1,  Nsecfilt);
   gfits_modify (&catalog[0].header, "NLENSING",  OFF_T_FMT, 1,  Nlensing_disk_new);
@@ -1062 +781 @@
 
   /*** Average Table ***/
-  if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Naves_off; // first disk row to write
+    start  = catalog[0].Naverage_off; // first disk row to write
     Nrows  = catalog[0].Naverage - first;
 
@@ -1071 +790 @@
     assert (first >= 0);
     assert (first <= catalog[0].Naverage);
-    assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
+    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
 
     /* convert internal to external format : also results in a byte-swapped, scaled output
@@ -1078 +797 @@
     
     // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (start, Nrows, Naves_disk_new, catalog->catcompress);
+    int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress);
 
     if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) {
@@ -1085 +804 @@
     }
 
-    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
       fprintf (stderr, "failure writing Average table\n");
       goto failure;
@@ -1105 +824 @@
 
   /*** Measure Table ***/
-  if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Nmeas_off; // first disk row to write
+    start  = catalog[0].Nmeasure_off; // first disk row to write
     Nrows  = catalog[0].Nmeasure - first;
 
@@ -1114 +833 @@
     assert (first >= 0);
     assert (first <= catalog[0].Nmeasure);
-    assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
+    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
 
     // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (start, Nrows, Nmeas_disk_new, catalog->catcompress);
+    int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
 
     // convert to external table format (note that the block above does not damage or free catalog.average) 
@@ -1126 +845 @@
 
     // write out Measure table
-    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
       fprintf (stderr, "trouble writing Measure table\n");
       goto failure;
@@ -1135 +854 @@
 
   /*** Missing Table ***/
-  if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) {
-
-    if (catalog[0].Nmiss_off != 0) {
+  if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) {
+
+    if (catalog[0].Nmissing_off != 0) {
       fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
       goto failure;
@@ -1161 +880 @@
 
   /*** Secfilt Table ***/
-  if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Nsecf_off; // first disk row to write
-    Nitems = catalog[0].Nsecf_mem;
+    start  = catalog[0].Nsecfilt_off; // first disk row to write
+    Nitems = catalog[0].Nsecfilt_mem;
     Nrows  = Nitems - first;
 
@@ -1171 +890 @@
     assert (first >= 0);
     assert (first <= Nitems);
-    assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
+    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
     // XXX check these for consistency...
 
     // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (start, Nrows, Nsecf_disk_new, catalog->catcompress);
+    int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
 
     // convert to external table format
@@ -1181 +900 @@
 
     // write out SecFilt table
-    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
       fprintf (stderr, "failure writing SecFilt table\n");
       goto failure;
@@ -1190 +909 @@
 
   /*** Lensing Table (optional, do not save if not loaded) ***/
-  if ((catalog[0].catflags & LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) {
+  if ((catalog[0].catflags & DVO_LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) {
 
     first  = 0;                    // first row in memory to write
@@ -1222 +941 @@
 
   /*** Lensobj Table (optional, do not save if not loaded) ***/
-  if ((catalog[0].catflags & LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) {
+  if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) {
 
     first  = 0;                    // first row in memory to write
@@ -1254 +973 @@
 
   /*** StarPar Table (optional, do not save if not loaded) ***/
-    if ((catalog[0].catflags & LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) {
+    if ((catalog[0].catflags & DVO_LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) {
 
     first  = 0;                    // first row in memory to write
@@ -1286 +1005 @@
 
   /*** GalShape Table (optional, do not save if not loaded) ***/
-  if ((catalog[0].catflags & LOAD_GALSHAPE) && catalog[0].galshape_catalog->f && catalog[0].galshape) {
+  if ((catalog[0].catflags & DVO_LOAD_GALSHAPE) && catalog[0].galshape_catalog->f && catalog[0].galshape) {
 
     first  = 0;                    // first row in memory to write
@@ -1343 +1062 @@
   SecFilt *primary, *secfilt;
   int Nsecfilt;
-  off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
+  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
   off_t first, start, Nrows;
 
@@ -1352 +1071 @@
   
   // skip empty catalogs: it is illegal to have Measures without corresponding Averages
-  if (catalog[0].Naves_off > 0) {
+  if (catalog[0].Naverage_off > 0) {
     fprintf (stderr, "ERROR: only partial catalog (Average) was loaded\n");
     goto failure;
   }
-  if (catalog[0].Nmeas_off > 0) {
+  if (catalog[0].Nmeasure_off > 0) {
     fprintf (stderr, "ERROR: only partial catalog (Measure) was loaded\n");
     goto failure;
   }
-  if (catalog[0].Nmiss_off > 0) {
+  if (catalog[0].Nmissing_off > 0) {
     fprintf (stderr, "ERROR: only partial catalog (Missing) was loaded\n");
     goto failure;
   }
-  if (catalog[0].Nsecf_off > 0) {
+  if (catalog[0].Nsecfilt_off > 0) {
     fprintf (stderr, "ERROR: only partial catalog (Secfilt) was loaded\n");
     goto failure;
@@ -1385 +1104 @@
   }
 
-  Naves_disk_new = catalog[0].Naverage;
-  if (Naves_disk_new == 0) {
+  Naverage_disk_new = catalog[0].Naverage;
+  if (Naverage_disk_new == 0) {
     if (VERBOSE) fprintf (stderr, "resulting catalog is empty; delete it\n");
     // unlink ();
@@ -1405 +1124 @@
   }
 
-  Nmeas_disk_new = catalog[0].Nmeasure;
-  Nmiss_disk_new = catalog[0].Nmissing;
-  Nsecf_disk_new = catalog[0].Naverage*Nsecfilt;
+  Nmeasure_disk_new = catalog[0].Nmeasure;
+  Nmissing_disk_new = catalog[0].Nmissing;
+  Nsecfilt_disk_new = catalog[0].Naverage*Nsecfilt;
   Nlensing_disk_new = catalog[0].Nlensing;
   Nlensobj_disk_new = catalog[0].Nlensobj;
@@ -1414 +1133 @@
 
   /* make sure header is consistent with data */
-  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naves_disk_new);
-  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeas_disk_new);
-  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmiss_disk_new);
+  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naverage_disk_new);
+  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeasure_disk_new);
+  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmissing_disk_new);
   gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt);
   gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
@@ -1428 +1147 @@
 
   /*** Average Table ***/
-  if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Naves_off; // first disk row to write
+    start  = catalog[0].Naverage_off; // first disk row to write
     Nrows  = catalog[0].Naverage - first;
 
@@ -1437 +1156 @@
     assert (first >= 0);
     assert (first <= catalog[0].Naverage);
-    assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
+    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
 
     /* convert internal to external format */
@@ -1445 +1164 @@
     }
 
-    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
       fprintf (stderr, "failure writing Average table\n");
       goto failure;
@@ -1465 +1184 @@
 
   /*** Measure Table ***/
-  if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Nmeas_off; // first disk row to write
+    start  = catalog[0].Nmeasure_off; // first disk row to write
     Nrows  = catalog[0].Nmeasure - first;
 
@@ -1474 +1193 @@
     assert (first >= 0);
     assert (first <= catalog[0].Nmeasure);
-    assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
+    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
 
     // convert to external table format
@@ -1483 +1202 @@
 
     // write out Measure table
-    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
       fprintf (stderr, "trouble writing Measure table\n");
       goto failure;
@@ -1492 +1211 @@
 
   /*** Missing Table ***/
-  if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) {
-
-    if (catalog[0].Nmiss_off != 0) {
+  if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) {
+
+    if (catalog[0].Nmissing_off != 0) {
       fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
       goto failure;
@@ -1515 +1234 @@
 
   /*** Secfilt Table ***/
-  if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) {
 
     first  = 0;                    // first row in memory to write
-    start  = catalog[0].Nsecf_off; // first disk row to write
+    start  = catalog[0].Nsecfilt_off; // first disk row to write
     Nitems = catalog[0].Naverage*Nsecfilt;
     Nrows  = Nitems - first;
@@ -1525 +1244 @@
     assert (first >= 0);
     assert (first <= Nitems);
-    assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
+    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
     // XXX check these for consistency...
 
@@ -1532 +1251 @@
 
     // write out SecFilt table
-    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
       fprintf (stderr, "failure writing SecFilt table\n");
       goto failure;
@@ -1541 +1260 @@
 
   /*** Lensing Table ***/
-  if ((catalog[0].catflags & LOAD_LENSING) && (catalog[0].lensing != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_LENSING) && (catalog[0].lensing != NULL)) {
 
     first  = 0;                    // first row in memory to write
@@ -1568 +1287 @@
 
   /*** Lensobj Table ***/
-  if ((catalog[0].catflags & LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) {
 
     first  = 0;                    // first row in memory to write
@@ -1595 +1314 @@
 
   /*** StarPar Table ***/
-  if ((catalog[0].catflags & LOAD_STARPAR) && (catalog[0].starpar != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_STARPAR) && (catalog[0].starpar != NULL)) {
 
     first  = 0;                    // first row in memory to write
@@ -1622 +1341 @@
 
   /*** GalShape Table ***/
-  if ((catalog[0].catflags & LOAD_GALSHAPE) && (catalog[0].galshape != NULL)) {
+  if ((catalog[0].catflags & DVO_LOAD_GALSHAPE) && (catalog[0].galshape != NULL)) {
 
     first  = 0;                    // first row in memory to write
@@ -1673 +1392 @@
   SecFilt *primary, *secfilt;
   int Nsecfilt;
-  off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
+  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalshape_disk_new;
   off_t first, start, Nrows;
 
@@ -1681 +1400 @@
 
   // skip empty catalogs: it is illegal to have Measures without corresponding Averages
-  Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off);
-  if (Naves_disk_new == 0) {
+  Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off);
+  if (Naverage_disk_new == 0) {
     if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n");
     return (TRUE);
@@ -1700 +1419 @@
   }
 
-  Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off);
-  Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off);
-  Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off);
+  Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off);
+  Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off);
+  Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off);
   Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off);
   Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off);
@@ -1709 +1428 @@
 
   /* make sure header is consistent with data */
-  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naves_disk_new);
-  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeas_disk_new);
-  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmiss_disk_new);
+  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naverage_disk_new);
+  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeasure_disk_new);
+  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmissing_disk_new);
   gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt);
   gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
@@ -1726 +1445 @@
   if (catalog[0].average != NULL) {
 
-    first  = catalog[0].Naves_disk - catalog[0].Naves_off; // first row to write (memory)
-    start  = catalog[0].Naves_disk;                        // first row to write (disk)
+    first  = catalog[0].Naverage_disk - catalog[0].Naverage_off; // first row to write (memory)
+    start  = catalog[0].Naverage_disk;                        // first row to write (disk)
     Nrows  = catalog[0].Naverage - first;
 
@@ -1733 +1452 @@
     assert (first >= 0);
     assert (first <= catalog[0].Naverage);
-    assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
+    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
 
     /* convert internal to external format */
@@ -1741 +1460 @@
     }
 
-    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
       fprintf (stderr, "failure writing Average table\n");
       goto failure;
@@ -1763 +1482 @@
   if (catalog[0].measure != NULL) {
 
-    first  = catalog[0].Nmeas_disk - catalog[0].Nmeas_off;  // first row in memory to write
-    start  = catalog[0].Nmeas_disk; // first disk row to write
+    first  = catalog[0].Nmeasure_disk - catalog[0].Nmeasure_off;  // first row in memory to write
+    start  = catalog[0].Nmeasure_disk; // first disk row to write
     Nrows  = catalog[0].Nmeasure - first;
 
@@ -1770 +1489 @@
     assert (first >= 0);
     assert (first <= catalog[0].Nmeasure);
-    assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
+    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
 
     // convert to external table format
@@ -1780 +1499 @@
 
     // write out Measure table
-    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
       fprintf (stderr, "trouble writing Measure table\n");
       goto failure;
@@ -1795 +1514 @@
   if (catalog[0].missing != NULL) {
 
-    if (catalog[0].Nmiss_off != 0) {
+    if (catalog[0].Nmissing_off != 0) {
       fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
       goto failure;
@@ -1818 +1537 @@
   if (catalog[0].secfilt != NULL) {
 
-    first  = catalog[0].Nsecf_disk - catalog[0].Nsecf_off;  // first row in memory to write
-    start  = catalog[0].Nsecf_disk; // first disk row to write
-    Nrows  = catalog[0].Nsecf_mem - first;
-
-    assert (Nrows >= 0);
-    assert (first >= 0);
-    assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
+    first  = catalog[0].Nsecfilt_disk - catalog[0].Nsecfilt_off;  // first row in memory to write
+    start  = catalog[0].Nsecfilt_disk; // first disk row to write
+    Nrows  = catalog[0].Nsecfilt_mem - first;
+
+    assert (Nrows >= 0);
+    assert (first >= 0);
+    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
 
     // convert to external table format
@@ -1830 +1549 @@
 
     // write out SecFilt table
-    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
+    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
       fprintf (stderr, "failure writing SecFilt table\n");
       goto failure;
@@ -1976 +1695 @@
 
 /* XXX EAM : update is not efficient.  MeasureToFtable should only 
-   convert the new rows (Nmeas_disk to Nmeasure). the resulting
+   convert the new rows (Nmeasure_disk to Nmeasure). the resulting
    table represents the end rows of the ftable.  we need to define
    the vtable based on the ftable, but with Ny = Nmeasure */  
@@ -1985 +1704 @@
     // * optionally write the PHU header/matrix
     // * advance to the start of the output data block:
-    // ** Nx * catalog[0].Nmeas_off
+    // ** Nx * catalog[0].Nmeasure_off
     // * write out the ftable data block
-    // * if Nmeas_off + Nmeasure >= Nmeas_disk, update padding
-    // ** start = Nmeas_off
+    // * if Nmeasure_off + Nmeasure >= Nmeasure_disk, update padding
+    // ** start = Nmeasure_off
     // ** Nrows = Nmeasure
 

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_convert.c

@@ -180 +180 @@
 
 // FtableToMeasure needs the Average since old formats stored measure.dR,dD only
+// other FtableToFOO conversions accept the average argument for macro construction
 Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeBytes) {
 
@@ -308 +309 @@
 }
 
+/*** Missing / FTable conversion functions ***/
+
+Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes) {
+
+  Missing *missing;
+  char extname[80];
+
+  /* convert to the internal format */
+  if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) {
+    fprintf (stderr, "EXTNAME missing for missing table\n");
+    return (FALSE);
+  }
+
+# define SKIPPING_FORMAT(NAME, FORMAT, TYPE)            \
+  if (!strcmp (extname, NAME)) {                                        \
+    fprintf (stderr, "ERROR: format %s not defined for missing, skipping\n", NAME); \
+    *Nmissing = 0; \
+    return NULL; \
+  }
+
+# define CONVERT_FORMAT(NAME, FORMAT, TYPE)             \
+  if (!strcmp (extname, NAME)) {                                        \
+    Missing_##TYPE *tmpMissing;                                         \
+    tmpMissing = gfits_table_get_Missing_##TYPE (ftable, Nmissing, NULL, NULL); \
+    if (!tmpMissing) {                                                  \
+      fprintf (stderr, "ERROR: failed to read missing\n");              \
+      exit (2);                                                         \
+    }                                                                   \
+    missing = Missing_##TYPE##_ToInternal (tmpMissing, *Nmissing); \
+    free (tmpMissing);                                                  \
+    *format = DVO_FORMAT_##FORMAT;                                      \
+    return (missing); }
+
+  // XXX the structure is here for future expansion, but no transformations are currently defined
+  if (TRUE) {
+    missing = gfits_table_get_Missing (ftable, Nmissing, NULL, NULL);
+    if (!missing) {
+      fprintf (stderr, "ERROR: failed to read missing\n");
+      exit (2);
+    }
+    *format = DVO_FORMAT_INTERNAL;
+    return (missing);
+  }
+
+  if (!strcmp (extname, "DVO_MISSING")) {
+    missing = gfits_table_get_Missing (ftable, Nmissing, NULL, NULL);
+    if (!missing) {
+      fprintf (stderr, "ERROR: failed to read missing\n");
+      exit (2);
+    }
+    *format = DVO_FORMAT_INTERNAL;
+    return (missing);
+  }
+
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_SIM",         PS1_SIM,         PS1_SIM);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_REF",         PS1_REF,         PS1_REF);
+  SKIPPING_FORMAT ("DVO_MISSING_ELIXIR",          ELIXIR,          Elixir);
+  SKIPPING_FORMAT ("DVO_MISSING_LONEOS",          LONEOS,          Loneos);
+  SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0);
+  SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_1",       PS1_DEV_1,       PS1_DEV_1);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_2",       PS1_DEV_2,       PS1_DEV_2);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_V1",          PS1_V1,          PS1_V1);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_V2",          PS1_V2,          PS1_V2);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_V3",          PS1_V3,          PS1_V3);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_V4",          PS1_V4,          PS1_V4);
+  SKIPPING_FORMAT ("DVO_MISSING_PS1_V5",          PS1_V5,          PS1_V5);
+# undef CONVERT_FORMAT
+# undef SKIPPING_FORMAT
+
+  fprintf (stderr, "table format unknown: %s\n", extname);
+
+  *Nmissing = 0;
+  return (NULL);
+}
+
 /*** SecFilt / FTable conversion functions ***/
 
-SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) {
+SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) {
 
   SecFilt *secfilt;
@@ -420 +497 @@
 /*** Lensing / FTable conversion functions ***/
 
-Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) {
+Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) {
 
   Lensing *lensing;
@@ -559 +636 @@
 /*** Lensobj / FTable conversion functions ***/
 
-Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) {
+Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) {
 
   Lensobj *lensobj;
@@ -674 +751 @@
 /*** StarPar / FTable conversion functions ***/
 
-StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) {
+StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) {
 
   StarPar *starpar;
@@ -778 +855 @@
 /*** GalShape / FTable conversion functions ***/
 
-GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format, char nativeBytes) {
+GalShape *FtableToGalShape (FTable *ftable, Average *average, off_t *Ngalshape, DVOCatFormat *format, char nativeBytes) {
 
   GalShape *galshape;

branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_util.c

@@ -207 +207 @@
       Catalog catalog;
       catalog.filename = skylist->filename[reg];
-      catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF;
+      catalog.catflags = GetMeasures ? DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT : DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
       catalog.Nsecfilt = 0;