Index: /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c
===================================================================
--- /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c	(revision 38421)
+++ /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c	(revision 38422)
@@ -12,4 +12,13 @@
   if (fullWrite && catcompress) return TRUE;  // we want to compress, so do not swap
   return FALSE;
+}
+
+int byteswap_varlength_ftable (Header *header, FTable *table) {
+  int i, Nfields;
+  if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE;
+  for (i = 0; i < Nfields; i++) {
+    if (!gfits_byteswap_varlength_column (table, i+1)) return FALSE;
+  }
+  return TRUE;
 }
 
@@ -124,7 +133,12 @@
     fprintf (stderr, "compress %s\n", catalog->filename);
     char *compressMode = dvo_catalog_compress_string (catalog->catcompress);
-    if (!gfits_compress_table (ftable, &cmptable, 0, compressMode)) {
+    // I should test how Ntile affects fpack/funpack and speed
+    if (!gfits_compress_table (ftable, &cmptable, 1000, compressMode)) {
       fprintf (stderr, "compression failure\n");
       return (FALSE);
+    }
+    if (!byteswap_varlength_ftable (&cmpheader, &cmptable)) {
+      fprintf (stderr, "failed to swap varlength column\n");
+      return FALSE;
     }
     if (!gfits_modify (cmptable.header, "DVO_CMP", "%s", 1, compressMode)) {
@@ -140,5 +154,5 @@
   if (fullWrite) {
     if (!gfits_fwrite_Theader (catalog->f, outtable->header)) {
-      fprintf (stderr, "can't write table data\n");
+      fprintf (stderr, "can't write table header\n");
       return (FALSE);
     }
@@ -149,5 +163,5 @@
   } else {
     if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) {
-      fprintf (stderr, "can't write table data\n");
+      fprintf (stderr, "can't write table data (range)\n");
       return (FALSE);
     }
@@ -257,5 +271,9 @@
     rawtable.header = &rawheader;
 
-    // NOTE: uncompress swaps the bytes into native order
+    // NOTE: uncompress swaps the data bytes into native order, but needs the varlength columns native
+    if (!byteswap_varlength_ftable (ftable->header, ftable)) {
+      fprintf (stderr, "failed to swap varlength column\n");
+      return FALSE;
+    }
     if (!gfits_uncompress_table (ftable, &rawtable)) {
       if (VERBOSE) fprintf (stderr, "failed to uncompress table\n");
@@ -819,4 +837,5 @@
     int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress);
 
+    // convert to external table format
     if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
@@ -855,6 +874,9 @@
     assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
+    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -878,6 +900,9 @@
     }
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nmissing_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
+    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -907,6 +932,12 @@
     // XXX check these for consistency...
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
+
     // convert to external table format
-    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
+    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative)) {
+      fprintf (stderr, "trouble converting format\n");
+      goto failure;
+    }
 
     // write out SecFilt table
@@ -931,6 +962,9 @@
     assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off);
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
+    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -958,6 +992,9 @@
     assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off);
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
+    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -985,6 +1022,9 @@
     assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off);
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
+    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -1012,6 +1052,9 @@
     assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off);
 
+    // if we are going to compress, we need to receive unswapped data -- 
+    int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress);
+
     // convert to external table format
-    if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
+    if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -1152,9 +1195,6 @@
     assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
 
-    // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
-
     // convert to external table format (note that the block above does not damage or free catalog.average) 
-    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
+    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -1178,9 +1218,6 @@
     }
 
-    // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (0, catalog->Nmissing, catalog->Nmissing, catalog->catcompress);
-
     // convert to external table format
-    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
+    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
       fprintf (stderr, "trouble converting format\n");
       goto failure;
@@ -1210,9 +1247,9 @@
     // XXX check these for consistency...
 
-    // if we are going to compress, we need to receive unswapped data -- 
-    int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
-
     // convert to external table format
-    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative);
+    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
+      fprintf (stderr, "trouble converting format\n");
+      goto failure;
+    }
 
     // write out SecFilt table
@@ -1238,9 +1275,6 @@
 
     if (catalog[0].Nlensing) {
-      // if we are going to compress, we need to receive unswapped data -- 
-      int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
-
       // convert to external table format (note that the block above does not damage or free catalog.average) 
-      if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
+      if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
 	fprintf (stderr, "trouble converting format\n");
 	goto failure;
@@ -1270,9 +1304,6 @@
 
     if (catalog[0].Nlensobj) {
-      // if we are going to compress, we need to receive unswapped data -- 
-      int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
-
       // convert to external table format (note that the block above does not damage or free catalog.average) 
-      if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
+      if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
 	fprintf (stderr, "trouble converting format\n");
 	goto failure;
@@ -1302,9 +1333,6 @@
 
     if (catalog[0].Nstarpar) {
-      // if we are going to compress, we need to receive unswapped data -- 
-      int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
-
       // convert to external table format (note that the block above does not damage or free catalog.average) 
-      if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
+      if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
 	fprintf (stderr, "trouble converting format\n");
 	goto failure;
@@ -1334,9 +1362,6 @@
 
     if (catalog[0].Ngalphot) {
-      // if we are going to compress, we need to receive unswapped data -- 
-      int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress);
-
       // convert to external table format (note that the block above does not damage or free catalog.average) 
-      if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) {
+      if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
 	fprintf (stderr, "trouble converting format\n");
 	goto failure;
@@ -1566,5 +1591,8 @@
 
     // convert to external table format
-    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
+    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
+      fprintf (stderr, "trouble converting format\n");
+      goto failure;
+    }
 
     // write out SecFilt table