Changeset 38418

Timestamp:

Jun 6, 2015, 8:40:19 PM (11 years ago)

Author:

eugene

Message:

trying to improve the speed of the compression shuffle step; added NONE_1 for testing; added timing traces in uncompress to test speed improvements

Location:

branches/eam_branches/ohana.20150429/src/libfits

Files:

• Makefile (modified) (1 diff)
• matrix/F_compress_M.c (modified) (1 diff)
• matrix/F_compress_data.c (modified) (1 diff)
• matrix/F_uncompress_M.c (modified) (1 diff)
• matrix/F_uncompress_data.c (modified) (1 diff)
• table/F_compress_T.c (modified) (2 diffs)
• table/F_uncompress_T.c (modified) (2 diffs)
• test/compress.sh (modified) (4 diffs)
• test/imagecomp.c (modified) (1 diff)
• test/tablecomp.c (modified) (1 diff)

branches/eam_branches/ohana.20150429/src/libfits/Makefile

@@ -32 +32 @@
         for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS); done
 test.verbose: $(TESTPROG)
-        for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 |& $(TESTHARNESS) -v; done
+        for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS) -v; done
 
 install: $(DESTLIB)/libFITS.a $(DESTLIB)/libFITS.$(DLLTYPE) $(DESTMAN)/fits.1

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_M.c

@@ -514 +514 @@
         }
         *rawptr = *srcptr; 
-        myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun");
-        myAssert (rawptr - raw < Nraw*size, "memory overrun");
+        // myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun");
+        // myAssert (rawptr - raw < Nraw*size, "memory overrun");
       }
 

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_data.c

@@ -42 +42 @@
 
   // do not actually compress : this is used for testing
-  if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_2")) {
+  if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1") || !strcasecmp(cmptype, "NONE_2")) {
     unsigned long Nbytes = Nrawpix * rawpix_size;
 

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c

@@ -569 +569 @@
         }
         *srcptr = *rawptr; 
-        myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun");
-        myAssert (rawptr - raw < Nraw*size, "memory overrun");
+        // myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun");
+        // myAssert (rawptr - raw < Nraw*size, "memory overrun");
       }
 

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_data.c

@@ -33 +33 @@
 
   // do not actually uncompress : this is used for testing
-  if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_2")) {
+  if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1") || !strcasecmp(cmptype, "NONE_2")) {
     memcpy (outdata, zdata, Nzdata);
     myAssert (*Nout == Nzdata, "invalid size");

branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c

@@ -289 +289 @@
   off_t i;
 
+  // we are copying NN rows into the column which starts at XX and has MM bytes per row
+
+  off_t Nx = table->header->Naxis[0];
+
+  char *tblbuffer = &table->buffer[Nx*row_start + field->offset];
+
+  int rowsize = field->rowsize;
+
+  if (VERBOSE) fprintf (stderr, "distribute: ");
+  for (i = 0; i < Nrows; i++, tblbuffer += Nx) {
+    memcpy (&raw[i*rowsize], tblbuffer, rowsize);
+# if (VERBOSE)
+    int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]);
+# endif
+  }
+  if (VERBOSE) fprintf (stderr, "\n");
+
+  return (TRUE);
+}
+
+int gfits_collect_table_data_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
+
+  off_t i;
+
   // we are copying NN rows of the column which starts at XX and has MM bytes per row
 
@@ -307 +331 @@
 
 int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
+
+  off_t i, j, k;
+
+  // we are copying NN rows into the column which starts at XX and has MM bytes per row
+
+  off_t Nx = table->header->Naxis[0];
+  int Nvalues = field->Nvalues;
+  int pixsize = field->pixsize;
+  int offset = field->offset;
+
+  char *rawptr = raw;
+
+  for (k = 0; k < field->pixsize; k++) {
+# ifdef BYTE_SWAP      
+    char *tblptr_start = &table->buffer[Nx*row_start + offset + (pixsize - k - 1)];
+# else
+    char *tblptr_start = &table->buffer[Nx*row_start + offset + k];
+# endif
+    for (i = 0; i < Nrows; i++, tblptr_start += Nx) {
+      char *tblptr = tblptr_start;
+      for (j = 0; j < Nvalues; j++, tblptr += pixsize, rawptr++) {
+        *rawptr = *tblptr;
+      }
+    }
+  }
+  return (TRUE);
+}
+
+int gfits_collect_table_gzp2_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
 
   off_t i, j, k;

branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c

@@ -60 +60 @@
   fprintf (stderr, "times: %f %f %f\n", timeSum1, timeSum2, timeSum3);
   fprintf (stderr, "times: %f %f %f %f %f %f\n", timeSum2a, timeSum2b, timeSum2c, timeSum2d, timeSum2e, timeSum2f);
+
+  timeSum1 = 0.0;
+  timeSum2 = 0.0;
+  timeSum2a = 0.0;
+  timeSum2b = 0.0;
+  timeSum2c = 0.0;
+  timeSum2d = 0.0;
+  timeSum2e = 0.0;
+  timeSum2f = 0.0;
+  timeSum3 = 0.0;
 }
 
@@ -346 +356 @@
   char *tblbuffer = &table->buffer[Nx*row_start + field->offset];
 
+  int rowsize = field->rowsize;
+
   if (VERBOSE) fprintf (stderr, "distribute: ");
   for (i = 0; i < Nrows; i++, tblbuffer += Nx) {
-    // int row = row_start + i;
-    memcpy (tblbuffer, &raw[i*field->rowsize], field->rowsize);
+    memcpy (tblbuffer, &raw[i*rowsize], rowsize);
 # if (VERBOSE)
     int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]);

branches/eam_branches/ohana.20150429/src/libfits/test/compress.sh

@@ -5 +5 @@
   tapPLAN 88
 
-  foreach mode NONE GZIP_1 GZIP_2 RICE_1 RICE_ONE
+  foreach mode NONE NONE_1 NONE_2 GZIP_1 GZIP_2 RICE_1 RICE_ONE
   # foreach mode GZIP_1
   # foreach mode GZIP_2
@@ -44 +44 @@
 
  if ($cmpmode == NONE) return
+ if ($cmpmode == NONE_1) return
+ if ($cmpmode == NONE_2) return
 
  if ($cmpmode == GZIP_1)
@@ -311 +313 @@
   local format cmpmode
 
-  foreach cmpmode NONE GZIP_1 GZIP_2 RICE_1 RICE_ONE
+  foreach cmpmode NONE NONE_1 NONE_2 GZIP_1 GZIP_2 RICE_1 RICE_ONE
     test_table_rnd $cmpmode
   end
@@ -417 +419 @@
 
  if ($cmpmode == NONE) return; # funpack will not recognize NONE
+ if ($cmpmode == NONE_1) return; # funpack will not recognize NONE
+ if ($cmpmode == NONE_2) return; # funpack will not recognize NONE
  if ($cmpmode == RICE_ONE) return; # funpack will not recognize RICE_ONE for tables
 

branches/eam_branches/ohana.20150429/src/libfits/test/imagecomp.c

@@ -8 +8 @@
 
 // char *cmptype[] = {"NONE", "NONE_2", "GZIP_1", "GZIP_2", "PLIO_1", "RICE_1", "RICE_ONE", "HCOMPRESS_1", NULL};
-char *cmptype[] = {"NONE", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", NULL};
+char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", NULL};
 int bitpix[] = {8, 16, 32, -32, -64, 0};
 

branches/eam_branches/ohana.20150429/src/libfits/test/tablecomp.c

@@ -8 +8 @@
 int test_compress_single_full (char *zcmptype);
 
-char *cmptype[] = {"NONE", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", "AUTO", NULL};
+char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", "AUTO", NULL};
 
 int main (int argc, char **argv) {