Changeset 38334

Timestamp:

May 29, 2015, 5:24:53 PM (11 years ago)

Author:

eugene

Message:

table compression is internally consistent (tables can be generated and compressed)

Location:

branches/eam_branches/ohana.20150429/src/libfits

Files:

• Makefile (modified) (2 diffs)
• extern/gzip.c (modified) (5 diffs)
• matrix/F_compress_data.c (modified) (3 diffs)
• matrix/F_uncompress_M.c (modified) (1 diff)
• matrix/F_uncompress_data.c (modified) (3 diffs)
• table/F_compress_T.c (modified) (3 diffs)
• table/F_table_varlength.c (modified) (2 diffs)
• table/F_uncompress_T.c (modified) (1 diff)
• test/tablecomp.c (added)
• test/zlib.c (added)

branches/eam_branches/ohana.20150429/src/libfits/Makefile

@@ -13 +13 @@
 INC     =       $(HOME)/include
 MAN     =       $(HOME)/doc
+TEST    =       $(HOME)/test
+TESTBIN =       $(HOME)/test
 include ../../Makefile.Common
 
@@ -19 +21 @@
 FULL_CPPFLAGS = $(BASE_CPPFLAGS) -I$(EXT)
 FULL_LDFLAGS  = $(BASE_LDFLAGS) -lohana
+
+TEST_CFLAGS   = $(BASE_CFLAGS)
+TEST_CPPFLAGS = $(BASE_CPPFLAGS)
+TEST_LDFLAGS  = $(BASE_LDFLAGS) -lFITS -lohana -ltap_ohana
+
+TESTPROG = tablecomp zlib
+$(TESTPROG) : % : $(TESTBIN)/%
+test: $(TESTPROG)
 
 install: $(DESTLIB)/libFITS.a $(DESTLIB)/libFITS.$(DLLTYPE) $(DESTMAN)/fits.1

branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c

@@ -120 +120 @@
 {
     z_stream stream;
-    int err;
+    int i, err;
 
     stream.next_in = (Bytef*)source;
@@ -135 +135 @@
     stream.opaque = Z_NULL;
 
+    if (0) {
+      fprintf (stderr, "inp unc: ");
+      for (i = 0; i < sourceLen; i++) {
+        fprintf (stderr, "0x%02hhx ", source[i]);
+      }
+      fprintf (stderr, "\n");
+    }
+
     // err = inflateInit2(&stream, -15);
     err = inflateInit(&stream);
+    if (0) fprintf (stderr, "inp buffers unc 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
+
     if (err != Z_OK) return err;
 
     err = inflate(&stream, Z_FINISH);
+    if (0) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
     if (err != Z_STREAM_END) {
         inflateEnd(&stream);
-        if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
-            return Z_DATA_ERROR;
+        if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) return Z_DATA_ERROR;
         return err;
     }
+    if (0) fprintf (stderr, "out buffers unc 2: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
+
+    if (0) {
+      fprintf (stderr, "out unc: ");
+      for (i = 0; i < stream.total_out; i++) {
+        fprintf (stderr, "0x%02hhx ", dest[i]);
+      }
+      fprintf (stderr, "\n");
+    }
+
     assert (stream.total_out <= *destLen);
     *destLen = stream.total_out;
@@ -156 +176 @@
 {
   z_stream stream;
-  int err;
+  int i, err;
   
   stream.next_in = (Bytef*)source;
@@ -172 +192 @@
   stream.opaque = Z_NULL;
 
+  if (0) {
+    fprintf (stderr, "inp cmp: ");
+    for (i = 0; i < sourceLen; i++) {
+      fprintf (stderr, "0x%02hhx ", source[i]);
+    }
+    fprintf (stderr, "\n");
+  }
+
   // the '5' is the compression level: make this a user argument
   err = deflateInit(&stream, 5);
+  if (0) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
+
   if (err != Z_OK) return err;
 
@@ -179 +209 @@
   // the compression occur in a series of steps
   err = deflate(&stream, Z_FINISH);
+  if (0) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
+
   if (err != Z_STREAM_END) {
-    deflateEnd(&stream);
-    return err;
-  }
+    err = deflateEnd(&stream);
+    return Z_BUF_ERROR;
+  }
+  if (0) fprintf (stderr, "out buffers cmp 2: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out);
+
+  if (0) {
+    fprintf (stderr, "out cmp: ");
+    for (i = 0; i < stream.total_out; i++) {
+      fprintf (stderr, "0x%02hhx ", dest[i]);
+    }
+    fprintf (stderr, "\n");
+  }
+
   assert (stream.total_out <= *destLen);
   *destLen = stream.total_out;

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_data.c

@@ -27 +27 @@
 int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen);
 
-static int pass = 0;
-
 int gfits_compress_data (char *zdata, int *Nzdata, char *cmptype, 
                          char **optname, char **optvalue, int Nopt, 
@@ -34 +32 @@
 
   int status;
+
+  // do not actually compress : this is used for testing
+  if (!strcasecmp(cmptype, "NONE")) {
+    unsigned long Nbytes = Nrawpix * rawpix_size;
+
+    memcpy (zdata, rawdata, Nbytes);
+    *Nzdata = Nbytes;
+    return (TRUE);
+  }
 
   if (!strcasecmp(cmptype, "GZIP_1")) {
@@ -49 +56 @@
       fprintf (stderr, "error in compress (GZIP)\n");
       return (FALSE);
-    }
-
-    // test of deflate (vs I/O)
-    { 
-      char *srcbuf, *tgtbuf;
-      ALLOCATE (srcbuf, char, destLen);
-      memcpy (srcbuf, zdata, destLen);
-
-      ALLOCATE (tgtbuf, char, Nbytes);
-      
-      uLongf Ndeflate = Nbytes;
-      status = gfits_uncompress ((Bytef *) tgtbuf, &Ndeflate, (Bytef *) zdata, destLen);
-      if (status != Z_OK) {
-        fprintf (stderr, "error in uncompress (GZIP) : %d vs %d\n", status, Z_OK);
-        return (FALSE);
-      }
-      
-      myAssert (Ndeflate == Nbytes, "invalid output size?");
-
-      int i; 
-      for (i = 0; i < Ndeflate; i++) {
-        myAssert (rawdata[i] == tgtbuf[i], "data mismatch?");
-      }
-
-    }
-
-    if (pass == 0) {
-      int i;
-      for (i = 0; i < 32; i++) {
-        fprintf (stderr, "0x%02hhx ", zdata[i]);
-      }
-      fprintf (stderr, "\n");
-      pass = 1;
     }
     return (TRUE);

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c

@@ -197 +197 @@
 
   // find the COMPRESSED_DATA column (format should be 1PB, 1PI, 1PJ)
+  // is it required that this be the only column?
   for (i = 1; TRUE; i++) {
     snprintf (key, 10, "TTYPE%d", i);

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_data.c

@@ -26 +26 @@
 int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen);
 
-static int pass = 0;
-
 int gfits_uncompress_data (char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize) {
 
@@ -33 +31 @@
   static int Ninsum = 0;
   static int Noutsum = 0;
+
+  // do not actually uncompress : this is used for testing
+  if (!strcasecmp(cmptype, "NONE")) {
+    memcpy (outdata, zdata, Nzdata);
+    *Nout = Nzdata / out_pixsize;
+    return (TRUE);
+  }
 
   if (!strcasecmp(cmptype, "GZIP_1")) {
@@ -40 +45 @@
     // XXX maybe not anymore
     // gfits_gz_stripheader ((unsigned char *)zdata, &Nzdata);
-
-    if (pass == 0) {
-      int i;
-      for (i = 0; i < 32; i++) {
-        fprintf (stderr, "0x%02hhx ", zdata[i]);
-      }
-      fprintf (stderr, "\n");
-      pass = 1;
-    }
 
     // uncompress does not require us to know the expected number of pixel; it tells us the number

branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c

@@ -56 +56 @@
 
   TableField *fields = NULL;
-  ALLOCATE (fields, TableField, Nfields);
+  ALLOCATE_ZERO (fields, TableField, Nfields);
   for (i = 0; i < Nfields; i++) {
     snprintf (keyword, 80, "TTYPE%d", i+1);
@@ -129 +129 @@
 
     fields[i].offset = offset;
-    offset += fields[i].offset;
+    offset += fields[i].rowsize;
   }
 
   // allocate the intermediate storage buffers
+  int Nzdata_alloc = max_width*ztilelen + 100;
   ALLOCATE (raw,   char, max_width*ztilelen);
-  ALLOCATE (zdata, char, max_width*ztilelen);
+  ALLOCATE (zdata, char, Nzdata_alloc);
 
   // compress the data : copy into a tile, compress the tile, then add to the output table
@@ -153 +154 @@
       
       int Nraw = Nrows*fields[j].Nvalues; // number of pixels
-      int Nzdata = max_width*ztilelen; // available space, replaced with actual output size on compression
+      int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression
       if (!gfits_compress_data (zdata, &Nzdata, fields[j].zctype, NULL, NULL, 0, raw, Nraw, fields[j].pixsize)) ESCAPE(A);
       

branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c

@@ -128 +128 @@
   Nx = ftable->header->Naxis[0];
 
-# ifdef BYTE_SWAP
-  char *pchar, tmp;
-  pchar = &ftable->buffer[row*Nx + column->offset];
-  SWAP_WORD;
-  pchar = &ftable->buffer[row*Nx + column->offset + 4];
-  SWAP_WORD;
-# endif
-
   if (column->mode == 'P') {
     int *ptr;
@@ -156 +148 @@
 // for varlength columns to the heap.  To start, assume we have defined the cartesian
 // portion of the table correctly, and are only extending the heap.
-
-// XXX NOTE: this function assumes the table data is currently in local byte swap order.
-// swap before writing out
 
 int gfits_varlength_column_add_data (FTable *table, char *data, off_t Ndata, int row, VarLengthColumn *column) {

branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c

@@ -68 +68 @@
 
     fields[i].offset = offset;
-    offset += fields[i].offset;
+    offset += fields[i].rowsize;
 
     if (!gfits_varlength_column_define (srctable, &fields[i].zdef, i + 1)) ESCAPE(A);