Changeset 38342

Timestamp:

May 31, 2015, 8:36:59 AM (11 years ago)

Author:

eugene

Message:

adding compressed table I/O tests

Location:

branches/eam_branches/ohana.20150429/src/libfits

Files:

• extern/gzip.c (modified) (1 diff)
• table/F_compress_T.c (modified) (2 diffs)
• table/F_table_varlength.c (modified) (1 diff)
• table/F_uncompress_T.c (modified) (2 diffs)
• test/compress.sh (modified) (4 diffs)

branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c

@@ -2 +2 @@
 # include <gfitsio.h>
 # include <zlib.h>
-# define EXTRA_VERBOSE 1
+# define EXTRA_VERBOSE 0
 
 /** the two functions in this file re-implement the uncompress function of zlib.  

branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c

@@ -178 +178 @@
 }
 
+# define VERBOSE 0
+
 // raw_pixsize is the bytes / pixel for the input matrix
 // place the raw image bytes for the current tile into the tile buffer
@@ -188 +190 @@
   off_t Nx = table->header->Naxis[0];
 
+  if (VERBOSE) fprintf (stderr, "collect: ");
   for (i = 0; i < Nrows; i++) {
     int row = row_start + i;
     memcpy (&raw[i*rowsize], &table->buffer[Nx*row + col], rowsize);
-  }
+# if (VERBOSE)
+    int j; for (j = 0; j < rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + col + j]);
+# endif
+  }
+  if (VERBOSE) fprintf (stderr, "\n");
 
   return (TRUE);

branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c

@@ -141 +141 @@
   }
 
+  // fprintf (stderr, "length: %d, offset: %d\n", (int) *length, (int) offset);
+
   result = (void *) (ftable->buffer + ftable->heap_start + offset);
   return result;

branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c

@@ -146 +146 @@
 }
 
+# define VERBOSE 0
+
 // raw_pixsize is the bytes / pixel for the input matrix
 // place the raw image bytes for the current tile into the tile buffer
@@ -156 +158 @@
   off_t Nx = table->header->Naxis[0];
 
+  if (VERBOSE) fprintf (stderr, "collect: ");
   for (i = 0; i < Nrows; i++) {
     int row = row_start + i;
     memcpy (&table->buffer[Nx*row + col], &raw[i*rowsize], rowsize);
+# if (VERBOSE)
+    int j; for (j = 0; j < rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + col + j]);
+# endif
   }
+  if (VERBOSE) fprintf (stderr, "\n");
 
   return (TRUE);

branches/eam_branches/ohana.20150429/src/libfits/test/compress.sh

@@ -1 +1 @@
 
-macro testall
-
-  foreach bitpix 8 16 32 -32 -64
-    echo ===== bitpix = $bitpix =====
-    test1 $bitpix
-  end
-end
-
-macro test1
- if ($0 != 2)
-   echo "USAGE: test_image (bitpix)"
-   break
- end
+macro test_image_all 
+
+  foreach mode NONE GZIP_1
+    foreach bitpix 8 16 32 -32 -64
+      echo ===== bitpix = $bitpix =====
+      test_image_single $bitpix $mode
+    end
+  end
+end
+
+macro test_image_single
+ if ($0 != 3)
+   echo "USAGE: test_image_single (bitpix) (mode)"
+   break
+ end
+
+ local bitpix cmpmode
+ $bitpix = $1
+ $cmpmode = $2
 
  mcreate z 8 8
  set x = xramp(z)
- wd x test.raw.fits -bitpix $1 -bzero 0 -bscale 1
- wd x test.cmp.fits -bitpix $1 -bzero 0 -bscale 1 -compress
+ wd x test.raw.fits -bitpix $bitpix -bzero 0 -bscale 1
+ wd x test.cmp.fits -bitpix $bitpix -bzero 0 -bscale 1 -compress-mode $cmpmode
  rd b test.cmp.fits -x 0
  set d = x - b
  stat d
+
+ if ($cmpmode == NONE) return
 
  if ($bitpix == -64)
@@ -42 +50 @@
 end
 
-macro test2
- mcreate z 500 500
- set x = xramp(z)
- wd x test.raw.fits -bitpix -32 -bzero 0 -bscale 1
- wd x test.cmp.fits -bitpix -32 -bzero 0 -bscale 1 -compress -compress-mode NONE
- rd b test.cmp.fits -x 0
- set d = x - b
- stat d
- # exec funpack -S test.cmp.fits > test.fun.fits
-end
-
-macro test3
+macro test_table_single_mode
+ if ($0 != 2)
+   echo "USAGE: test_table_single_mode (mode)"
+   break
+ end
+
+ local cmpmode
+ $cmpmode = $1
+
+ delete -q x y ID
 
  $NPT = 10
@@ -60 +66 @@
  create ID 1 {$NPT + 1} -int
 
- write -fits test test.raw.tbl x y ID
- write -fits test test.cmp.tbl x y ID -compress-mode NONE
-
- foreach f x y ID
+ $fields = ID x y 
+ $format = IEE
+
+ write -fits test test.raw.tbl $fields -format $format
+ write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode
+
+ foreach f $fields
    set $f\_raw = $f
    delete $f
@@ -70 +79 @@
  echo "===== raw ====="
  data test.raw.tbl
- read -fits test x y ID
-
- foreach f x y ID
-   set dv = $f - $f\_raw
-   vstat dv
- end
- delete -q x y ID
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+ delete -q $fields
 
  echo "===== cmp ====="
  data test.cmp.tbl
- read -fits test x y ID
-
- foreach f x y ID
-   set dv = $f - $f\_raw
-   vstat dv
- end
-end
-
-macro test4
-
- create x 0 100
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+end
+
+macro test_table_all_single_column
+
+  local format cmpmode
+
+  foreach cmpmode NONE GZIP_1
+    foreach format B I J K E D
+      test_table_single_column $cmpmode $format
+    end
+  end
+end
+
+macro test_table_single_column
+ if ($0 != 3)
+   echo "USAGE: test_table_single_column (mode) (format)"
+   break
+ end
+
+ local cmpmode
+ $cmpmode = $1
+
+ delete -q x
+
+ $NPT = 10
+ create x 0 $NPT
+
+ local format
+
+ $fields = x 
+ $format = $2
+
+ write -fits test test.raw.tbl $fields -format $format
+ write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode
+
+ foreach f $fields
+   set $f\_raw = $f
+   delete $f
+ end
+
+ echo "===== $cmpmode $format raw ====="
+ data test.raw.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+ delete -q $fields
+
+ echo "===== $cmpmode $format cmp ====="
+ data test.cmp.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+end
+
+macro test_table_all_multi_column
+
+  local format cmpmode
+
+  foreach cmpmode NONE GZIP_1
+    foreach format B I J K E D
+      test_table_multi_column $cmpmode $format
+    end
+  end
+end
+
+macro test_table_multi_column
+ if ($0 != 3)
+   echo "USAGE: test_table_multi_column (mode) (format)"
+   break
+ end
+
+ local cmpmode
+ $cmpmode = $1
+
+ delete -q x y z 
+
+ $NPT = 10
+ create x 0 $NPT
  set y = x^2
- create ID 1 101 -int
-
- write -fits test test.raw.tbl x y ID
- write -fits test test.cmp.tbl x y ID -compress
-
- foreach f x y ID
-   set $f\_raw = $f
-   delete $f
- end
-
- echo "===== raw ====="
- data test.raw.tbl
- read -fits test x y ID
-
- foreach f x y ID
-   set dv = $f - $f\_raw
-   vstat dv
- end
- delete -q x y ID
-
- echo "===== cmp ====="
- data test.cmp.tbl
- read -fits test x y ID
-
- foreach f x y ID
-   set dv = $f - $f\_raw
-   vstat dv
- end
-end
+ set z = 0.01*x
+
+ local format
+
+ $fields = x y z
+ $format = $2\$2\$2
+
+ $fields = x y 
+ $format = $2\$2
+
+ write -fits test test.raw.tbl $fields -format $format
+ write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode
+
+ foreach f $fields
+   set $f\_raw = $f
+   delete $f
+ end
+
+ echo "===== $cmpmode $format raw ====="
+ data test.raw.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+ delete -q $fields
+
+ echo "===== $cmpmode $format cmp ====="
+ data test.cmp.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+end
+
+
+macro test_table_all_rnd
+
+  local format cmpmode
+
+  foreach cmpmode NONE GZIP_1
+    test_table_rnd $cmpmode
+  end
+end
+
+macro test_table_rnd
+ if ($0 != 2)
+   echo "USAGE: test_table_rnd (mode)"
+   break
+ end
+
+ local cmpmode
+ $cmpmode = $1
+
+ delete -q Achar Ashort Aint Along
+ delete -q Ffloat Fdouble
+
+ # lrnd generates rnd int between 0 and 0x7fffff (2^31 - 1 inclusive)
+
+ $NPT = 1000
+ create ni 0 $NPT -int
+ set Achar  = lrnd(ni) & 0x7f
+ set Ashort = lrnd(ni) & 0x7fff
+ set Aint   = lrnd(ni)
+ set Along  = lrnd(ni); # I cannot represent the full dynamic range of 64 bit with mana vector ints
+
+ create nf 0 $NPT
+ set Ffloat = drnd(nf)*1e20
+ set Fdouble = drnd(nf)*1e100
+
+ local format
+
+ $fields = Achar Ashort Aint Along Ffloat Fdouble
+ $format = BIJKED
+
+ write -fits test test.raw.tbl $fields -format $format
+ write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode
+
+ foreach f $fields
+   set $f\_raw = $f
+   delete $f
+ end
+
+ echo "===== $cmpmode $format raw ====="
+ data test.raw.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+ delete -q $fields
+
+ echo "===== $cmpmode $format cmp ====="
+ data test.cmp.tbl
+ read -fits test $fields
+
+ foreach f $fields
+   set dv = $f - $f\_raw
+   vstat dv
+ end
+end