Changeset 38322

Timestamp:

May 25, 2015, 8:08:57 PM (11 years ago)

Author:

eugene

Message:

adding compression of images in prep for compression / uncompression of tables

Location:

branches/eam_branches/ohana.20150429/src/libfits

Files:

• doc/notes-compress.txt (modified) (1 diff)
• extern/gzip.c (modified) (1 diff)
• matrix/F_compress_data.c (added)
• matrix/F_compress_utils.c (added)
• matrix/F_uncompress_M.c (moved) (moved from branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_M.c ) (3 diffs)

branches/eam_branches/ohana.20150429/src/libfits/doc/notes-compress.txt

@@ +1 @@
+
+20150525 : 
+
+uncompression steps:
+
+ * read in a table: gfits_fread_ftable_data (f, &ftable, FALSE); (no byte swap)
+ * uncompress the image: gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable);
+   * get ZIMAGE from heaer & delete
+   * get ZCMPTYPE 
+   * get ZBITPIX, ZNAXIS, ZNAXISnn
+   * get ZTILEn (if missing ZTILE1 = Nx, others 1)
+   * get ZNAME, ZVAL
+   * ZSIMPLE, ZTENSION, ZEXTEND, ZBLOCKED, ZPCOUNT, ZGCOUNT, etc
+   * ZSCALE, ZBLANK, OBLANK, ZZERO
+
+   * data is stored in fits table COMPRESSED_DATA (1PB, 1PI, 1PJ)
+
+------------------------------------------------
 
 TFORMn : rPt(e_max)

branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c

@@ -151 +151 @@
     return err;
 }
+
+int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)
+{
+  z_stream stream;
+  int err;
+
+  stream.next_in = (Bytef*)source;
+  stream.avail_in = (uInt)sourceLen;
+
+  /* Check for source > 64K on 16-bit machine: */
+  if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR;
+
+  stream.next_out = dest;
+  stream.avail_out = (uInt)*destLen; // allocated space
+  if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
+
+  stream.zalloc = (alloc_func)0;
+  stream.zfree = (free_func)0;
+
+  err = deflateInit(&stream);
+  if (err != Z_OK) return err;
+
+  // XXX this is written to do the compression in a single pass.  it could be re-done to have 
+  // the compression occur in a series of steps
+  err = deflate(&stream, Z_FINISH);
+  if (err != Z_STREAM_END) {
+    deflateEnd(&stream);
+    return err;
+  }
+  assert (stream.total_out <= *destLen);
+  *destLen = stream.total_out;
+  
+  err = deflateEnd(&stream);
+  return err;
+}

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c

@@ -221 +221 @@
     ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]);
     max_tile_size *= ztile[i];
+
+    // ztile[i] is the default (or max) tile size in the i-th dimension
+    // ntile[i] is the number of tiles in the i-th dimension
+    // otile[i] is the current output tile counter in the i-th dimension
   }
 
@@ -282 +286 @@
   }
   return (TRUE);
-}
-
-// gfits_varlength returns a pointer to a chunk of Nzdata of data elements starting at zdata.
-
-// XXX need to have an API to send the user data
-
-// XXX need to byte-swap the table column; this needs to be worked out more clearly
-// in the APIs: do the table read / column extract functions swap or not?.  we have
-// put the byte-swapping in gifts_varlength_column_pointer, but this is fairly weak.
-
-// XXX inconsistency between data element size between compressed data, uncompressed data, 
-// and output pixel data.....
-
-// true sizes of this tile (in pixels)
-off_t gfits_tile_size (Matrix *matrix, int *otile, int *ztile) {
-
-  off_t i, Npixels, Ndimen;
-
-  Npixels = 1;
-  for (i = 0; i < matrix->Naxes; i++) {
-    Ndimen = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]);
-    Npixels *= Ndimen;
-  }
-  
-  return (Npixels);
 }
 
@@ -424 +403 @@
 }
 
-int gfits_byteswap_zdata (char *zdata, int Nzdata, int pixsize) {
-
-# define DOSWAP(A,B) { char tmp = A; A = B; B = tmp; }
-
-# ifdef BYTE_SWAP
-
-  int i;
-
-  switch (pixsize) {
-    case 1:
-      break;
-
-    case 2:
-      for (i = 0; i < 2*Nzdata; i+=2) {
-        DOSWAP (zdata[i+0], zdata[i+1]);
-      }
-      break;
-
-    case 4:
-      for (i = 0; i < 4*Nzdata; i+=4) {
-        DOSWAP (zdata[i+0], zdata[i+3]);
-        DOSWAP (zdata[i+1], zdata[i+2]);
-      }
-      break;
-
-    case 8:
-      for (i = 0; i < 8*Nzdata; i+=8) {
-        DOSWAP (zdata[i+0], zdata[i+7]);
-        DOSWAP (zdata[i+1], zdata[i+6]);
-        DOSWAP (zdata[i+2], zdata[i+5]);
-        DOSWAP (zdata[i+3], zdata[i+4]);
-      }
-      break;
-  }
-# endif
-  return (TRUE);
-}
-
-int gfits_extension_is_compressed (Header *header) {
-
-    int has_extension, has_extname, has_zimage, zimage;
-    char extname[80], extension[80];
-
-    has_extension = gfits_scan (header, "XTENSION", "%s", 1, extension);
-    has_extname = gfits_scan (header, "EXTNAME", "%s", 1, extname);
-    has_zimage  = gfits_scan_alt (header, "ZIMAGE", "%t", 1, &zimage);
-
-    if (has_extension && !strcmp (extension, "IMAGE")) return (FALSE);
-    if (has_zimage && zimage) return (TRUE);
-    if (has_extname) {
-        if (!strcmp (extname, "COMPRESSED_IMAGE")) return (TRUE);
-    }
-
-    return (FALSE);
-}
-
-int gfits_compressed_is_primary (Header *header) {
-
-  int zimage = FALSE;
-  // int ztension = FALSE;
-
-    int has_zimage   = gfits_scan_alt (header, "ZIMAGE",   "%t", 1, &zimage);
-    // int has_ztension = gfits_scan_alt (header, "ZTENSION", "%t", 1, &ztension);
-
-    if (has_zimage && zimage) return (TRUE);
-
-    return (FALSE);
-}
-
-int gfits_uncompressed_data_pixsize (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) {
-
-  int i, Nbyte;
-
-  if (!strcasecmp(cmptype, "GZIP_1")) {
-    if (out_bitpix == 8)  return (1);
-    if (out_bitpix == 16) return (2);
-    if (out_bitpix == 32) return (4);
-    if (out_bitpix == -32) return (4);
-    if (out_bitpix == -64) return (8);
-    return (1);
-  }
-  if (!strcasecmp(cmptype, "RICE_1")) {
-
-    // if BYTEPIX option is specified, use that for Nbyte
-    for (i = 0; i < Noptions; i++) {
-      if (!strcmp(optname[i], "BYTEPIX")) {
-        Nbyte = atoi (optvalue[i]);
-        return (Nbyte);
-      }
-    }
-
-    return (4);
-  }
-  if (!strcasecmp(cmptype, "PLIO_1")) {
-    return (4);
-  }
-  if (!strcasecmp(cmptype, "HCOMPRESS_1")) {
-    if (out_bitpix == 8)  return (4);
-    if (out_bitpix == 16) return (4);
-    return (8);
-  }
-  return (0);
-}
-
-int gfits_vartable_heap_pixsize (char format) {
-
-  if (format == 'B') {
-      return (1);
-  }
-  if (format == 'I') {
-      return (2);
-  }
-  if (format == 'J') {
-      return (4);
-  }
-  fprintf (stderr, "invalid size for compressed data: %c\n", format);  
-  abort ();
-}