Changeset 38326

Timestamp:

May 26, 2015, 4:29:20 PM (11 years ago)

Author:

eugene

Message:

updating compression functions

Location:

branches/eam_branches/ohana.20150429/src/libfits

Files:

• matrix/F_compress_M.c (modified) (5 diffs)
• matrix/F_compress_utils.c (modified) (1 diff)
• table/F_table_format.c (modified) (2 diffs)
• table/F_table_varlength.c (modified) (2 diffs)

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_M.c

@@ -26 +26 @@
 // zcmptype
 
-int gfits_compress_image (Header *header, Matrix *matrix, FTable *ftable) {
-
-  off_t Nzdata, Nzrows, zcol;
-  int i, j, status, zimage, Nout, max_tile_size;
-  char cmptype[80];
-  char zaxis[10], naxis[10], key[10], word[81], exttype[81], checksum[81], datasum[81];
-  int Noptions, NOPTIONS, zblank, oblank;
-  VarLengthColumn zdef;
-  float zscale, zzero;
-
-  int zdata_pixsize, odata_pixsize;
-
-  int *ztile = NULL;
-  int *otile = NULL;
-  int *ntile = NULL;
-  char **optname = NULL;
-  char **optvalue = NULL;
-  char *out = NULL;
-  char *zdata = NULL;
-
-  Noptions = 0;
-
-  // is ZIMAGE present?
-  // NOTE target of %t must be int length
-  status = gfits_scan_alt (ftable->header, "ZIMAGE", "%t", 1, &zimage);
-  if (!status || !zimage) ESCAPE;
-
+// steps to convert an image to a compressed table:
+// * determine the number of tiles (from ztile[] and image size)
+// * construct an empty table with the right dimensions (1 column, Ntile rows)
+// * loop over tiles
+// * extract tile data to a buffer
+// * compress the buffer data
+// * insert in table heap
+// * update header keywords
+
+// XXX do not try to support compression of an image with an associated table (not valid)
+
+int gfits_compress_image (Header *header, Matrix *matrix, FTable *ftable, int *ztile, char *zcmptype) {
+
+  // off_t Nzdata, Nzrows, zcol;
+  // int i, j, status, zimage, Nout, max_tile_size;
+  // char cmptype[80];
+  // char zaxis[10], naxis[10], key[10], word[81], exttype[81], checksum[81], datasum[81];
+  // int Noptions, NOPTIONS, zblank, oblank;
+  // VarLengthColumn zdef;
+  // float zscale, zzero;
+
+  // int zdata_pixsize, odata_pixsize;
+  // int *otile = NULL;
+  // int *ntile = NULL;
+  // char **optname = NULL;
+  // char **optvalue = NULL;
+  // char *out = NULL;
+  // char *zdata = NULL;
+
+  // determine the number of tiles (from ztile[] and image size)
+  if (!ztile) {
+    ALLOCATE (ztile, int, matrix->Naxes);
+    ztile[0] = matrix->Naxis[0];
+    for (i = 1; i < matrix->Naxes; i++) ztile[i] = 1;
+  }
+  ALLOCATE (ntile, int, matrix->Naxes);
+  ALLOCATE (otile, int, matrix->Naxes);
+
+  int Ntile = gfits_imtile_count (matrix, ztile, ntile);
+
+  // creates an empty (Naxes = 2, Naxis[i] = 0) table header with XTENSION = BINTABLE, EXTNAME = COMPRESSED_IMAGE
+  gfits_create_table_header (&theader, "BINTABLE", "COMPRESSED_IMAGE");
+
+  // by using "P" format, we are limited to 32bit pointers (2GB heap)
+  // XXX how is the "B" format used?
+  gfits_define_bintable_column (&theader, "1PB(0)", "COMPRESSED_DATA", "compressed image data", "none", 1.0, 0.0);
+
+  // XXX I need to copy the bulk of the header from the matrix header -> ftable.header
+  gfits_create_table (&theader, ftable);
   // copy original header to output header
-  gfits_copy_header (header, theader);
+  // XXX gfits_copy_header (header, theader); -- I need to copy only the desired keywords.
+  
+  // this allocates the data array 
+  char *tmpbuffer = NULL;
+  ALLOCATE_ZERO (tmpbuffer, char, Ntile);
+  gfits_set_bintable_column (&theader, &ftable, "COMPRESSED_DATA", tmpbuffer, Ntile);
 
   // add ZIMAGE from output header
@@ -60 +87 @@
 
   // translate image-specific keywords to Z versions (updates the Z version, removes the old version)
-  MOD_KEYWORD ("ZBITPIX", "BITPIX", "%d", header->bitpix);
-  MOD_KEYWORD ("ZNAXIS",  "NAXIS",  "%d", header->Naxes);
+  // XXX use the header or the structure as authoratative?
+  MOVE_KEYWORD ("ZBITPIX", "BITPIX", "%d");
+  MOVE_KEYWORD ("ZNAXIS",  "NAXIS",  "%d");
 
   for (i = 0; i < header->Naxes; i++) {
@@ -149 +177 @@
   MOD_KEYWORD ("ZHECKSUM", "CHECKSUM", "%s", checksum,        checksum);
   MOD_KEYWORD ("ZDATASUM", "DATASUM",  "%s", datasum,         datasum);
-# endif
 
   zscale = 1;
@@ -162 +189 @@
   zzero = 0;
   gfits_scan (header, "ZZERO", "%f", 1, &zzero);
-
-  // XXX do not try to support compression of an image with an associated table (not valid)
-
-  // create a table with a column called COMPRESSED_DATA 
-  // output format may be 1PB, 1PI, 1PJ
-
-  // find the COMPRESSED_DATA column (format should be 1PB, 1PI, 1PJ)
-  for (i = 1; TRUE; i++) {
-    snprintf (key, 10, "TTYPE%d", i);
-    if (!gfits_scan (ftable->header, key, "%s", 1, word)) ESCAPE;
-    if (!strcmp (word, "COMPRESSED_DATA")) break;
-  }
-  zcol = i;
-
+# endif
+
+  // XXX how to define zdef?
+  VarLengthColumn zdef;
   if (!gfits_varlength_column_define (ftable, &zdef, zcol)) ESCAPE;
-  gfits_delete (header, "TFIELDS", 1);
-  snprintf (key, 10, "TTYPE"OFF_T_FMT,  zcol);
-  gfits_delete (header, key, 1);
-  snprintf (key, 10, "TFORM"OFF_T_FMT,  zcol);
-  gfits_delete (header, key, 1);
-
-  // counters for tile numbers
-  ALLOCATE (otile, int, matrix->Naxes);
-  ALLOCATE (ntile, int, matrix->Naxes);
-  max_tile_size = 1;
-  for (i = 0; i < matrix->Naxes; i++) {
-    otile[i] = 0;
-    ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]);
-    max_tile_size *= ztile[i];
-
-    // ztile[i] is the default (or max) tile size in the i-th dimension
-    // ntile[i] is the number of tiles in the i-th dimension
-    // otile[i] is the current output tile counter in the i-th dimension
-  }
+
+  max_tile_size = gfits_imtile_maxsize (matrix, ztile);
+
+  // size of an element in the vartable heap section
+  zdata_pixsize = gfits_vartable_heap_pixsize (zdef.format);
+
+  // size of a pixel in the final image (not needed)
+  // raw_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions)
+
+  // size of a pixel in the raw image tile:
+  raw_pixsize = abs(header[0].bitpix) / 8;
+
+  ALLOCATE (raw,   char, raw_pixsize*max_tile_size);
+  ALLOCATE (zdata, char, raw_pixsize*max_tile_size); // 
+
+  // init the otile[] counters
+  gfits_imfile_start (matrix, otile);
+
+  // compress the data : copy into a tile, compress the tile, then add to the output table
+  // each tile -> 1 row of the output table
+  for (i = 0; i < Ntile; i++) {
+
+    // size of the current tile
+    Nraw = gfits_tile_size (matrix, otile, ztile);
+
+    // copy the raw pixels from their native matrix locations to the temporary output buffer
+    if (!gfits_collect_data (matrix, raw, Nraw, raw_pixsize, otile, oblank, ztile, zblank, zscale, zzero)) return (FALSE);
+
+    // XXX the tile must not be > 2GB
+
+    Nzdata = raw_pixsize*max_tile_size; // available space, replaced with actual output size on compression
+    if (!gfits_compress_data (zdata, &Nzdata, zcmptype, optname, optvalue, Noptions, raw, Nraw, raw_pixsize)) return (FALSE);
+
+    if (!gfits_varlength_column_add_data (ftable, zdata, Nzdata, i, &zdef)) return FALSE;
+
+    // XXX need to activate
+    if (FALSE) gfits_byteswap_zdata (zdata, Nzdata, zdata_pixsize) return (FALSE);
+
+    // update the otile[] counters, carrying to the next dimension if needed
+    gfits_imfile_next (matrix, ztile, ntile, otile);
+  }
+  return (TRUE);
+}
 
   // this takes place in three major steps:
@@ -207 +248 @@
   // idata.pixsize : header.bitpix
 
-  // size of an element in the vartable heap section
-  zdata_pixsize = gfits_vartable_heap_pixsize (zdef.format);
-
-  // size of a pixel in the final image (not needed)
-  // idata_pixsize = abs(header[0].bitpix) / 8;
-
-  // size of a pixel in the output from the decompression routine
-  raw_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions)
-  ALLOCATE (raw_buffer, char, raw_pixsize*max_tile_size);
-  ALLOCATE (zdata, char, raw_pixsize*max_tile_size); // 
-
-  // compress the data : copy into a tile, compress the tile, then add to the output table
-  // each tile -> 1 row of the output table
-  int done = FALSE;
-  while (!done) {
-
-    // size of the current tile
-    Nraw = gfits_tile_size (matrix, otile, ztile);
-
-    // copy the raw pixels from their native matrix locations to the temporary output buffer
-    if (!gfits_collect_data (matrix, raw, Nraw, raw_pixsize, otile, oblank, ztile, zblank, zscale, zzero)) return (FALSE);
-
-    // XXX the tile must not be > 2GB
-
-    Nzdata = raw_pixsize*max_tile_size; // available space, replaced with actual output size on compression
-    if (!gfits_compress_data (zdata, &Nzdata, cmptype, optname, optvalue, Noptions, raw, Nraw, raw_pixsize)) return (FALSE);
-
-    if (!gfits_varlength_column_addrow (ftable, zdata, Nzdata, &zdef)) return FALSE;
-
-    // XXX need to activate
-    if (FALSE) gfits_byteswap_zdata (zdata, Nzdata, zdata_pixsize) return (FALSE);
-
-    // update the tile counters, carrying to the next dimension if needed
-    for (j = 0; j < matrix->Naxes; j++) {
-      otile[j] ++;
-      if (otile[j] == ntile[j]) {
-        if (j == matrix->Naxes - 1) { 
-          done = TRUE;
-          break;
-        }
-        otile[j] = 0;
-      } else {
-        break;
-      }
-    }
-  }
-  return (TRUE);
-}
 
 // raw_pixsize is the bytes / pixel for the input matrix

branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_utils.c

@@ -13 +13 @@
 // XXX inconsistency between data element size between compressed data, uncompressed data, 
 // and output pixel data.....
+
+// need a structure to describe tiles, current tile?
+
+typedef struct {
+  int *ztile; // max size of a tile (edge tiles may be smaller)
+  int *otile; // counter for current tile (eg, for (2,3,0) otile[0] = 2, otile[1] = 3, otile[2] = 0)
+  int *ntile; // number of tiles in dimension [i]
+} ImageTile;
+
+// advance the otile counter. 
+int gfits_imtile_next (Matrix *matrix, int *ztile, int *ntile, int *otile) {
+
+  int i;
+
+  // update the tile counters, carrying to the next dimension if needed
+  for (i = 0; i < matrix->Naxes; i++) {
+    otile[i] ++;
+    if (otile[i] == ntile[i]) {
+      if (i == matrix->Naxes - 1) return FALSE;
+      otile[i] = 0;
+    } else {
+      return TRUE;
+    }
+  }
+  return TRUE;
+}
+
+// how many tiles needed for this image (given ztile[])
+int gfits_imtile_start (Matrix *matrix, int *otile) {
+
+  int i;
+
+  // update the tile counters, carrying to the next dimension if needed
+  for (i = 0; i < matrix->Naxes; i++) {
+    otile[i] = 0;
+  }
+}
+
+// how many tiles needed for this image (given ztile[])
+int gfits_imtile_count (Matrix *matrix, int *ztile, int *ntile) {
+
+  int i;
+
+  int Ntiles = 1;
+  for (i = 0; i < matrix->Naxes; i++) {
+    ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]);
+    Ntiles *= ntile[i];
+  }
+  
+  return (Ntiles);
+}
+
+// how many tiles needed for this image (given ztile[])
+int gfits_imtile_maxsize (Matrix *matrix, int *ztile) {
+
+  int i;
+
+  int max_tile_size = 1;
+  for (i = 0; i < matrix->Naxes; i++) {
+    max_tile_size *= ztile[i];
+  }
+  
+  return (max_tile_size);
+}
 
 // true sizes of this tile (in pixels)

branches/eam_branches/ohana.20150429/src/libfits/table/F_table_format.c

@@ -52 +52 @@
     break;
   case  'P':
-    { *Nbytes = 8;  strcpy (type, "var");    *Nval = 2*Nv;             }
+    { *Nbytes = 4;  strcpy (type, "var");    *Nval = 2*Nv;             }
+    break;
+  case  'Q':
+    { *Nbytes = 8;  strcpy (type, "var64"); *Nval = 2*Nv;             }
     break;
   case  'C':
@@ -80 +83 @@
    C - complex float
    M - complex double
-   P - var length array descrpt (64 bit)
+   P - var length array descrpt (32 bit pointer)
+   Q - var length array descrpt (64 bit pointer)
    
    all can be preceeded by integer N which specified number
    of entries in array
-   
 */
 

branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c

@@ -123 +123 @@
 // Here are operations we need to do:
 
-int gfits_varlength_column_add_data (FTable *table, char *data, off_t Ndata, int row) {
+int gfits_varlength_column_add_data (FTable *table, char *data, off_t Ndata, int row, VarLengthColumn *column) {
 
   // find the current starting point for new data (end of current buffer main data + current heap
@@ -161 +161 @@
   column->maxlen = MAX (column->maxlen, Ndata);
 
+  off_t pcount;
+  gfits_scan (ftable->header, "PCOUNT", OFF_T_FMT, 1, &pcount);
+
+  pcount += Ndata;
+  gfits_modify (header, "PCOUNT", OFF_T_FMT, 1, pcount);
+
   return TRUE;
 }