IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38415


Ignore:
Timestamp:
Jun 6, 2015, 2:28:50 PM (11 years ago)
Author:
eugene
Message:

add timing test; handle empty table

Location:
branches/eam_branches/ohana.20150429/src/libfits
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_utils.c

    r38399 r38415  
    173173int gfits_cmptype_valid (char *cmptype) {
    174174
    175   if (!strcasecmp(cmptype, "NONE")) return TRUE;
     175  if (!strcasecmp(cmptype, "NONE"))   return TRUE;
     176  if (!strcasecmp(cmptype, "NONE_1")) return TRUE; // do not compress, but shuffle a la GZIP_1
    176177  if (!strcasecmp(cmptype, "NONE_2")) return TRUE; // do not compress, but shuffle a la GZIP_2
    177178  if (!strcasecmp(cmptype, "GZIP_1")) return TRUE;
    … …  
    192193      !strcasecmp(cmptype, "GZIP_2") ||
    193194      !strcasecmp(cmptype, "NONE_2") ||
     195      !strcasecmp(cmptype, "NONE_1") ||
    194196      !strcasecmp(cmptype, "NONE")) {
    195197    if (out_bitpix ==   8) return (1);
    … …  
    235237
    236238  // NONE should be swapped to mimic swapping types
    237   if (!strcasecmp(cmptype, "NONE")) {
     239  if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1")) {
    238240    if (out_bitpix ==   8) return (1);
    239241    if (out_bitpix ==  16) return (2);
    … …  
    292294  if (!strcasecmp(cmptype, "GZIP_1") ||
    293295      !strcasecmp(cmptype, "GZIP_2") ||
     296      !strcasecmp(cmptype, "NONE_1") ||
    294297      !strcasecmp(cmptype, "NONE_2") ||
    295298      !strcasecmp(cmptype, "NONE")) {

  • branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c

    r38408 r38415  
    3939  if (!ztilelen) ztilelen = srcheader->Naxis[1];
    4040
    41   if (srcheader->Naxis[1] % ztilelen) {
    42     Ntile = srcheader->Naxis[1] / ztilelen + 1;
    43     ztilelast = srcheader->Naxis[1] % ztilelen;
     41  if (!srcheader->Naxis[1]) {
     42    Ntile = 0;
     43    ztilelast = 0;
    4444  } else {
    45     Ntile = srcheader->Naxis[1] / ztilelen;
     45    if (srcheader->Naxis[1] % ztilelen) {
     46      Ntile = srcheader->Naxis[1] / ztilelen + 1;
     47      ztilelast = srcheader->Naxis[1] % ztilelen;
     48    } else {
     49      Ntile = srcheader->Naxis[1] / ztilelen;
    4650    ztilelast = ztilelen;
     51    }
    4752  }
    4853
    … …  
    237242      }
    238243
    239       if (strcasecmp(fields[j].zctype, "NONE") &&
    240           strcasecmp(fields[j].zctype, "NONE_2") &&
     244      if (strcasecmp(fields[j].zctype, "NONE_2") && // NONE and NONE_1 not swapped?
    241245          strcasecmp(fields[j].zctype, "GZIP_1") &&
    242246          strcasecmp(fields[j].zctype, "GZIP_2") &&

  • branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c

    r38366 r38415  
    4747}
    4848
     49static float timeSum1 = 0.0;
     50static float timeSum2 = 0.0;
     51static float timeSum2a = 0.0;
     52static float timeSum2b = 0.0;
     53static float timeSum2c = 0.0;
     54static float timeSum2d = 0.0;
     55static float timeSum2e = 0.0;
     56static float timeSum2f = 0.0;
     57static float timeSum3 = 0.0;
     58
     59void gfits_uncompress_timing () {
     60  fprintf (stderr, "times: %f %f %f\n", timeSum1, timeSum2, timeSum3);
     61  fprintf (stderr, "times: %f %f %f %f %f %f\n", timeSum2a, timeSum2b, timeSum2c, timeSum2d, timeSum2e, timeSum2f);
     62}
     63
    4964int gfits_uncompress_table (FTable *srctable, FTable *tgttable) {
    5065
    … …  
    5873  Header *srcheader = srctable->header;
    5974  Header *tgtheader = tgttable->header;
     75
     76  // XXX EAM:
     77  struct timeval startTimer, stopTimer;
     78  float dtime;
     79  gettimeofday (&startTimer, (void *) NULL);
    6080
    6181  int ztilelen;
    … …  
    126146  myAssert (Nx == tgtheader->Naxis[0], "table definition error?");
    127147
     148  // XXX TIMER 1
     149  gettimeofday (&stopTimer, (void *) NULL);
     150  dtime = DTIME (stopTimer, startTimer);
     151  timeSum1 += dtime;
     152  gettimeofday (&startTimer, (void *) NULL);
     153
    128154  // this allocates the pointer data array for the full set of tiles (NOT the heap)
    129155  char *tmpbuffer = NULL;
    … …  
    135161   
    136162  int Ntile = srcheader->Naxis[1];
    137   int ztilelast = (tgtheader->Naxis[1] % ztilelen) ? tgtheader->Naxis[1] % ztilelen : ztilelen;
     163  int ztilelast;
     164  if (Ntile == 0) {
     165    ztilelast = 0;
     166  } else {
     167    ztilelast = (tgtheader->Naxis[1] % ztilelen) ? tgtheader->Naxis[1] % ztilelen : ztilelen;
     168  }
    138169
    139170  // allocate the intermediate storage buffers
    … …  
    147178  gfits_dump_cmp_table (srctable, "unc");
    148179
     180  // XXX TIMER 2
     181  gettimeofday (&stopTimer, (void *) NULL);
     182  dtime = DTIME (stopTimer, startTimer);
     183  timeSum2 += dtime;
     184  gettimeofday (&startTimer, (void *) NULL);
     185
    149186  off_t row;
    150187  for (row = 0; row < srcheader->Naxis[1]; row++) {
    … …  
    157194      // XXX note that this function currently byteswaps.  I should redo the code to make
    158195      // byteswap of the varlength data segment a separate function
     196
     197      gettimeofday (&startTimer, (void *) NULL);
    159198
    160199      off_t Nzdata;
    … …  
    173212      }
    174213
    175       if (strcasecmp(fields[i].zctype, "NONE") &&
    176           strcasecmp(fields[i].zctype, "NONE_2") &&
     214      // XXX TIMER 2a
     215      gettimeofday (&stopTimer, (void *) NULL);
     216      dtime = DTIME (stopTimer, startTimer);
     217      timeSum2a += dtime;
     218      gettimeofday (&startTimer, (void *) NULL);
     219 
     220      if (strcasecmp(fields[i].zctype, "NONE_2") &&
    177221          strcasecmp(fields[i].zctype, "GZIP_1") &&
    178222          strcasecmp(fields[i].zctype, "GZIP_2") &&
    … …  
    183227      }
    184228
     229      // XXX TIMER 2b
     230      gettimeofday (&stopTimer, (void *) NULL);
     231      dtime = DTIME (stopTimer, startTimer);
     232      timeSum2b += dtime;
     233      gettimeofday (&startTimer, (void *) NULL);
     234 
    185235      if (VERBOSE_DUMP) {
    186236        int k;
    … …  
    194244      }
    195245
     246      // XXX TIMER 2c
     247      gettimeofday (&stopTimer, (void *) NULL);
     248      dtime = DTIME (stopTimer, startTimer);
     249      timeSum2c += dtime;
     250      gettimeofday (&startTimer, (void *) NULL);
     251 
    196252      int Nrows = (row == Ntile - 1) ? ztilelast : ztilelen;
    197253      int Nraw = Nrows*fields[i].Nvalues*fields[i].pixsize; // expected number of bytes
    … …  
    209265      }
    210266
     267      // XXX TIMER 2d
     268      gettimeofday (&stopTimer, (void *) NULL);
     269      dtime = DTIME (stopTimer, startTimer);
     270      timeSum2d += dtime;
     271      gettimeofday (&startTimer, (void *) NULL);
     272 
    211273      if (!strcasecmp(fields[i].zctype, "GZIP_1")) {
    212274        myAssert ((fields[i].zdef.format != 'C') && (fields[i].zdef.format != 'M'), "swap is probably wrong for C or M columns");
    … …  
    214276      }
    215277
     278      // XXX TIMER 2e
     279      gettimeofday (&stopTimer, (void *) NULL);
     280      dtime = DTIME (stopTimer, startTimer);
     281      timeSum2e += dtime;
     282      gettimeofday (&startTimer, (void *) NULL);
     283 
    216284      if (VERBOSE_DUMP) {
    217285        int k;
    … …  
    237305        if (!gfits_distribute_table_data (tgttable, &fields[i], raw, row_start, Nrows)) ESCAPE;
    238306      }     
     307
     308      // XXX TIMER 2f
     309      gettimeofday (&stopTimer, (void *) NULL);
     310      dtime = DTIME (stopTimer, startTimer);
     311      timeSum2f += dtime;
     312      gettimeofday (&startTimer, (void *) NULL);
    239313    }
    240314  }
     315
     316  // XXX TIMER 3
     317  gettimeofday (&stopTimer, (void *) NULL);
     318  dtime = DTIME (stopTimer, startTimer);
     319  timeSum3 += dtime;
     320  gettimeofday (&startTimer, (void *) NULL);
    241321
    242322  gfits_dump_raw_table (tgttable, "unc");
    … …  
    264344  off_t Nx = table->header->Naxis[0];
    265345
     346  char *tblbuffer = &table->buffer[Nx*row_start + field->offset];
     347
    266348  if (VERBOSE) fprintf (stderr, "distribute: ");
    267   for (i = 0; i < Nrows; i++) {
    268     int row = row_start + i;
    269     memcpy (&table->buffer[Nx*row + field->offset], &raw[i*field->rowsize], field->rowsize);
     349  for (i = 0; i < Nrows; i++, tblbuffer += Nx) {
     350    // int row = row_start + i;
     351    memcpy (tblbuffer, &raw[i*field->rowsize], field->rowsize);
    270352# if (VERBOSE)
    271353    int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]);
    … …  
    284366
    285367  off_t Nx = table->header->Naxis[0];
    286 
     368  int Nvalues = field->Nvalues;
     369  int pixsize = field->pixsize;
     370  int offset = field->offset;
     371
     372  char *rawptr = raw;
     373
     374  for (k = 0; k < field->pixsize; k++) {
     375# ifdef BYTE_SWAP     
     376    char *tblptr_start = &table->buffer[Nx*row_start + offset + (pixsize - k - 1)];
     377# else
     378    char *tblptr_start = &table->buffer[Nx*row_start + offset + k];
     379# endif
     380    for (i = 0; i < Nrows; i++, tblptr_start += Nx) {
     381      char *tblptr = tblptr_start;
     382      for (j = 0; j < Nvalues; j++, tblptr += pixsize, rawptr++) {
     383        *tblptr = *rawptr;
     384      }
     385    }
     386  }
     387  return (TRUE);
     388}
     389
     390int gfits_distribute_table_gzp2_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
     391 
     392  off_t i, j, k;
     393 
     394  // we are copying NN rows into the column which starts at XX and has MM bytes per row
     395 
     396  off_t Nx = table->header->Naxis[0];
     397 
    287398  for (k = 0; k < field->pixsize; k++) {
    288399    for (i = 0; i < Nrows; i++) {
    289400      int row = row_start + i;
    290401      char *rawptr = &raw[i*field->Nvalues + k*Nrows*field->Nvalues];
    291 # ifdef BYTE_SWAP      
     402# ifdef BYTE_SWAP     
    292403      char *tblptr = &table->buffer[Nx*row + field->offset + (field->pixsize - k - 1)];
    293404# else

  • branches/eam_branches/ohana.20150429/src/libfits/test/tablecomp.c

    r38404 r38415  
    44
    55int test_compress (char *zcmptype);
     6int test_compress_empty (char *zcmptype);
    67int test_compress_fullrange (char *zcmptype);
    78int test_compress_single_full (char *zcmptype);
    … …  
    1516  diag ("libfits tablecomp.c tests");
    1617
    17   // test_compress ("RICE_1");
    18   // test_compress_fullrange ("RICE_1");
    19 
    20 # if (1)
     18  // test_compress_empty ("GZIP_1");
     19  // exit (0);
     20
    2121  test_compress (NULL);
    2222  test_compress_fullrange (NULL);
    … …  
    2626    test_compress (cmptype[i]);
    2727    test_compress_fullrange (cmptype[i]);
    28   }
    29 # endif
    30 
    31   // test_compress_single_full ("GZIP_1");
    32   // test_compress_single_full ("RICE_1");
    33 
    34   // test_compress (NULL);
    35   // test_compress ("NONE");
    36   // test_compress ("NONE_2");
    37   // test_compress ("GZIP_1");
    38   // test_compress ("GZIP_2");
    39   // test_compress ("RICE_1");
    40   // test_compress ("RICE_ONE");
    41   // test_compress ("AUTO");
    42 
    43   // test_compress_fullrange (NULL);
    44   // test_compress_fullrange ("NONE");
     28    test_compress_empty (cmptype[i]);
     29    ok (ohana_memcheck_func (TRUE), "no memory corruption");
     30  }
    4531
    4632  return exit_status();
    … …  
    349335}
    350336
     337int test_compress_empty (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE"
     338
     339  Header header;
     340  FTable ftable;
     341
     342  diag ("--- starting test_compress with zcmptype %s ---", zcmptype);
     343  ok (gfits_init_header (&header), "inited the header");
     344  ok (gfits_init_table (&ftable), "inited the table");
     345
     346  ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header");
     347
     348  ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte",   "none", 1.0, 0.0), "defined byte column");
     349  ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short",  "none", 1.0, 0.0), "defined short column");
     350  ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int",    "none", 1.0, 0.0),  "defined int column");
     351  ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long",   "none", 1.0, 0.0),  "defined long column");
     352  ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float",  "degree", 1.0, 0.0),   "defined float column");
     353  ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0),   "defined double column");
     354 
     355  // generate the output array that carries the data
     356  ok (gfits_create_table (&header, &ftable), "created the basic table");
     357
     358  int Nval = 0;
     359  char    *VAL_B;  ALLOCATE (VAL_B, char,    Nval);
     360  short   *VAL_I;  ALLOCATE (VAL_I, short,   Nval);
     361  int     *VAL_J;  ALLOCATE (VAL_J, int,     Nval);
     362  int64_t *VAL_K;  ALLOCATE (VAL_K, int64_t, Nval);
     363  float   *VAL_E;  ALLOCATE (VAL_E, float,   Nval);
     364  double  *VAL_D;  ALLOCATE (VAL_D, double,  Nval);
     365 
     366  int i;
     367  for (i = 0; i < Nval; i++) {
     368    VAL_B[i] = (i % 255) - 128;
     369    VAL_I[i] = i;
     370    VAL_J[i] = 10000*i + 100*i;
     371    VAL_K[i] = 10000*i + 330*i;
     372    VAL_E[i] = 0.1*i;
     373    VAL_D[i] = 1.001*i;
     374  }     
     375
     376  // add the columns to the output array
     377  ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte   table column");
     378  ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short  table column");
     379  ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int    table column");
     380  ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long   table column");
     381  ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float  table column");
     382  ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column");
     383
     384  Header *outheader = &header;
     385  FTable *outtable = &ftable;
     386
     387  if (zcmptype) {
     388    Header cmpheader;
     389    FTable cmptable;
     390    cmptable.header = &cmpheader;
     391    ok (gfits_compress_table (&ftable, &cmptable, 0, zcmptype), "compressed table");
     392
     393    Header rawheader;
     394    FTable rawtable;
     395    rawtable.header = &rawheader;
     396    ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table");
     397   
     398    gfits_free_header (&cmpheader);
     399    gfits_free_table (&cmptable);
     400
     401    outheader = &rawheader;
     402    outtable  = &rawtable;
     403  }
     404
     405  char type[16];
     406  int Ncol;
     407  off_t Nrow;
     408
     409  char    *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"),    "read byte    table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     410  short   *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"),   "read short   table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     411  int     *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"),     "read int     table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     412  int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     413  float   *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"),   "read float   table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     414  double  *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"),  "read double  table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols");
     415
     416  // count mismatched values
     417  int NVAL_B_bad = 0;
     418  int NVAL_I_bad = 0;
     419  int NVAL_J_bad = 0;
     420  int NVAL_K_bad = 0;
     421  int NVAL_E_bad = 0;
     422  int NVAL_D_bad = 0;
     423
     424  // for (i = 0; i < Nval; i++) {
     425  //   fprintf (stderr, "%0llx : %0llx\n", (long long) VAL_K[i], (long long) VAL_K_t[i]);
     426  // }
     427
     428  for (i = 0; i < Nval; i++) {
     429    if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++;
     430    if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++;
     431    if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++;
     432    if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++;
     433    if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++;
     434    if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++;
     435  }
     436
     437  ok (!NVAL_B_bad, "byte    values match (input vs output)");
     438  ok (!NVAL_I_bad, "short   values match (input vs output)");
     439  ok (!NVAL_J_bad, "int     values match (input vs output)");
     440  ok (!NVAL_K_bad, "int64_t values match (input vs output)");
     441  ok (!NVAL_E_bad, "float   values match (input vs output)");
     442  ok (!NVAL_D_bad, "double  values match (input vs output)");
     443
     444  gfits_free_header (&header);
     445  gfits_free_table (&ftable);
     446  if (zcmptype) {
     447    gfits_free_header (outheader);
     448    gfits_free_table (outtable);
     449  }
     450  return TRUE;
     451}
     452
     453
Note: See TracChangeset for help on using the changeset viewer.