Index: /branches/eam_branches/ohana.20150429/src/libfits/Makefile =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38425) @@ -27,5 +27,5 @@ # zlib -TESTPROG = imagecomp tablecomp ricetest +TESTPROG = imagecomp tablecomp tcomptiming ricetest $(TESTPROG) : % : $(TESTBIN)/% test: $(TESTPROG) Index: /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_matrix.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_matrix.c (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_matrix.c (revision 38425) @@ -1,28 +1,4 @@ # include # include - -off_t gfits_data_size (Header *header) { - - int i; - off_t Nrec, size; - - if (header[0].Naxes == 0) return (0); - - size = abs(header[0].bitpix / 8); - - for (i = 0; i < header[0].Naxes; i++) - size *= header[0].Naxis[i]; - - // XXX do I multiply this times gcount? - size += header[0].pcount; - - /* round up to next complete block */ - if (size % FT_RECORD_SIZE) { - Nrec = 1 + (int) (size / FT_RECORD_SIZE); - size = FT_RECORD_SIZE * Nrec; - } - - return (size); -} off_t gfits_heap_start (Header *header) { @@ -43,16 +19,8 @@ off_t gfits_data_min_size (Header *header) { - int i; - off_t size; + off_t size = gfits_heap_start (header); + size += header[0].pcount; - if (header[0].Naxes == 0) return (0); - - size = abs(header[0].bitpix / 8); - - for (i = 0; i < header[0].Naxes; i++) - size *= header[0].Naxis[i]; - - // XXX do I multiply this times gcount? - size += header[0].pcount; + // XXX what do I do with gcount? return (size); @@ -61,11 +29,9 @@ off_t gfits_data_pad_size (off_t rawsize) { - off_t Nrec, size; - - size = rawsize; + off_t size = rawsize; /* round up to next complete block */ if (rawsize % FT_RECORD_SIZE) { - Nrec = 1 + (int) (rawsize / FT_RECORD_SIZE); + off_t Nrec = 1 + (int) (rawsize / FT_RECORD_SIZE); size = FT_RECORD_SIZE * Nrec; } @@ -73,2 +39,10 @@ return (size); } + +off_t gfits_data_size (Header *header) { + off_t rawsize = gfits_data_min_size(header); + off_t size = gfits_data_pad_size (rawsize); + + return (size); +} + Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c (revision 38425) @@ -157,5 +157,5 @@ // extend the buffer size to add the new data - off_t Nbytes = heap_offset + Ndata; + off_t Nbytes = gfits_data_pad_size (heap_offset + Ndata); REALLOCATE (table->buffer, char, Nbytes); @@ -198,4 +198,5 @@ table->header->pcount = pcount; table->datasize = gfits_data_size (table->header); + myAssert (table->datasize == Nbytes, "inconsistent header and data"); return TRUE; Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c (revision 38425) @@ -179,5 +179,5 @@ // allocate the intermediate storage buffers - int Nraw_alloc = max_width*ztilelen + 100; + int Nraw_alloc = max_width*ztilelen + 1000; ALLOCATE (raw, char, Nraw_alloc); ALLOCATE (zdata, char, max_width*ztilelen); Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_T.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_T.c (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_T.c (revision 38425) @@ -14,4 +14,5 @@ status = fseeko (f, 0LL, SEEK_END); /* write table to end of file! */ + if (status) { perror ("fseeko: "); return (FALSE); } status = gfits_fwrite_table (f, table); @@ -26,6 +27,5 @@ Nbytes = fwrite (table[0].buffer, sizeof(char), table[0].datasize, f); - - if (Nbytes != table[0].datasize) return (FALSE); + if (Nbytes != table[0].datasize) { perror ("fwrite: "); return (FALSE); } return (TRUE); } @@ -45,4 +45,5 @@ /* file pointer is at beginning of desired table data */ start = ftello (f); + if (start < 0) { perror ("ftello: "); return FALSE; } for (i = 0; i < Nrow; i++) { @@ -50,5 +51,5 @@ fseeko (f, offset, SEEK_SET); Nbytes = fwrite (table[0].buffer[i], sizeof (char), Nx, f); - if (Nbytes != Nx) { return (FALSE); } + if (Nbytes != Nx) { perror ("fwrite: "); return (FALSE); } } @@ -58,7 +59,9 @@ offset = start + Nx*Ny; - fseeko (f, offset, SEEK_SET); + int status = fseeko (f, offset, SEEK_SET); + if (status) { perror ("fseeko: "); return FALSE; } + Nbytes = fwrite (pad, sizeof (char), Npad, f); - if (Nbytes != Npad) { return (FALSE); } + if (Nbytes != Npad) { perror ("fwrite: "); return (FALSE); } free (pad); @@ -97,4 +100,5 @@ // cursor must be at start of the table (after table header) if (fseeko (f, Nskip, SEEK_CUR)) { + perror ("fseeko: "); fprintf (stderr, "can't seek table start\n"); return (FALSE); @@ -102,7 +106,5 @@ Nwrite = fwrite (ftable[0].buffer, sizeof (char), Nbytes, f); - if (Nwrite != Nbytes) { - return (FALSE); - } + if (Nwrite != Nbytes) { perror ("fwrite: "); return (FALSE); } if (Ntotal >= Ndisk) { @@ -113,5 +115,5 @@ free (pad); - if (Nbytes != Npad) return (FALSE); + if (Nbytes != Npad) { perror ("fwrite: "); return (FALSE); } } Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_TH.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_TH.c (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_write_TH.c (revision 38425) @@ -14,4 +14,5 @@ status = fseeko (f, 0LL, SEEK_END); /* write header to end of file! */ + if (status) { perror ("fseeko: "); return (FALSE); } status = gfits_fwrite_Theader (f, header); @@ -26,6 +27,6 @@ Nbytes = fwrite (header[0].buffer, sizeof(char), header[0].datasize, f); + if (Nbytes != header[0].datasize) { perror ("fwrite: "); return (FALSE); } - if (Nbytes != header[0].datasize) return (FALSE); return (TRUE); } Index: /branches/eam_branches/ohana.20150429/src/libfits/test/compress.sh =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/compress.sh (revision 38424) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/compress.sh (revision 38425) @@ -1,3 +1,8 @@ input tap.dvo + +macro tests + test_image_all + test_table_all_rnd +end macro test_image_all @@ -308,5 +313,4 @@ end - macro test_table_all_rnd @@ -334,5 +338,5 @@ # lrnd generates rnd int between 0 and 0x7fffff (2^31 - 1 inclusive) - $NPT = 1000 + $NPT = 1000000 # $NPT = 30 create ni 0 $NPT -int Index: /branches/eam_branches/ohana.20150429/src/libfits/test/tcomptiming.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/tcomptiming.c (revision 38425) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/tcomptiming.c (revision 38425) @@ -0,0 +1,149 @@ +# include +# include +# include "tap_ohana.h" + +int test_compress_timing (char *zcmptype, int Ntile); +void gfits_uncompress_timing (); + +char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", "AUTO", NULL}; + +static int Nval = 30000000; + +int main (int argc, char **argv) { + + plan_tests (668); + // plan_skip_all ("skipping"); + + diag ("libfits tablecomp.c tests"); + + // test_compress_empty ("GZIP_1"); + // exit (0); + + float lNtileMax = log10(Nval) + 0.1; + float lNtile = 0.0; + for (lNtile = 1.0; lNtile < lNtileMax; lNtile += 1.0) { + int Ntile = pow (10.0, lNtile); + test_compress_timing ("NONE_2", Ntile); + fprintf (stderr, "--- Ntile = %d ---\n", Ntile); + gfits_uncompress_timing(); + } + ok (ohana_memcheck_func (TRUE), "no memory corruption"); + + return exit_status(); +} + +int test_compress_timing (char *zcmptype, int Ntile) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte", "none", 1.0, 0.0), "defined byte column"); + ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long", "none", 1.0, 0.0), "defined long column"); + ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + char *VAL_B; ALLOCATE (VAL_B, char, Nval); + short *VAL_I; ALLOCATE (VAL_I, short, Nval); + int *VAL_J; ALLOCATE (VAL_J, int, Nval); + int64_t *VAL_K; ALLOCATE (VAL_K, int64_t, Nval); + float *VAL_E; ALLOCATE (VAL_E, float, Nval); + double *VAL_D; ALLOCATE (VAL_D, double, Nval); + + long A = time(NULL); + srand48(A); + + int i; + for (i = 0; i < Nval; i++) { + VAL_B[i] = 0xff & lrand48(); + VAL_I[i] = 0xffff & lrand48(); + VAL_J[i] = 0xffffffff & lrand48(); + VAL_K[i] = (((int64_t)lrand48()) << 32) + (int64_t)lrand48(); + VAL_E[i] = FLT_MAX * (2.0*drand48() - 1.0); + VAL_D[i] = DBL_MAX * (2.0*drand48() - 1.0); + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, Ntile, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + char *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"), "read byte table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + short *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + float *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + double *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // count mismatched values + int NVAL_B_bad = 0; + int NVAL_I_bad = 0; + int NVAL_J_bad = 0; + int NVAL_K_bad = 0; + int NVAL_E_bad = 0; + int NVAL_D_bad = 0; + + for (i = 0; i < Nval; i++) { + if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++; + if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++; + if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++; + if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++; + if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++; + if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++; + } + + ok (!NVAL_B_bad, "byte values match (input vs output)"); + ok (!NVAL_I_bad, "short values match (input vs output)"); + ok (!NVAL_J_bad, "int values match (input vs output)"); + ok (!NVAL_K_bad, "int64_t values match (input vs output)"); + ok (!NVAL_E_bad, "float values match (input vs output)"); + ok (!NVAL_D_bad, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + +