Index: /branches/eam_branches/ohana.20150429/src/libfits/Makefile =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38334) @@ -13,4 +13,6 @@ INC = $(HOME)/include MAN = $(HOME)/doc +TEST = $(HOME)/test +TESTBIN = $(HOME)/test include ../../Makefile.Common @@ -19,4 +21,12 @@ FULL_CPPFLAGS = $(BASE_CPPFLAGS) -I$(EXT) FULL_LDFLAGS = $(BASE_LDFLAGS) -lohana + +TEST_CFLAGS = $(BASE_CFLAGS) +TEST_CPPFLAGS = $(BASE_CPPFLAGS) +TEST_LDFLAGS = $(BASE_LDFLAGS) -lFITS -lohana -ltap_ohana + +TESTPROG = tablecomp zlib +$(TESTPROG) : % : $(TESTBIN)/% +test: $(TESTPROG) install: $(DESTLIB)/libFITS.a $(DESTLIB)/libFITS.$(DLLTYPE) $(DESTMAN)/fits.1 Index: /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c (revision 38334) @@ -120,5 +120,5 @@ { z_stream stream; - int err; + int i, err; stream.next_in = (Bytef*)source; @@ -135,15 +135,35 @@ stream.opaque = Z_NULL; + if (0) { + fprintf (stderr, "inp unc: "); + for (i = 0; i < sourceLen; i++) { + fprintf (stderr, "0x%02hhx ", source[i]); + } + fprintf (stderr, "\n"); + } + // err = inflateInit2(&stream, -15); err = inflateInit(&stream); + if (0) fprintf (stderr, "inp buffers unc 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + if (err != Z_OK) return err; err = inflate(&stream, Z_FINISH); + if (0) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); if (err != Z_STREAM_END) { inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; + if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) return Z_DATA_ERROR; return err; } + if (0) fprintf (stderr, "out buffers unc 2: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + + if (0) { + fprintf (stderr, "out unc: "); + for (i = 0; i < stream.total_out; i++) { + fprintf (stderr, "0x%02hhx ", dest[i]); + } + fprintf (stderr, "\n"); + } + assert (stream.total_out <= *destLen); *destLen = stream.total_out; @@ -156,5 +176,5 @@ { z_stream stream; - int err; + int i, err; stream.next_in = (Bytef*)source; @@ -172,6 +192,16 @@ stream.opaque = Z_NULL; + if (0) { + fprintf (stderr, "inp cmp: "); + for (i = 0; i < sourceLen; i++) { + fprintf (stderr, "0x%02hhx ", source[i]); + } + fprintf (stderr, "\n"); + } + // the '5' is the compression level: make this a user argument err = deflateInit(&stream, 5); + if (0) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + if (err != Z_OK) return err; @@ -179,8 +209,20 @@ // the compression occur in a series of steps err = deflate(&stream, Z_FINISH); + if (0) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err; - } + err = deflateEnd(&stream); + return Z_BUF_ERROR; + } + if (0) fprintf (stderr, "out buffers cmp 2: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + + if (0) { + fprintf (stderr, "out cmp: "); + for (i = 0; i < stream.total_out; i++) { + fprintf (stderr, "0x%02hhx ", dest[i]); + } + fprintf (stderr, "\n"); + } + assert (stream.total_out <= *destLen); *destLen = stream.total_out; Index: /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_data.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_data.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_compress_data.c (revision 38334) @@ -27,6 +27,4 @@ int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); -static int pass = 0; - int gfits_compress_data (char *zdata, int *Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, @@ -34,4 +32,13 @@ int status; + + // do not actually compress : this is used for testing + if (!strcasecmp(cmptype, "NONE")) { + unsigned long Nbytes = Nrawpix * rawpix_size; + + memcpy (zdata, rawdata, Nbytes); + *Nzdata = Nbytes; + return (TRUE); + } if (!strcasecmp(cmptype, "GZIP_1")) { @@ -49,37 +56,4 @@ fprintf (stderr, "error in compress (GZIP)\n"); return (FALSE); - } - - // test of deflate (vs I/O) - { - char *srcbuf, *tgtbuf; - ALLOCATE (srcbuf, char, destLen); - memcpy (srcbuf, zdata, destLen); - - ALLOCATE (tgtbuf, char, Nbytes); - - uLongf Ndeflate = Nbytes; - status = gfits_uncompress ((Bytef *) tgtbuf, &Ndeflate, (Bytef *) zdata, destLen); - if (status != Z_OK) { - fprintf (stderr, "error in uncompress (GZIP) : %d vs %d\n", status, Z_OK); - return (FALSE); - } - - myAssert (Ndeflate == Nbytes, "invalid output size?"); - - int i; - for (i = 0; i < Ndeflate; i++) { - myAssert (rawdata[i] == tgtbuf[i], "data mismatch?"); - } - - } - - if (pass == 0) { - int i; - for (i = 0; i < 32; i++) { - fprintf (stderr, "0x%02hhx ", zdata[i]); - } - fprintf (stderr, "\n"); - pass = 1; } return (TRUE); Index: /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_M.c (revision 38334) @@ -197,4 +197,5 @@ // find the COMPRESSED_DATA column (format should be 1PB, 1PI, 1PJ) + // is it required that this be the only column? for (i = 1; TRUE; i++) { snprintf (key, 10, "TTYPE%d", i); Index: /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_data.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_data.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/matrix/F_uncompress_data.c (revision 38334) @@ -26,6 +26,4 @@ int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); -static int pass = 0; - int gfits_uncompress_data (char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize) { @@ -33,4 +31,11 @@ static int Ninsum = 0; static int Noutsum = 0; + + // do not actually uncompress : this is used for testing + if (!strcasecmp(cmptype, "NONE")) { + memcpy (outdata, zdata, Nzdata); + *Nout = Nzdata / out_pixsize; + return (TRUE); + } if (!strcasecmp(cmptype, "GZIP_1")) { @@ -40,13 +45,4 @@ // XXX maybe not anymore // gfits_gz_stripheader ((unsigned char *)zdata, &Nzdata); - - if (pass == 0) { - int i; - for (i = 0; i < 32; i++) { - fprintf (stderr, "0x%02hhx ", zdata[i]); - } - fprintf (stderr, "\n"); - pass = 1; - } // uncompress does not require us to know the expected number of pixel; it tells us the number Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c (revision 38334) @@ -56,5 +56,5 @@ TableField *fields = NULL; - ALLOCATE (fields, TableField, Nfields); + ALLOCATE_ZERO (fields, TableField, Nfields); for (i = 0; i < Nfields; i++) { snprintf (keyword, 80, "TTYPE%d", i+1); @@ -129,10 +129,11 @@ fields[i].offset = offset; - offset += fields[i].offset; + offset += fields[i].rowsize; } // allocate the intermediate storage buffers + int Nzdata_alloc = max_width*ztilelen + 100; ALLOCATE (raw, char, max_width*ztilelen); - ALLOCATE (zdata, char, max_width*ztilelen); + ALLOCATE (zdata, char, Nzdata_alloc); // compress the data : copy into a tile, compress the tile, then add to the output table @@ -153,5 +154,5 @@ int Nraw = Nrows*fields[j].Nvalues; // number of pixels - int Nzdata = max_width*ztilelen; // available space, replaced with actual output size on compression + int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression if (!gfits_compress_data (zdata, &Nzdata, fields[j].zctype, NULL, NULL, 0, raw, Nraw, fields[j].pixsize)) ESCAPE(A); Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_table_varlength.c (revision 38334) @@ -128,12 +128,4 @@ Nx = ftable->header->Naxis[0]; -# ifdef BYTE_SWAP - char *pchar, tmp; - pchar = &ftable->buffer[row*Nx + column->offset]; - SWAP_WORD; - pchar = &ftable->buffer[row*Nx + column->offset + 4]; - SWAP_WORD; -# endif - if (column->mode == 'P') { int *ptr; @@ -156,7 +148,4 @@ // for varlength columns to the heap. To start, assume we have defined the cartesian // portion of the table correctly, and are only extending the heap. - -// XXX NOTE: this function assumes the table data is currently in local byte swap order. -// swap before writing out int gfits_varlength_column_add_data (FTable *table, char *data, off_t Ndata, int row, VarLengthColumn *column) { Index: /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c (revision 38333) +++ /branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c (revision 38334) @@ -68,5 +68,5 @@ fields[i].offset = offset; - offset += fields[i].offset; + offset += fields[i].rowsize; if (!gfits_varlength_column_define (srctable, &fields[i].zdef, i + 1)) ESCAPE(A); Index: /branches/eam_branches/ohana.20150429/src/libfits/test/tablecomp.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/tablecomp.c (revision 38334) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/tablecomp.c (revision 38334) @@ -0,0 +1,114 @@ +# include +# include +# include "tap_ohana.h" + +int test_compress (char *zcmptype); + +int main (int argc, char **argv) { + + plan_tests (64); + + diag ("libfits tablecomp.c tests"); + + test_compress (NULL); + test_compress ("NONE"); + test_compress ("GZIP_1"); + + exit (0); +} + +int test_compress (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "I", "SEQ", "seq", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "ID", "ID", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "E", "X", "x", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "Y", "y", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + int Nval = 1000; + short *SEQ; ALLOCATE (SEQ, short, Nval); + int *ID; ALLOCATE (ID, int, Nval); + float *X; ALLOCATE (X, float, Nval); + double *Y; ALLOCATE (Y, double, Nval); + + int i; + for (i = 0; i < Nval; i++) { + SEQ[i] = i; + ID[i] = 10000*i + 100*i; + X[i] = 0.1*i; + Y[i] = 1.001*i; + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "SEQ", SEQ, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "ID", ID, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "X", X, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "Y", Y, Nval), "set double table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, 10, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable, 10, zcmptype), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + short *SEQ_t = gfits_get_bintable_column_data (outheader, outtable, "SEQ", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); + int *ID_t = gfits_get_bintable_column_data (outheader, outtable, "ID", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); + float *X_t = gfits_get_bintable_column_data (outheader, outtable, "X", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); + double *Y_t = gfits_get_bintable_column_data (outheader, outtable, "Y", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); + + // count mismatched values + int Nseq = 0; + int Nid = 0; + int Nx = 0; + int Ny = 0; + for (i = 0; i < Nval; i++) { + if (SEQ[i] != SEQ_t[i]) Nseq++; + if ( ID[i] != ID_t[i]) Nid++; + if ( X[i] != X_t[i]) Nx++; + if ( Y[i] != Y_t[i]) Ny++; + } + + ok (!Nseq, "short values match (input vs output)"); + ok (!Nid, "int values match (input vs output)"); + ok (!Nx, "float values match (input vs output)"); + ok (!Ny, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + Index: /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c (revision 38334) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c (revision 38334) @@ -0,0 +1,179 @@ +# include +# include +# include +# include +# include + +# define NSRC 10 +# define NTGT 20 +# define NOUT 125 + +# define SWAP_DBLE { \ + char tmp; \ + tmp = Pout[0]; Pout[0] = Pout[7]; Pout[7] = tmp; \ + tmp = Pout[1]; Pout[1] = Pout[6]; Pout[6] = tmp; \ + tmp = Pout[2]; Pout[2] = Pout[5]; Pout[5] = tmp; \ + tmp = Pout[3]; Pout[3] = Pout[4]; Pout[4] = tmp; } + +int main (int argc, char **argv) { + + int i, err; + double srcbuf[NSRC], tgtbuf[NTGT], outbuf[NOUT]; + + { + z_stream zdn; + zdn.next_in = (char *) srcbuf; + zdn.avail_in = NSRC * sizeof(double); + zdn.next_out = (char *) tgtbuf; + zdn.avail_out = NTGT * sizeof(double); + + zdn.zalloc = Z_NULL; + zdn.zfree = Z_NULL; + zdn.opaque = Z_NULL; + + for (i = 0; i < NSRC; i++) srcbuf[i] = 1.001*(i + 10); + memset (tgtbuf, 0, NTGT * sizeof(double)); + memset (outbuf, 0, NOUT * sizeof(double)); + + for (i = 0; i < NSRC; i++) { + char *Pout = (char *) &srcbuf[i]; + SWAP_DBLE; + } + + char *rawdata = (char *) srcbuf; + fprintf (stderr, "inp: "); + for (i = 0; i < NSRC*8; i++) { + fprintf (stderr, "0x%02hhx ", rawdata[i]); + } + fprintf (stderr, "\n"); + + // the '5' is the compression level: make this a user argument + err = deflateInit(&zdn, 5); + fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = deflate(&zdn, Z_FINISH); + fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + fprintf (stderr, "out buffers cmp 2: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + + // dump out the data which failed to uncompress (and the input data) + char *cmpdata = (char *) tgtbuf; + fprintf (stderr, "out: "); + for (i = 0; i < zdn.total_out; i++) { + fprintf (stderr, "0x%02hhx ", cmpdata[i]); + } + fprintf (stderr, "\n"); + + assert (zdn.total_out <= NTGT*sizeof(double)); + + fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + // for (i = 0; i < zdn.total_out / sizeof(double); i++) { + // fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); + // } + } + + { + z_stream zup; + zup.next_in = (char *) tgtbuf; + zup.avail_in = NTGT * sizeof(double); + zup.next_out = (char *) outbuf; + zup.avail_out = NOUT * sizeof(double); + + zup.zalloc = Z_NULL; + zup.zfree = Z_NULL; + zup.opaque = Z_NULL; + + // the '5' is the compression level: make this a user argument + err = inflateInit(&zup); + fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = inflate(&zup, Z_FINISH); + fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + fprintf (stderr, "out buffers unc 2: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + + char *outdata = (char *) outbuf; + fprintf (stderr, "unc: "); + for (i = 0; i < zup.total_out; i++) { + fprintf (stderr, "0x%02hhx ", outdata[i]); + } + fprintf (stderr, "\n"); + + assert (zup.total_out <= NTGT*sizeof(double)); + + fprintf (stderr, "result: %d bytes\n", (int) zup.total_out); + for (i = 0; 0 && i < zup.total_out / sizeof(double); i++) { + fprintf (stderr, "%d : %f : %f\n", i, outbuf[i], srcbuf[i]); + assert (outbuf[i] == srcbuf[i]); + } + } + + exit (0); +} + +int main_old (int argc, char **argv) { + + int i, err; + char srcbuf[NSRC], tgtbuf[NTGT], outbuf[NTGT]; + + { + z_stream zdn; + zdn.next_in = srcbuf; + zdn.avail_in = NSRC; + zdn.next_out = tgtbuf; + zdn.avail_out = NTGT; + + zdn.zalloc = Z_NULL; + zdn.zfree = Z_NULL; + zdn.opaque = Z_NULL; + + for (i = 0; i < NSRC; i++) srcbuf[i] = i; + memset (tgtbuf, 0, NTGT); + memset (outbuf, 0, NTGT); + + // the '5' is the compression level: make this a user argument + err = deflateInit(&zdn, 5); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = deflate(&zdn, Z_FINISH); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + + assert (zdn.total_out <= NTGT); + + fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; i < zdn.total_out; i++) { + fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); + } + } + + { + z_stream zup; + zup.next_in = tgtbuf; + zup.avail_in = NTGT; + zup.next_out = outbuf; + zup.avail_out = NTGT; + + zup.zalloc = Z_NULL; + zup.zfree = Z_NULL; + zup.opaque = Z_NULL; + + // the '5' is the compression level: make this a user argument + err = inflateInit(&zup); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = inflate(&zup, Z_FINISH); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + + assert (zup.total_out <= NTGT); + + fprintf (stderr, "result: %d bytes\n", (int) zup.total_out); + for (i = 0; i < zup.total_out; i++) { + fprintf (stderr, "%d : 0x%02hhx : 0x%02hhx\n", i, outbuf[i], srcbuf[i]); + assert (outbuf[i] == srcbuf[i]); + } + } + + exit (0); +}