Index: /branches/eam_branches/ohana.20150429/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38427) @@ -176,15 +176,13 @@ // if no catalog already exists, use the input catalog to define the format or the specified format - if (outcatalog.Naverage_disk == 0) { - if (UPDATE_CATFORMAT) { - outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); - } else { - outcatalog.catformat = incatalog.catformat; - } - if (UPDATE_CATCOMPRESS) { - outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS); - } else { - outcatalog.catcompress = incatalog.catcompress; - } + if (UPDATE_CATFORMAT) { + outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } else { + outcatalog.catformat = incatalog.catformat; + } + if (UPDATE_CATCOMPRESS) { + outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS); + } else { + outcatalog.catcompress = incatalog.catcompress; } Index: /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c (revision 38427) @@ -884,4 +884,5 @@ // write out Measure table + catalog->measure_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { fprintf (stderr, "trouble writing Measure table\n"); @@ -910,4 +911,5 @@ // write out Missing table (must write out entire table) + catalog->missing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { fprintf (stderr, "trouble writing Missing Table\n"); @@ -942,4 +944,5 @@ // write out SecFilt table + catalog->secfilt_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { fprintf (stderr, "failure writing SecFilt table\n"); @@ -972,4 +975,5 @@ // write out Lensing table + catalog->lensing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) { fprintf (stderr, "trouble writing Lensing table\n"); @@ -1002,4 +1006,5 @@ // write out Lensobj table + catalog->lensobj_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) { fprintf (stderr, "trouble writing Lensobj table\n"); @@ -1032,4 +1037,5 @@ // write out StarPar table + catalog->starpar_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { fprintf (stderr, "trouble writing StarPar table\n"); @@ -1062,4 +1068,5 @@ // write out GalPhot table + catalog->galphot_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { fprintf (stderr, "trouble writing GalPhot table\n"); Index: /branches/eam_branches/ohana.20150429/src/libfits/Makefile =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/libfits/Makefile (revision 38427) @@ -27,6 +27,6 @@ -TESTXTRA = zlib tcomptiming -TESTPROG = imagecomp tablecomp ricetest +TESTXTRA = tcomptiming +TESTPROG = zlib ricetest imagecomp tablecomp $(TESTPROG) : % : $(TESTBIN)/% $(TESTXTRA) : % : $(TESTBIN)/% Index: /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/libfits/extern/gzip.c (revision 38427) @@ -119,7 +119,6 @@ # define ESCAPE(RET) { fprintf (stderr, "gzip error in %s @ %s:%d\n", __func__, __FILE__, __LINE__); return (RET); } -// NOTE: CFITSIO uses gzip format, not just zlib -int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) -{ +int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { + z_stream stream; int i, err; @@ -130,65 +129,12 @@ /* Check for source > 64K on 16-bit machine: */ if ((uLong)stream.avail_in != sourceLen) ESCAPE (Z_BUF_ERROR); - + stream.next_out = dest; - stream.avail_out = (uInt)*destLen; + stream.avail_out = (uInt)*destLen; // allocated space if ((uLong)stream.avail_out != *destLen) ESCAPE (Z_BUF_ERROR); - + stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; - - if (EXTRA_VERBOSE) { - fprintf (stderr, "inp unc: "); - for (i = 0; i < sourceLen; i++) { - fprintf (stderr, "0x%02hhx ", source[i]); - } - fprintf (stderr, "\n"); - } - - // a value of 32 tells inflate to use the window size used in the compression AND to look for both zlib and gzip headers - err = inflateInit2(&stream, 32); - if (err != Z_OK) ESCAPE (err); - - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) ESCAPE (Z_DATA_ERROR); - ESCAPE(err); - } - - if (EXTRA_VERBOSE) { - fprintf (stderr, "out unc: "); - for (i = 0; i < stream.total_out; i++) { - fprintf (stderr, "0x%02hhx ", dest[i]); - } - fprintf (stderr, "\n"); - } - - assert (stream.total_out <= *destLen); - *destLen = stream.total_out; - - err = inflateEnd(&stream); - return err; -} - -int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) -{ - z_stream stream; - int i, err; - - stream.next_in = (Bytef*)source; - stream.avail_in = (uInt)sourceLen; - - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) ESCAPE (Z_BUF_ERROR); - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; // allocated space - if ((uLong)stream.avail_out != *destLen) ESCAPE (Z_BUF_ERROR); - - stream.zalloc = Z_NULL; - stream.zfree = Z_NULL; - stream.opaque = Z_NULL; if (EXTRA_VERBOSE) { @@ -203,4 +149,5 @@ // NOTE: cfitsio uses a fixed value of 1 // I am using deflateInit2 because cfitsio expects gzip, not just zlib + // windowBits = 31 = (15 + 16) = (2^15 window bits) + (create a gzip stream) err = deflateInit2(&stream, 1, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); if (0) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); @@ -230,2 +177,60 @@ return err; } + +// NOTE: CFITSIO uses gzip format, not just zlib +int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { + + z_stream stream; + int i, err; + + stream.next_in = (Bytef*)source; + stream.avail_in = (uInt)sourceLen; + + /* Check for source > 64K on 16-bit machine: */ + if ((uLong)stream.avail_in != sourceLen) ESCAPE (Z_BUF_ERROR); + + stream.next_out = dest; + stream.avail_out = (uInt)*destLen; + if ((uLong)stream.avail_out != *destLen) ESCAPE (Z_BUF_ERROR); + + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + + if (EXTRA_VERBOSE) { + fprintf (stderr, "inp unc: "); + for (i = 0; i < sourceLen; i++) { + fprintf (stderr, "0x%02hhx ", source[i]); + } + fprintf (stderr, "\n"); + } + + // windowBits = 47 = (15 + 32) = (2^15 window bits) + (test for either gzip or zlib + // streams). NOTE: modern versions of zlib (version > 1.2.3.4) allow for windowBits = 0 + // (or 16 or 32) to use the compressed stream windowBits value. IPP cluster still has + // zlib version = 1.2.3, so we cannot use this feature. + err = inflateInit2(&stream, 47); + if (err != Z_OK) ESCAPE (err); + + err = inflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + inflateEnd(&stream); + if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) ESCAPE (Z_DATA_ERROR); + ESCAPE(err); + } + + if (EXTRA_VERBOSE) { + fprintf (stderr, "out unc: "); + for (i = 0; i < stream.total_out; i++) { + fprintf (stderr, "0x%02hhx ", dest[i]); + } + fprintf (stderr, "\n"); + } + + assert (stream.total_out <= *destLen); + *destLen = stream.total_out; + + err = inflateEnd(&stream); + return err; +} + Index: /branches/eam_branches/ohana.20150429/src/libfits/test/imagecomp.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/imagecomp.c (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/imagecomp.c (revision 38427) @@ -11,6 +11,6 @@ int bitpix[] = {8, 16, 32, -32, -64, 0}; -// char *cmptype[] = {"NONE_2", NULL}; -// int bitpix[] = {16, 0}; +//char *cmptype[] = {"GZIP_1", NULL}; +//int bitpix[] = {16, 0}; static int NX = 10; Index: /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c =================================================================== --- /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c (revision 38426) +++ /branches/eam_branches/ohana.20150429/src/libfits/test/zlib.c (revision 38427) @@ -1,8 +1,8 @@ -# include -# include -# include -# include +# include +# include # include - +# include "tap_ohana.h" + +# define VERBOSE 0 # define NSRC 10 # define NTGT 20 @@ -18,4 +18,11 @@ int main (int argc, char **argv) { + plan_tests (7); + + diag ("libfits zlib.c tests"); + + diag ("zlib version: %s", ZLIB_VERSION); + diag ("zlib vernum: %x", ZLIB_VERNUM); + int i, err; double srcbuf[NSRC], tgtbuf[NTGT], outbuf[NOUT]; @@ -41,39 +48,34 @@ } -# ifdef VERBOSE char *rawdata = (char *) srcbuf; - fprintf (stderr, "inp: "); - for (i = 0; i < NSRC*8; i++) { + if (VERBOSE) fprintf (stderr, "inp: "); + for (i = 0; VERBOSE && (i < NSRC*8); i++) { fprintf (stderr, "0x%02hhx ", rawdata[i]); } - fprintf (stderr, "\n"); -# endif + if (VERBOSE) fprintf (stderr, "\n"); // the '5' is the compression level: make this a user argument err = deflateInit(&zdn, 5); - // fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); - if (err != Z_OK) { fprintf (stdout, "not ok: error 1: %d vs %d\n", err, Z_OK); exit (1); } + if (VERBOSE) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); err = deflate(&zdn, Z_FINISH); - // fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); - if (err != Z_STREAM_END) { fprintf (stdout, "not ok: error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } - // fprintf (stderr, "out buffers cmp 2: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + if (VERBOSE) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); // dump out the data which failed to uncompress (and the input data) -# ifdef VERBOSE char *cmpdata = (char *) tgtbuf; - fprintf (stderr, "out: "); - for (i = 0; i < zdn.total_out; i++) { + if (VERBOSE) fprintf (stderr, "out: "); + for (i = 0; VERBOSE && (i < zdn.total_out); i++) { fprintf (stderr, "0x%02hhx ", cmpdata[i]); } - fprintf (stderr, "\n"); -# endif - - assert (zdn.total_out <= NTGT*sizeof(double)); - - // fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); - // for (i = 0; i < zdn.total_out / sizeof(double); i++) { - // fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); - // } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zdn.total_out <= NTGT*sizeof(double), "output compressed size is good"); + + if (VERBOSE) fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; VERBOSE && (i < zdn.total_out / sizeof(double)); i++) { + fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); + } } @@ -91,44 +93,34 @@ // the '5' is the compression level: make this a user argument err = inflateInit(&zup); - // fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); - if (err != Z_OK) { fprintf (stdout, "not ok: error 1: %d vs %d\n", err, Z_OK); exit (1); } + if (VERBOSE) fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); err = inflate(&zup, Z_FINISH); - // fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); - if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } - // fprintf (stderr, "out buffers unc 2: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); - -# ifdef VERBOSE + if (VERBOSE) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + char *outdata = (char *) outbuf; - fprintf (stderr, "unc: "); - for (i = 0; i < zup.total_out; i++) { + if (VERBOSE) fprintf (stderr, "unc: "); + for (i = 0; VERBOSE && (i < zup.total_out); i++) { fprintf (stderr, "0x%02hhx ", outdata[i]); } - fprintf (stderr, "\n"); -# endif - - assert (zup.total_out <= NTGT*sizeof(double)); - - // fprintf (stderr, "result: %d bytes\n", (int) zup.total_out); + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zup.total_out <= NTGT*sizeof(double), "uncompressed size is good"); + + int Nbad = 0; for (i = 0; 0 && i < zup.total_out / sizeof(double); i++) { - // fprintf (stderr, "%d : %f : %f\n", i, outbuf[i], srcbuf[i]); - assert (outbuf[i] == srcbuf[i]); - } - } - - return exit_status(); -} - -int main_old (int argc, char **argv) { - - int i, err; - char srcbuf[NSRC], tgtbuf[NTGT], outbuf[NTGT]; - + if (outbuf[i] != srcbuf[i]) Nbad++; + } + ok (Nbad == 0, "no mismatched bytes"); + } + + /**** TEST 2 : use gzip headers ****/ { z_stream zdn; - zdn.next_in = srcbuf; - zdn.avail_in = NSRC; - zdn.next_out = tgtbuf; - zdn.avail_out = NTGT; + zdn.next_in = (char *) srcbuf; + zdn.avail_in = NSRC * sizeof(double); + zdn.next_out = (char *) tgtbuf; + zdn.avail_out = NTGT * sizeof(double); zdn.zalloc = Z_NULL; @@ -136,19 +128,42 @@ zdn.opaque = Z_NULL; - for (i = 0; i < NSRC; i++) srcbuf[i] = i; - memset (tgtbuf, 0, NTGT); - memset (outbuf, 0, NTGT); + for (i = 0; i < NSRC; i++) srcbuf[i] = 1.001*(i + 10); + memset (tgtbuf, 0, NTGT * sizeof(double)); + memset (outbuf, 0, NOUT * sizeof(double)); + + for (i = 0; i < NSRC; i++) { + char *Pout = (char *) &srcbuf[i]; + SWAP_DBLE; + } + + char *rawdata = (char *) srcbuf; + if (VERBOSE) fprintf (stderr, "inp: "); + for (i = 0; VERBOSE && (i < NSRC*8); i++) { + fprintf (stderr, "0x%02hhx ", rawdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); // the '5' is the compression level: make this a user argument - err = deflateInit(&zdn, 5); - if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + // windowBits = 31 = (15 + 16) = (2^15 window bits) + (create a gzip stream) + err = deflateInit2(&zdn, 1, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + if (VERBOSE) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); err = deflate(&zdn, Z_FINISH); - if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } - - assert (zdn.total_out <= NTGT); - - fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); - for (i = 0; i < zdn.total_out; i++) { + if (VERBOSE) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + // dump out the data which failed to uncompress (and the input data) + char *cmpdata = (char *) tgtbuf; + if (VERBOSE) fprintf (stderr, "out: "); + for (i = 0; VERBOSE && (i < zdn.total_out); i++) { + fprintf (stderr, "0x%02hhx ", cmpdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zdn.total_out <= NTGT*sizeof(double), "output compressed size is good"); + + if (VERBOSE) fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; VERBOSE && (i < zdn.total_out / sizeof(double)); i++) { fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); } @@ -157,8 +172,8 @@ { z_stream zup; - zup.next_in = tgtbuf; - zup.avail_in = NTGT; - zup.next_out = outbuf; - zup.avail_out = NTGT; + zup.next_in = (char *) tgtbuf; + zup.avail_in = NTGT * sizeof(double); + zup.next_out = (char *) outbuf; + zup.avail_out = NOUT * sizeof(double); zup.zalloc = Z_NULL; @@ -166,19 +181,30 @@ zup.opaque = Z_NULL; - // the '5' is the compression level: make this a user argument - err = inflateInit(&zup); - if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + // a value of 32 tells inflate to use the window size used in the compression AND to look for both zlib and gzip headers + // windowBits = 47 = (15 + 32) = (2^15 window bits) + (test for either gzip or zlib streams) + err = inflateInit2(&zup, 47); + if (VERBOSE) fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_OK, "inflateInit2 result is Z_OK: %d vs %d", err, Z_OK); err = inflate(&zup, Z_FINISH); - if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } - - assert (zup.total_out <= NTGT); - - fprintf (stderr, "result: %d bytes\n", (int) zup.total_out); - for (i = 0; i < zup.total_out; i++) { - fprintf (stderr, "%d : 0x%02hhx : 0x%02hhx\n", i, outbuf[i], srcbuf[i]); - assert (outbuf[i] == srcbuf[i]); - } - } + if (VERBOSE) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_STREAM_END, "inflate result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + char *outdata = (char *) outbuf; + if (VERBOSE) fprintf (stderr, "unc: "); + for (i = 0; VERBOSE && (i < zup.total_out); i++) { + fprintf (stderr, "0x%02hhx ", outdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zup.total_out <= NTGT*sizeof(double), "uncompressed size is good"); + + int Nbad = 0; + for (i = 0; 0 && i < zup.total_out / sizeof(double); i++) { + if (outbuf[i] != srcbuf[i]) Nbad++; + } + ok (Nbad == 0, "no mismatched bytes"); + } + return exit_status(); }