Index: /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/bcatalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/bcatalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/bcatalog.c (revision 38447) @@ -408,5 +408,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = oldcatalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * oldcatalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * oldcatalog[0].Nsecfilt; return (TRUE); @@ -515,5 +515,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = oldcatalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * oldcatalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * oldcatalog[0].Nsecfilt; return (TRUE); @@ -609,5 +609,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = catalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * catalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * catalog[0].Nsecfilt; return (TRUE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/checkastro_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/checkastro_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/checkastro_images.c (revision 38447) @@ -11,4 +11,5 @@ /* register database handle with shutdown procedure */ set_db (&db); + gfits_db_init (&db); /* lock and load the image db table */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_catalogs.c (revision 38447) @@ -35,5 +35,5 @@ pcatalog->catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data pcatalog->catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - pcatalog->catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; // don't need to load all data at this point + pcatalog->catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; // don't need to load all data at this point pcatalog->Nsecfilt = GetPhotcodeNsecfilt (); @@ -43,5 +43,5 @@ exit (1); } - if (!pcatalog[0].Naves_disk) { + if (!pcatalog[0].Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", pcatalog[0].filename); dvo_catalog_unlock (pcatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/checkastro/src/load_images.c (revision 38447) @@ -10,5 +10,5 @@ // convert database table to internal structure - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_images.c (revision 38447) @@ -16,5 +16,5 @@ Image *image, *outimage; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -61,5 +61,5 @@ fprintf (stderr, "removing "OFF_T_FMT" images (leaving "OFF_T_FMT" of "OFF_T_FMT")\n", Nimage - Noutimage, Noutimage, Nimage); - // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage); + // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage, TRUE); gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Noutimage); @@ -110,5 +110,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -119,5 +119,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_measures.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 38447) @@ -45,5 +45,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -54,5 +54,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP.c (revision 38447) @@ -43,5 +43,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -52,5 +52,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges.c (revision 38447) @@ -51,5 +51,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -60,5 +60,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges_delete.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges_delete.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_fix_LAP_edges_delete.c (revision 38447) @@ -91,5 +91,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -100,5 +100,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_image_photcodes.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_image_photcodes.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_image_photcodes.c (revision 38447) @@ -12,5 +12,5 @@ PhotCode **photcodes = ParsePhotcodeList (PHOTCODE_LIST, &Nphotcodes, FALSE); - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -33,5 +33,5 @@ if (VERBOSE) fprintf (stderr, "removing "OFF_T_FMT" images (leaving "OFF_T_FMT" of "OFF_T_FMT")\n", Nimage - Noutimage, Noutimage, Nimage); - // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage); + // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage, TRUE); gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Noutimage); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagefile.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagefile.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagefile.c (revision 38447) @@ -26,5 +26,5 @@ catalog.filename = sky[0].filename[i]; /* don't free region before catalog! */ catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -33,5 +33,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagename.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagename.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_imagename.c (revision 38447) @@ -18,5 +18,5 @@ SkyTableSetFilenames (sky, CATDIR, "cpt"); - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -41,5 +41,5 @@ catalog.filename = skylist[0].filename[i]; /* don't free region before catalog! */ catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -48,5 +48,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); @@ -82,5 +82,5 @@ if (VERBOSE) fprintf (stderr, "removing "OFF_T_FMT" images (leaving "OFF_T_FMT" of "OFF_T_FMT")\n", Nimlist, Noutimage, Nimage); - // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage); + // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage, TRUE); gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Noutimage); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_orphans.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_orphans.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_orphans.c (revision 38447) @@ -252,5 +252,5 @@ catalog.Nmeasure = Nmeas; catalog.Nmissing = Nmiss; - catalog.Nsecf_mem = Nave*Nsecfilt; + catalog.Nsecfilt_mem = Nave*Nsecfilt; if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: %d %d %d\n", Nave, Nmeas, Nmiss); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes.c (revision 38447) @@ -41,5 +41,5 @@ catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -50,5 +50,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes_single.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes_single.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_photcodes_single.c (revision 38447) @@ -10,5 +10,5 @@ catalog.filename = cptname; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); @@ -19,5 +19,5 @@ return FALSE; } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_times.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_times.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delete_times.c (revision 38447) @@ -25,5 +25,5 @@ skylist[0].ownElements = FALSE; // free these elements when freeing the list - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -62,5 +62,5 @@ catalog.filename = skylist[0].filename[i]; /* don't free region before catalog! */ catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -69,5 +69,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delstar.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delstar.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/delstar.c (revision 38447) @@ -12,4 +12,5 @@ if (!SKIP_IMAGES) { set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD); /* XCLD */ if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); @@ -30,5 +31,5 @@ case MODE_FIX_LAP: { off_t Nimage; - Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -67,5 +68,5 @@ case MODE_FIX_LAP_STATS: { off_t Nimage; - Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_image_db.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_image_db.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_image_db.c (revision 38447) @@ -16,5 +16,5 @@ } - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -46,5 +46,5 @@ int code; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); start = timage[0].tzero - MAX(0.01*timage[0].trate*timage[0].NY, 1); @@ -76,5 +76,5 @@ Image *image; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); Nlist = 0; Index: /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_matches.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_matches.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/delstar/src/find_matches.c (revision 38447) @@ -245,5 +245,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nmissing = Nmiss; - catalog[0].Nsecf_mem = Nave*Nsecfilt; + catalog[0].Nsecfilt_mem = Nave*Nsecfilt; if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT"\n", Nave, Nmeas, Nmiss); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvolens/src/update_objects.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvolens/src/update_objects.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvolens/src/update_objects.c (revision 38447) @@ -61,5 +61,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_LENSING | LOAD_LENSOBJ; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ; catalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog @@ -68,5 +68,5 @@ exit (1); } - if (VERBOSE && (catalog.Naves_disk == 0)) { + if (VERBOSE && (catalog.Naverage_disk == 0)) { fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 38447) @@ -53,5 +53,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_LENSOBJ; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_LENSOBJ; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -60,5 +60,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps.c (revision 38447) @@ -65,5 +65,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -72,5 +72,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -138,24 +138,23 @@ } - char command[1024]; - snprintf (command, 1024, "dvopsps_client detections -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "dvopsps_client detections -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[1024]; - snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbuser %s", command, DATABASE_USER); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbpass %s", command, DATABASE_PASS); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); - - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (SAVE_REMOTE) { snprintf (tmpline, 1024, "%s -save %s", command, table->hosts[i].results); strcpy (command, tmpline); } + strextend (&command, "-dbhost %s", DATABASE_HOST); + strextend (&command, "-dbuser %s", DATABASE_USER); + strextend (&command, "-dbpass %s", DATABASE_PASS); + strextend (&command, "-dbname %s", DATABASE_NAME); + + if (VERBOSE) { strextend (&command, "-v");} + if (SAVE_REMOTE) { strextend (&command, "-save %s", table->hosts[i].results);} // some filters -- these are the detections we skip - if (TIME_START) { snprintf (tmpline, 1024, "%s -time-start %s", command, TIME_START); strcpy (command, tmpline); } - if (TIME_END) { snprintf (tmpline, 1024, "%s -time-end %s", command, TIME_END); strcpy (command, tmpline); } + if (TIME_START) { strextend (&command, "-time-start %s", TIME_START);} + if (TIME_END) { strextend (&command, "-time-end %s", TIME_END);} - if (PHOTCODE_START > 0) { snprintf (tmpline, 1024, "%s -photcode-start %d", command, PHOTCODE_START); strcpy (command, tmpline); } - if (PHOTCODE_END <= INT_MAX) { snprintf (tmpline, 1024, "%s -photcode-end %d", command, PHOTCODE_END); strcpy (command, tmpline); } + if (PHOTCODE_START > 0) { strextend (&command, "-photcode-start %d", PHOTCODE_START);} + if (PHOTCODE_END <= INT_MAX) { strextend (&command, "-photcode-end %d", PHOTCODE_END);} fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 38447) @@ -215,5 +215,9 @@ fprintf (stderr, "%s\n", mysql_error(mysql)); fprintf (stderr, "Nbuffer: %d\n", buffer->Nbuffer); - if (DEBUG) fprintf (stderr, "%s\n", buffer->buffer); + if (DEBUG) { + FILE *f = fopen ("dvopsps.dump.sql", "w"); + fprintf (f, "%s\n", buffer->buffer); + fclose (f); + } status = FALSE; } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c (revision 38447) @@ -53,5 +53,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -60,5 +60,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -126,17 +126,16 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "dvopsps_client diffobj -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "dvopsps_client diffobj -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[1024]; - snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbuser %s", command, DATABASE_USER); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbpass %s", command, DATABASE_PASS); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); + strextend (&command, "-dbhost %s", DATABASE_HOST); + strextend (&command, "-dbuser %s", DATABASE_USER); + strextend (&command, "-dbpass %s", DATABASE_PASS); + strextend (&command, "-dbname %s", DATABASE_NAME); - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_objects_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_objects_dvopsps.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvopsps/src/insert_objects_dvopsps.c (revision 38447) @@ -53,5 +53,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -60,5 +60,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -126,17 +126,16 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "dvopsps_client objects -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "dvopsps_client objects -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[1024]; - snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbuser %s", command, DATABASE_USER); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbpass %s", command, DATABASE_PASS); strcpy (command, tmpline); - snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); + strextend (&command, "-dbhost %s", DATABASE_HOST); + strextend (&command, "-dbuser %s", DATABASE_USER); + strextend (&command, "-dbpass %s", DATABASE_PASS); + strextend (&command, "-dbname %s", DATABASE_NAME); - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/dvosplit.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/dvosplit.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/dvosplit.c (revision 38447) @@ -80,5 +80,5 @@ incatalog.filename = skylist[0].filename[i]; incatalog.Nsecfilt = GetPhotcodeNsecfilt (); - incatalog.catflags = LOAD_NONE; + incatalog.catflags = DVO_LOAD_NONE; // an error exit status here is a significant error @@ -89,5 +89,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/open_output_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 38447) @@ -14,5 +14,5 @@ outcatalogs[i].filename = outlist[0].filename[i]; outcatalogs[i].Nsecfilt = GetPhotcodeNsecfilt (); - outcatalogs[i].catflags = LOAD_NONE; + outcatalogs[i].catflags = DVO_LOAD_NONE; outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat; // set the default catformat from config data outcatalogs[i].catmode = CATMODE ? dvo_catalog_catmode (CATMODE) : catmode; // set the default catmode from config data Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_averages.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_averages.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_averages.c (revision 38447) @@ -10,6 +10,6 @@ AveLinks *avelinks; - ALLOCATE (outref, int, incatalog[0].Naves_disk); - ALLOCATE (outcat, int, incatalog[0].Naves_disk); + ALLOCATE (outref, int, incatalog[0].Naverage_disk); + ALLOCATE (outcat, int, incatalog[0].Naverage_disk); ALLOCATE (outmem, int, outlist[0].Nregions); @@ -25,9 +25,9 @@ // split out the average & secfilt entries: - incatalog[0].catflags = LOAD_AVES | LOAD_SECF; + incatalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) { - Nblocks = incatalog[0].Naves_disk / NROWS; - if (incatalog[0].Naves_disk % NROWS) Nblocks ++; + Nblocks = incatalog[0].Naverage_disk / NROWS; + if (incatalog[0].Naverage_disk % NROWS) Nblocks ++; } else { Nblocks = 1; @@ -40,5 +40,5 @@ dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks); - assert (block*NROWS == incatalog[0].Naves_off); + assert (block*NROWS == incatalog[0].Naverage_off); } else { dvo_catalog_load (incatalog, VERBOSE); @@ -48,5 +48,5 @@ // distribute data to the output catalogs for (ave = 0; ave < incatalog[0].Naverage; ave++) { - averef = ave + incatalog[0].Naves_off; + averef = ave + incatalog[0].Naverage_off; inR = incatalog[0].average[ave].R; @@ -75,5 +75,5 @@ Nout = outcatalogs[Ncat].Naverage; - outref[averef] = Nout + outcatalogs[Ncat].Naves_off; + outref[averef] = Nout + outcatalogs[Ncat].Naverage_off; outcat[averef] = Ncat; @@ -85,5 +85,5 @@ for (n = 0; n < Nsecfilt; n++) { outcatalogs[Ncat].secfilt[Nout*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n]; - outcatalogs[Ncat].Nsecf_mem++; + outcatalogs[Ncat].Nsecfilt_mem++; } @@ -96,5 +96,5 @@ dvo_catalog_free_data (incatalog); - // double check the values of Naverage, Nsecf_mem? + // double check the values of Naverage, Nsecfilt_mem? // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below @@ -103,19 +103,19 @@ if (!FULL_TABLE) { for (cat = 0; cat < outlist[0].Nregions; cat++) { - outcatalogs[cat].catflags = LOAD_AVES | LOAD_SECF; + outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; dvo_catalog_save (&outcatalogs[cat], VERBOSE); - // fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt); + // fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecfilt_mem, outcatalogs[cat].Nsecfilt_disk, outcatalogs[cat].Nsecfilt_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt); // advance the pointers and free the current data // XXX these should be done within save segment: - outcatalogs[cat].Naves_disk += outcatalogs[cat].Naverage; - outcatalogs[cat].Naves_off += outcatalogs[cat].Naverage; - outcatalogs[cat].Nsecf_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; - outcatalogs[cat].Nsecf_off += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; + outcatalogs[cat].Naverage_disk += outcatalogs[cat].Naverage; + outcatalogs[cat].Naverage_off += outcatalogs[cat].Naverage; + outcatalogs[cat].Nsecfilt_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; + outcatalogs[cat].Nsecfilt_off += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; outcatalogs[cat].Nsecfilt = Nsecfilt; outcatalogs[cat].Naverage = 0; - outcatalogs[cat].Nsecf_mem = 0; + outcatalogs[cat].Nsecfilt_mem = 0; } } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_measures.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_measures.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_measures.c (revision 38447) @@ -19,15 +19,15 @@ // split out the measure entries: - incatalog[0].catflags = LOAD_MEAS | LOAD_AVES | LOAD_SECF; + incatalog[0].catflags = DVO_LOAD_MEASURE | DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; // if ((incatalog[0].catformat == DVO_FORMAT_ELIXIR) || (incatalog[0].catformat == DVO_FORMAT_LONEOS)) { // // for these two formats, we need the average and secfilt values around until we do the measures... // // XXX I am loading these 2x -- perhaps I can make the API smarter about reloading? - // incatalog[0].catflags |= LOAD_AVES | LOAD_SECF; + // incatalog[0].catflags |= DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; // } if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) { - Nblocks = incatalog[0].Nmeas_disk / NROWS; - if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++; + Nblocks = incatalog[0].Nmeasure_disk / NROWS; + if (incatalog[0].Nmeasure_disk % NROWS) Nblocks ++; } else { Nblocks = 1; @@ -40,5 +40,5 @@ dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks); - assert (block*NROWS == incatalog[0].Nmeas_off); + assert (block*NROWS == incatalog[0].Nmeasure_off); } else { dvo_catalog_load (incatalog, VERBOSE); @@ -72,9 +72,9 @@ if (!FULL_TABLE) { for (cat = 0; cat < outlist[0].Nregions; cat++) { - outcatalogs[cat].catflags = LOAD_MEAS; + outcatalogs[cat].catflags = DVO_LOAD_MEASURE; dvo_catalog_save (&outcatalogs[cat], VERBOSE); - outcatalogs[cat].Nmeas_disk += outcatalogs[cat].Nmeasure; - outcatalogs[cat].Nmeas_off += outcatalogs[cat].Nmeasure; + outcatalogs[cat].Nmeasure_disk += outcatalogs[cat].Nmeasure; + outcatalogs[cat].Nmeasure_off += outcatalogs[cat].Nmeasure; outcatalogs[cat].Nmeasure = 0; } @@ -84,5 +84,5 @@ if (FULL_TABLE) { for (cat = 0; cat < outlist[0].Nregions; cat++) { - outcatalogs[cat].catflags = LOAD_AVES | LOAD_SECF | LOAD_MEAS; + outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE; dvo_catalog_save (&outcatalogs[cat], VERBOSE); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_missings.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_missings.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvosplit/src/split_missings.c (revision 38447) @@ -15,7 +15,7 @@ // split out the missing entries: - incatalog[0].catflags = LOAD_MISS; - Nblocks = incatalog[0].Nmiss_disk / NROWS; - if (incatalog[0].Nmiss_disk % NROWS) Nblocks ++; + incatalog[0].catflags = DVO_LOAD_MISSING; + Nblocks = incatalog[0].Nmissing_disk / NROWS; + if (incatalog[0].Nmissing_disk % NROWS) Nblocks ++; for (block = 0; block < Nblocks; block++) { @@ -35,9 +35,9 @@ for (cat = 0; cat < outlist[0].Nregions; cat++) { - outcatalogs[cat].catflags = LOAD_MISS; + outcatalogs[cat].catflags = DVO_LOAD_MISSING; dvo_catalog_save_segment (&outcatalog[cat], VERBOSE); - outcatalog[cat].Nmiss_disk += outcatalog[cat].Nmissing; - outcatalog[cat].Nmiss_off += outcatalog[cat].Nmissing; + outcatalog[cat].Nmissing_disk += outcatalog[cat].Nmissing; + outcatalog[cat].Nmissing_off += outcatalog[cat].Nmissing; outcatalog[cat].Nmissing = 0; Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/include/fakeastro.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/include/fakeastro.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/include/fakeastro.h (revision 38447) @@ -149,6 +149,4 @@ double int_gauss (int i); -int strextend (char *input, char *format,...); - int fakeastro_images (); Image *load_template_images (int *nimage); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_2mass.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_2mass.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_2mass.c (revision 38447) @@ -21,5 +21,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_MEAS | LOAD_STARPAR; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_STARPAR; catalog.Nsecfilt = GetPhotcodeNsecfilt (); if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { @@ -27,5 +27,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) { fprintf (stderr, "no data in %s, skipping\n", catalog.filename); } dvo_catalog_unlock (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakeastro_images.c (revision 38447) @@ -67,4 +67,5 @@ /* setup image table format and lock */ + gfits_db_init (db); db->mode = dvo_catalog_catmode (CATMODE); db->format = dvo_catalog_catformat (CATFORMAT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakestar_catalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakestar_catalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/fakestar_catalog.c (revision 38447) @@ -9,5 +9,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_STARPAR; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_STARPAR; catalog.Nsecfilt = GetPhotcodeNsecfilt (); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/insert_fakestar.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/insert_fakestar.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/insert_fakestar.c (revision 38447) @@ -118,5 +118,5 @@ catalog[0].Naverage = Nave; catalog[0].Nstarpar = Nstarpar; - catalog[0].Nsecf_mem = Nave*Nsecfilt; + catalog[0].Nsecfilt_mem = Nave*Nsecfilt; if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nstarpar: %d "OFF_T_FMT" "OFF_T_FMT" ("OFF_T_FMT" matches)\n", Nstars, Nave, Nstarpar, Nmatch); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_fake_stars.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_fake_stars.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_fake_stars.c (revision 38447) @@ -15,5 +15,5 @@ catalog[i].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog[i].catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog[i].catflags = LOAD_AVES | LOAD_SECF | LOAD_STARPAR; + catalog[i].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_STARPAR; catalog[i].Nsecfilt = GetPhotcodeNsecfilt (); if (!dvo_catalog_open (&catalog[i], skylist[0].regions[i], VERBOSE, "r")) { @@ -23,5 +23,5 @@ dvo_catalog_unlock (&catalog[i]); - if (!catalog[i].Naves_disk) { + if (!catalog[i].Naverage_disk) { if (VERBOSE2) { fprintf (stderr, "no data in %s, skipping\n", catalog[i].filename); } dvo_catalog_free (&catalog[i]); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_template_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_template_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/load_template_images.c (revision 38447) @@ -5,4 +5,5 @@ FITS_DB db; + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); @@ -24,5 +25,5 @@ off_t Nimage; - Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: no template images in %s (though db exists)\n", IMAGES_INPUT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/match_fake_stars.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/match_fake_stars.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/match_fake_stars.c (revision 38447) @@ -282,5 +282,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nstarpar = Nstarpar; - catalog[0].Nsecf_mem = Nave*Nsecfilt; + catalog[0].Nsecfilt_mem = Nave*Nsecfilt; if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fake_stars.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fake_stars.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fake_stars.c (revision 38447) @@ -23,6 +23,6 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - // catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_MEAS | LOAD_STARPAR; // use this for non-update mode - catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_STARPAR; + // catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_STARPAR; // use this for non-update mode + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_STARPAR; catalog.Nsecfilt = GetPhotcodeNsecfilt (); Index: /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fakestars.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fakestars.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/fakeastro/src/save_fakestars.c (revision 38447) @@ -48,12 +48,12 @@ // need to generate the remote command - char command[1024]; - snprintf (command, 1024, "fakestar_client"); - strextend (command, "-galaxy"); - strextend (command, "-hostID %d", host->hostID); - strextend (command, "-D CATDIR %s", CATDIR); - strextend (command, "-hostdir %s", host->pathname); - strextend (command, "-cpt %s", region->name); - strextend (command, "-input %s", filename); + char *command = NULL; + strextend (&command, "fakestar_client"); + strextend (&command, "-galaxy"); + strextend (&command, "-hostID %d", host->hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", host->pathname); + strextend (&command, "-cpt %s", region->name); + strextend (&command, "-input %s", filename); // launch the job on the remote machine (no handshake) @@ -70,15 +70,2 @@ return TRUE; } - -int strextend (char *input, char *format,...) { - - char tmpextra[1024], tmpline[1024]; - va_list argp; - - va_start (argp, format); - vsnprintf (tmpextra, 1024, format, argp); - snprintf (tmpline, 1024, "%s %s", input, tmpextra); - strcpy (input, tmpline); - - return TRUE; -} Index: /branches/eam_branches/ipp-20150419/Ohana/src/gastro/src/getptolemy.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/gastro/src/getptolemy.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/gastro/src/getptolemy.c (revision 38447) @@ -26,5 +26,5 @@ // set the parameters which guide catalog open/load/create catalog.filename = skylist[0].filename[i]; - catalog.catflags = LOAD_AVES | LOAD_MEAS; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE; catalog.Nsecfilt = 0; @@ -36,7 +36,7 @@ dvo_catalog_unlock (&catalog); - // Naves_disk == 0 implies an empty catalog file + // Naverage_disk == 0 implies an empty catalog file // for only_match, skip empty catalogs - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_free (&catalog); continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/getptolemy.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/getptolemy.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/getptolemy.c (revision 38447) @@ -28,5 +28,5 @@ // set the parameters which guide catalog open/load/create catalog.filename = skylist[0].filename[i]; - catalog.catflags = LOAD_AVES | LOAD_MEAS; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE; catalog.Nsecfilt = 0; @@ -38,7 +38,7 @@ dvo_catalog_unlock (&catalog); - // Naves_disk == 0 implies an empty catalog file + // Naverage_disk == 0 implies an empty catalog file // for only_match, skip empty catalogs - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_free (&catalog); continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/rfits.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/rfits.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/gastro2/src/rfits.c (revision 38447) @@ -14,5 +14,5 @@ if (!gfits_fread_ftable (f, &table, "SMPFILE")) goto escape; - stars = gfits_table_get_SMPData (&table, &Nstars, NULL); + stars = gfits_table_get_SMPData (&table, &Nstars, NULL, NULL); if (!stars) { fprintf (stderr, "ERROR: failed to read stars\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/WriteImageFITS.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/WriteImageFITS.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/WriteImageFITS.c (revision 38447) @@ -36,5 +36,5 @@ table.header = &theader; - gfits_table_set_SMPData (&table, NULL, Nstars); + gfits_table_set_SMPData (&table, NULL, Nstars, TRUE); gfits_fwrite_Theader (f, &theader); gfits_fwrite_table (f, &table); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageExtract.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageExtract.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageExtract.c (revision 38447) @@ -14,4 +14,5 @@ /*** update the image table ***/ /* setup image table format and lock */ + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); @@ -31,5 +32,5 @@ // convert database table to internal structure - images = gfits_table_get_Image (&db.ftable, &Nimages, &db.swapped); + images = gfits_table_get_Image (&db.ftable, &Nimages, &db.scaledValue, &db.nativeOrder); if (!images) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageOverlaps.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageOverlaps.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImageOverlaps.c (revision 38447) @@ -17,4 +17,5 @@ /*** update the image table ***/ /* setup image table format and lock */ + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); @@ -34,5 +35,5 @@ // convert database table to internal structure - dbImages = gfits_table_get_Image (&db.ftable, &NdbImages, &db.swapped); + dbImages = gfits_table_get_Image (&db.ftable, &NdbImages, &db.scaledValue, &db.nativeOrder); if (!dbImages) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImagesAtCoords.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 38447) @@ -43,4 +43,5 @@ /*** update the image table ***/ /* setup image table format and lock */ + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); @@ -60,5 +61,5 @@ // convert database table to internal structure - dbImages = gfits_table_get_Image (&db.ftable, &NdbImages, &db.swapped); + dbImages = gfits_table_get_Image (&db.ftable, &NdbImages, &db.scaledValue, &db.nativeOrder); if (!dbImages) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/getstar.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/getstar.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/getstar.c (revision 38447) @@ -28,5 +28,5 @@ output.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data output.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - output.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; + output.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; output.Nsecfilt = GetPhotcodeNsecfilt (); @@ -63,6 +63,6 @@ catalog.filename = skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_SECF; - catalog.catflags |= needMeas ? LOAD_MEAS : SKIP_MEAS; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; + catalog.catflags |= needMeas ? DVO_LOAD_MEASURE : DVO_SKIP_MEASURE; // an error exit status here is a significant error @@ -71,5 +71,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); @@ -95,6 +95,6 @@ catalog.filename = skylist[0].filename[i]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_SECF; - catalog.catflags |= needMeas ? LOAD_MEAS : SKIP_MEAS; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; + catalog.catflags |= needMeas ? DVO_LOAD_MEASURE : DVO_SKIP_MEASURE; // an error exit status here is a significant error @@ -104,5 +104,5 @@ } /* skip empty catalogs */ - if (!catalog.Naves_disk) continue; + if (!catalog.Naverage_disk) continue; stars = select_by_image (&catalog, &image, 0, 0, stars, &Nstars); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/select_by_region.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/select_by_region.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/select_by_region.c (revision 38447) @@ -162,5 +162,5 @@ output[0].Naverage = Nave; output[0].Nmeasure = Nmeas; - output[0].Nsecf_mem = Nave*Nsecfilt; + output[0].Nsecfilt_mem = Nave*Nsecfilt; fprintf (stderr, "output catalog has "OFF_T_FMT" stars ("OFF_T_FMT" measures, %d secfilt)\n", Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_0.c (revision 38447) @@ -111,5 +111,5 @@ ftable.header = &theader; - gfits_table_set_Getstar_PS1_DEV_0 (&ftable, output, Noutput); + gfits_table_set_Getstar_PS1_DEV_0 (&ftable, output, Noutput, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_1.c (revision 38447) @@ -114,5 +114,5 @@ ftable.header = &theader; - gfits_table_set_Getstar_PS1_DEV_1 (&ftable, output, Noutput); + gfits_table_set_Getstar_PS1_DEV_1 (&ftable, output, Noutput, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/getstar/src/write_getstar_ps1_dev_2.c (revision 38447) @@ -120,5 +120,5 @@ ftable.header = &theader; - gfits_table_set_Getstar_PS1_DEV_2 (&ftable, output, Noutput); + gfits_table_set_Getstar_PS1_DEV_2 (&ftable, output, Noutput, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imclean/src/wfits.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imclean/src/wfits.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imclean/src/wfits.c (revision 38447) @@ -19,5 +19,5 @@ table.header = &theader; - gfits_table_set_SMPData (&table, stars, Nstars); + gfits_table_set_SMPData (&table, stars, Nstars, TRUE); gfits_write_header (filename, header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/delete.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/delete.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/delete.c (revision 38447) @@ -41,5 +41,5 @@ /** we may later want to pull this out and put it elsewhere **/ /** free db[0].theader, db[0].table.buffer? **/ - gfits_table_set_DetReg (&db[0].ftable, subset, Nsubset); + gfits_table_set_DetReg (&db[0].ftable, subset, Nsubset, TRUE); gfits_db_save (db); gfits_db_close (db); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/output.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/output.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/detrend/output.c (revision 38447) @@ -50,5 +50,5 @@ ftable.header = &theader; - gfits_table_set_DetReg (&ftable, subset, Nmatch); + gfits_table_set_DetReg (&ftable, subset, Nmatch, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imphot/dumpfits.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imphot/dumpfits.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imphot/dumpfits.c (revision 38447) @@ -34,5 +34,5 @@ ftable.header = &theader; - // gfits_table_set_Image (&ftable, subset, Nmatch); + // gfits_table_set_Image (&ftable, subset, Nmatch, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/delete.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/delete.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/delete.c (revision 38447) @@ -41,5 +41,5 @@ /** we may later want to pull this out and put it elsewhere **/ /** free db[0].theader, db[0].table.buffer? **/ - gfits_table_set_RegImage (&db[0].ftable, subset, Nsubset); + gfits_table_set_RegImage (&db[0].ftable, subset, Nsubset, TRUE); gfits_db_save (db); gfits_db_close (db); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/imregclient.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/imregclient.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/imregclient.c (revision 38447) @@ -66,7 +66,7 @@ if (NoReg) dump_data (image, Nentry); + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, TempDB)) { @@ -77,5 +77,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_RegImage (&db.ftable, NULL, 0); + gfits_table_set_RegImage (&db.ftable, NULL, 0, TRUE); } else { if (!gfits_db_load (&db)) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/output.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/output.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/imreg/output.c (revision 38447) @@ -77,5 +77,5 @@ ftable.header = &theader; - gfits_table_set_RegImage (&ftable, subset, Nmatch); + gfits_table_set_RegImage (&ftable, subset, Nmatch, TRUE); gfits_fwrite_header (f, &header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/delete.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/delete.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/delete.c (revision 38447) @@ -38,5 +38,5 @@ } - gfits_table_set_PhotPars (&db[0].ftable, subset, Nsubset); + gfits_table_set_PhotPars (&db[0].ftable, subset, Nsubset, TRUE); gfits_db_save (db); gfits_db_close (db); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/output.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/output.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/photreg/output.c (revision 38447) @@ -55,5 +55,5 @@ ftable.header = &theader; - gfits_table_set_PhotPars (&ftable, subset, Nmatch); + gfits_table_set_PhotPars (&ftable, subset, Nmatch, TRUE); /* EXTNAME is set to ZERO_POINTS_3.0 by default */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/delete.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/delete.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/delete.c (revision 38447) @@ -39,5 +39,5 @@ /** we may later want to pull this out and put it elsewhere **/ /** free db[0].theader, db[0].table.buffer? **/ - gfits_table_set_Spectrum (&db[0].ftable, subset, Nsubset); + gfits_table_set_Spectrum (&db[0].ftable, subset, Nsubset, TRUE); gfits_db_save (db); gfits_db_close (db); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/output.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/output.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/spreg/output.c (revision 38447) @@ -61,5 +61,5 @@ ftable.header = &theader; - gfits_table_set_Spectrum (&ftable, subset, Nmatch); + gfits_table_set_Spectrum (&ftable, subset, Nmatch, TRUE); gfits_fwrite_header (f, &header); @@ -96,5 +96,5 @@ /* create an empty table which we will fill in by hand */ ftable.header = &theader; - gfits_table_set_SpectrumASCII (&ftable, NULL, 0); + gfits_table_set_SpectrumASCII (&ftable, NULL, 0, TRUE); /* add data to table */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detregister.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detregister.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detregister.c (revision 38447) @@ -17,7 +17,7 @@ SaveEntry (argv[1], &newdata, descriptor.imageID); + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); dBFile = set_dBFile (); @@ -29,5 +29,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_DetReg (&db.ftable, NULL, 0); + gfits_table_set_DetReg (&db.ftable, NULL, 0, TRUE); } else { if (!gfits_db_load (&db)) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detsearch.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detsearch.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/detsearch.c (revision 38447) @@ -17,7 +17,7 @@ if (output.Criteria) PrintCriteria (); + gfits_db_init (&db); db.lockstate = (output.Modify || output.Delete) ? LCK_HARD : LCK_SOFT; db.timeout = 300.0; - gfits_db_init (&db); dBFile = set_dBFile (); @@ -35,5 +35,5 @@ if (!output.Modify && !output.Delete && !output.Altpath) gfits_db_close (&db); - detrend = gfits_table_get_DetReg (&db.ftable, &Ndetrend, &db.swapped); + detrend = gfits_table_get_DetReg (&db.ftable, &Ndetrend, &db.scaledValue, &db.nativeOrder); if (!detrend) { fprintf (stderr, "ERROR: failed to read detrend info\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imphotsearch.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imphotsearch.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imphotsearch.c (revision 38447) @@ -14,7 +14,7 @@ args (argc, argv); + gfits_db_init (&db); db.lockstate = (options.modify) ? LCK_HARD : LCK_SOFT; db.timeout = 300.0; - gfits_db_init (&db); /* don't create a new image table if not available */ @@ -32,5 +32,5 @@ if (!options.modify) gfits_db_close (&db); - image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregister.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregister.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregister.c (revision 38447) @@ -17,7 +17,7 @@ if (NoReg) goto skip_reg; + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, ImageDB)) { @@ -28,5 +28,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_RegImage (&db.ftable, NULL, 0); + gfits_table_set_RegImage (&db.ftable, NULL, 0, TRUE); } else { if (!gfits_db_load (&db)) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregtable.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregtable.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imregtable.c (revision 38447) @@ -33,5 +33,5 @@ gfits_read_ftable (infile, &ftable, "IMAGE_DATABASE"); - image = gfits_table_get_RegImage (&ftable, &Nimage, NULL); + image = gfits_table_get_RegImage (&ftable, &Nimage, NULL, NULL); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -40,7 +40,7 @@ /* load database table */ + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, ImageDB)) { @@ -51,5 +51,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_RegImage (&db.ftable, NULL, 0); + gfits_table_set_RegImage (&db.ftable, NULL, 0, TRUE); } else { if (!gfits_db_load (&db)) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imsearch.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imsearch.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imsearch.c (revision 38447) @@ -12,7 +12,7 @@ args (argc, argv); + gfits_db_init (&db); db.lockstate = (output.modify || output.delete) ? LCK_HARD : LCK_SOFT; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, ImageDB)) { @@ -29,5 +29,5 @@ if (!output.modify && !output.delete) gfits_db_close (&db); - image = gfits_table_get_RegImage (&db.ftable, &Nimage, &db.swapped); + image = gfits_table_get_RegImage (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imstatreg.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imstatreg.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/imstatreg.c (revision 38447) @@ -40,7 +40,7 @@ } + gfits_db_init (&image_db); image_db.lockstate = LCK_HARD; image_db.timeout = 300.0; - gfits_db_init (&image_db); temp_db.lockstate = LCK_HARD; @@ -66,5 +66,5 @@ exit (1); } - subset = gfits_table_get_RegImage (&temp_db.ftable, &Nsubset, &temp_db.swapped); + subset = gfits_table_get_RegImage (&temp_db.ftable, &Nsubset, &temp_db.scaledValue, &temp_db.nativeOrder); if (!subset) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -96,5 +96,5 @@ we will just drop the temp db data */ } - image = gfits_table_get_RegImage (&image_db.ftable, &Nimage, &image_db.swapped); + image = gfits_table_get_RegImage (&image_db.ftable, &Nimage, &image_db.scaledValue, &image_db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photreg.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photreg.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photreg.c (revision 38447) @@ -12,7 +12,7 @@ regargs (argc, argv, &newdata); + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); filename = PhotDB; @@ -26,5 +26,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_PhotPars (&db.ftable, NULL, 0); + gfits_table_set_PhotPars (&db.ftable, NULL, 0, TRUE); /* EXTNAME is set to ZERO_POINTS_3.0 by default */ if (!strcmp (output.db, "trans")) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photsearch.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photsearch.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/photsearch.c (revision 38447) @@ -15,7 +15,7 @@ args (argc, argv); + gfits_db_init (&db); db.lockstate = (output.modify || output.delete) ? LCK_HARD : LCK_SOFT; db.timeout = 300.0; - gfits_db_init (&db); if (!strcmp (output.db, "phot")) { @@ -41,5 +41,5 @@ /* add test to EXTNAME? */ if (output.convert) { - photpars_old = gfits_table_get_PhotParsOld (&db.ftable, &Nphotpars, &db.swapped); + photpars_old = gfits_table_get_PhotParsOld (&db.ftable, &Nphotpars, &db.scaledValue, &db.nativeOrder); if (!photpars_old) { fprintf (stderr, "ERROR: failed to read photometry parameters\n"); @@ -48,5 +48,5 @@ photpars = PhotParsOld_to_PhotPars (photpars_old, Nphotpars); } else { - photpars = gfits_table_get_PhotPars (&db.ftable, &Nphotpars, &db.swapped); + photpars = gfits_table_get_PhotPars (&db.ftable, &Nphotpars, &db.scaledValue, &db.nativeOrder); if (!photpars) { fprintf (stderr, "ERROR: failed to read photometry parameters\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spregister.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spregister.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spregister.c (revision 38447) @@ -17,7 +17,7 @@ if (NoReg) exit (0); + gfits_db_init (&db); db.lockstate = LCK_HARD; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, SpectrumDB)) { @@ -28,5 +28,5 @@ if (db.dbstate == LCK_EMPTY) { gfits_db_create (&db); - gfits_table_set_Spectrum (&db.ftable, NULL, 0); + gfits_table_set_Spectrum (&db.ftable, NULL, 0, TRUE); } else { if (!gfits_db_load (&db)) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spsearch.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spsearch.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/imregister/src/spsearch.c (revision 38447) @@ -12,7 +12,7 @@ args (argc, argv); + gfits_db_init (&db); db.lockstate = (output.modify || output.delete) ? LCK_HARD : LCK_SOFT; db.timeout = 300.0; - gfits_db_init (&db); if (!gfits_db_lock (&db, SpectrumDB)) { @@ -29,5 +29,5 @@ if (!output.modify && !output.delete) gfits_db_close (&db); - spectrum = gfits_table_get_Spectrum (&db.ftable, &Nspectrum, &db.swapped); + spectrum = gfits_table_get_Spectrum (&db.ftable, &Nspectrum, &db.scaledValue, &db.nativeOrder); if (!spectrum) { fprintf (stderr, "ERROR: failed to read spectrum\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/Makefile.Targets =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/Makefile.Targets (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/Makefile.Targets (revision 38447) @@ -55,4 +55,6 @@ $(ASRC)/starpar-ps1-v5.$(ARCH).o \ $(ASRC)/starpar-ps1-sim.$(ARCH).o \ +$(ASRC)/galshape.$(ARCH).o \ +$(ASRC)/galshape-ps1-v5.$(ARCH).o \ $(ASRC)/missing.$(ARCH).o \ $(ASRC)/photcode.$(ARCH).o \ @@ -165,4 +167,6 @@ $(AINC)/starpar-ps1-v5.h \ $(AINC)/starpar-ps1-sim.h \ +$(AINC)/galshape.h \ +$(AINC)/galshape-ps1-v5.h \ $(AINC)/missing.h \ $(AINC)/photcode.h \ Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.c (revision 38447) @@ -27,5 +27,6 @@ /*** add test of EXTNAME and header-defined columns? ***/ /* return internal structure representation */ -$STRUCT *gfits_table_get_$STRUCT (FTable *ftable, off_t *Ndata, char *swapped) { +/* scaledData & nativeBytes describe the current state of the ftable.buffer */ +$STRUCT *gfits_table_get_$STRUCT (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder) { int Ncols; @@ -40,15 +41,21 @@ *Ndata = ftable[0].header[0].Naxis[1]; data = ($STRUCT *) ftable[0].buffer; - if ((swapped == NULL) || (*swapped == FALSE)) { + + // if the pointer is not passed, we need to swap + if (!nativeOrder || !*nativeOrder) { if (!gfits_convert_$STRUCT (data, sizeof ($STRUCT), *Ndata)) { return NULL; } + if (nativeOrder) *nativeOrder = TRUE; + } + // if the pointer is not passed, we need to scale + if (!scaledValue || !*scaledValue) { gfits_table_scale_data (ftable); - if (swapped != NULL) *swapped = TRUE; + if (scaledValue) *scaledValue = TRUE; } return (data); } -int gfits_table_set_$STRUCT (FTable *ftable, $STRUCT *data, off_t Ndata) { +int gfits_table_set_$STRUCT (FTable *ftable, $STRUCT *data, off_t Ndata, int swapFromNative) { Header *header; @@ -67,5 +74,7 @@ /* add data values */ if (!gfits_table_scale_data (ftable)) return (FALSE); - if (!gfits_convert_$STRUCT (data, sizeof ($STRUCT), Ndata)) return (FALSE); + if (swapFromNative) { + if (!gfits_convert_$STRUCT (data, sizeof ($STRUCT), Ndata)) return (FALSE); + } if (!gfits_add_rows (ftable, (char *) data, Ndata, sizeof ($STRUCT))) return (FALSE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/autocode.h (revision 38447) @@ -2,6 +2,6 @@ /** STRUCT DEFINITION **/ -$STRUCT *gfits_table_get_$STRUCT (FTable *table, off_t *Ndata, char *swapped); -int gfits_table_set_$STRUCT (FTable *ftable, $STRUCT *data, off_t Ndata); +$STRUCT *gfits_table_get_$STRUCT (FTable *table, off_t *Ndata, char *scaledValue, char *nativeOrder); +int gfits_table_set_$STRUCT (FTable *ftable, $STRUCT *data, off_t Ndata, int swapFromNative); int gfits_table_mkheader_$STRUCT (Header *header); int gfits_convert_$STRUCT ($STRUCT *data, off_t size, off_t nitems); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v1.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v1.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v1.d (revision 38447) @@ -26,5 +26,5 @@ FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements FIELD Nmissing, NMISSING, unsigned short, number of missings -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements FIELD measureOffset, OFF_MEASURE, uint32_t, offset to first psf measurement Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v2.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v2.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v2.d (revision 38447) @@ -31,5 +31,5 @@ FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements FIELD Nmissing, NMISSING, unsigned short, number of missings -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements FIELD measureOffset, OFF_MEASURE, uint32_t, offset to first psf measurement Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v3.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v3.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v3.d (revision 38447) @@ -31,5 +31,5 @@ FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements FIELD Nmissing, NMISSING, unsigned short, number of missings -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements FIELD measureOffset, OFF_MEASURE, uint32_t, offset to first psf measurement Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v4.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v4.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v4.d (revision 38447) @@ -34,5 +34,5 @@ FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements FIELD Nmissing, NMISSING, unsigned short, number of missings -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphotd measurements FIELD measureOffset, OFF_MEASURE, uint32_t, offset to first psf measurement Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v5.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v5.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average-ps1-v5.d (revision 38447) @@ -41,12 +41,12 @@ FIELD Nlensing, NLENSING, unsigned short, number of lensing measurements FIELD Nlensobj, NLENSOBJ, unsigned short, number of lensing measurements -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements -FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement -FIELD missingOffset, OFF_MISSING, int, offset to first missing obs -FIELD lensingOffset, OFF_LENSING, int, offset to first lensing obs -FIELD lensobjOffset, OFF_LENSOBJ, int, offset to mean lensing data -FIELD starparOffset, OFF_STARPAR, int, offset to stellar parameter data -FIELD extendOffset, OFF_EXTEND, int, offset to extended object entry +FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement +FIELD missingOffset, OFF_MISSING, int, offset to first missing obs +FIELD lensingOffset, OFF_LENSING, int, offset to first lensing obs +FIELD lensobjOffset, OFF_LENSOBJ, int, offset to mean lensing data +FIELD starparOffset, OFF_STARPAR, int, offset to stellar parameter data +FIELD galphotOffset, OFF_GALPHOT, int, offset to galphot object entry FIELD refColorBlue, REF_COLOR_BLUE, float, color of astrometry ref stars Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average.d (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/average.d (revision 38447) @@ -41,12 +41,12 @@ FIELD Nlensing, NLENSING, unsigned short, number of lensing measurements FIELD Nlensobj, NLENSOBJ, unsigned short, number of lensing measurements -FIELD Nextend, NEXTEND, unsigned short, number of extended measurements +FIELD Ngalphot, NGALPHOT, unsigned short, number of galphot measurements -FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement -FIELD missingOffset, OFF_MISSING, int, offset to first missing obs -FIELD lensingOffset, OFF_LENSING, int, offset to first lensing obs -FIELD lensobjOffset, OFF_LENSOBJ, int, offset to mean lensing data -FIELD starparOffset, OFF_STARPAR, int, offset to stellar parameter data -FIELD extendOffset, OFF_EXTEND, int, offset to extended object entry +FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement +FIELD missingOffset, OFF_MISSING, int, offset to first missing obs +FIELD lensingOffset, OFF_LENSING, int, offset to first lensing obs +FIELD lensobjOffset, OFF_LENSOBJ, int, offset to mean lensing data +FIELD starparOffset, OFF_STARPAR, int, offset to stellar parameter data +FIELD galphotOffset, OFF_GALPHOT, int, offset to extended object entry FIELD refColorBlue, REF_COLOR_BLUE, float, color of astrometry ref stars Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape-ps1-v5.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape-ps1-v5.d (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape-ps1-v5.d (revision 38447) @@ -0,0 +1,33 @@ +STRUCT GalPhot_PS1_V5 +EXTNAME DVO_GALPHOT_PS1_V5 +TYPE BINTABLE +SIZE 72 +DESCRIPTION DVO Galaxy Shape Table + +FIELD Xfit, XFIT, float, centroid for fit +FIELD Yfit, YFIT, float, centroid for fit + +FIELD mag, MAG, float, galaxy magnitude +FIELD magErr, MAG_ERR, float, galaxy magnitude error + +FIELD majorAxis, MAJOR_AXIS, float, major axis size +FIELD minorAxis, MINOR_AXIS, float, minor axis size + +FIELD majorAxisErr, MAJOR_AXIS_ERR, float, major axis size error +FIELD minorAxisErr, MINOR_AXIS_ERR, float, minor axis size error + +FIELD theta, THETA, float, angle +FIELD theta_err, THETA_ERR, float, angle error + +FIELD index, INDEX, float, sersic index (if relevant) +FIELD chisq, CHISQ, float, fit chisq + +FIELD Npix, NPIX, float, fitted pixels + +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD detID, DET_ID, unsigned int, detection ID +FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID + +FIELD photcode, PHOTCODE, short +FIELD modelType, MODEL_TYPE, short, mean lensing smear object E2 Index: /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape.d =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape.d (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libautocode/def/galshape.d (revision 38447) @@ -0,0 +1,28 @@ +STRUCT GalPhot +EXTNAME DVO_GALPHOT +TYPE BINTABLE +SIZE 72 +DESCRIPTION DVO Galaxy Shape Table + +FIELD Xfit, XFIT, float, centroid for fit +FIELD Yfit, YFIT, float, centroid for fit +FIELD mag, MAG, float, galaxy magnitude +FIELD magErr, MAG_ERR, float, galaxy magnitude error +FIELD majorAxis, MAJOR_AXIS, float, major axis size +FIELD minorAxis, MINOR_AXIS, float, minor axis size +FIELD majorAxisErr, MAJOR_AXIS_ERR, float, major axis size error +FIELD minorAxisErr, MINOR_AXIS_ERR, float, minor axis size error +FIELD theta, THETA, float, angle +FIELD theta_err, THETA_ERR, float, angle error +FIELD index, INDEX, float, sersic index (if relevant) +FIELD chisq, CHISQ, float, fit chisq + +FIELD Npix, NPIX, float, fitted pixels + +FIELD objID, OBJ_ID, unsigned int, unique ID for object in table +FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD detID, DET_ID, unsigned int, detection ID +FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID + +FIELD photcode, PHOTCODE, short +FIELD modelType, MODEL_TYPE, short, mean lensing smear object E2 Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/Makefile =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/Makefile (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/Makefile (revision 38447) @@ -10,4 +10,6 @@ MAN = $(HOME)/doc INC = $(HOME)/include +TEST = $(HOME)/test +TESTBIN = $(HOME)/test ASRC = $(AUTO)/src AINC = $(AUTO)/include @@ -153,4 +155,5 @@ rm -f $@ cp $< $@ + echo "done making autocode.h" $(AINC)/autocode.h: $(AINCS) $(ADEF)/autocode.h $(ADEF)/common.h Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/dvo.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/dvo.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/dvo.h (revision 38447) @@ -10,9 +10,12 @@ /* DVO table modes */ -typedef enum {DVO_MODE_UNDEF, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOTableMode; +typedef enum {DVO_MODE_UNDEF = 0, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOCatMode; + +/* DVO table modes */ +typedef enum {DVO_COMPRESS_NONE = 0, DVO_COMPRESS_NONE_1, DVO_COMPRESS_NONE_2, DVO_COMPRESS_AUTO, DVO_COMPRESS_GZIP_1, DVO_COMPRESS_GZIP_2, DVO_COMPRESS_RICE_1} DVOCatCompress; // /* DVO table formats */ typedef enum { - DVO_FORMAT_UNDEF, + DVO_FORMAT_UNDEF = 0, DVO_FORMAT_INTERNAL, DVO_FORMAT_ELIXIR, @@ -30,11 +33,32 @@ DVO_FORMAT_PS1_V4, DVO_FORMAT_PS1_V5, -} DVOTableFormat; +} DVOCatFormat; + +typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; + +/* catalog values to be loaded */ +typedef enum { + DVO_LOAD_NONE = 0x0000, + DVO_LOAD_AVERAGE = 0x0001, + DVO_LOAD_MEASURE = 0x0002, + DVO_LOAD_MISSING = 0x0004, + DVO_LOAD_SECFILT = 0x0008, + DVO_SKIP_AVERAGE = 0x0010, + DVO_SKIP_MEASURE = 0x0020, + DVO_SKIP_MISSING = 0x0040, + DVO_SKIP_SECFILT = 0x0080, + DVO_LOAD_LENSING = 0x0100, + DVO_LOAD_LENSOBJ = 0x0200, + DVO_SKIP_LENSING = 0x0400, + DVO_SKIP_LENSOBJ = 0x0800, + DVO_LOAD_STARPAR = 0x1000, + DVO_SKIP_STARPAR = 0x2000, + DVO_LOAD_GALPHOT = 0x4000, + DVO_SKIP_GALPHOT = 0x8000, +} DVOCatFlags; /* image data modes in RegImage */ typedef enum {T_UNDEF = -1, T_NONE, T_OBJECT, T_DARK, T_BIAS, T_FLAT, T_MASK, T_FRINGE, T_SCATTER, T_MODES, T_FRINGEPTS, T_ANY, N_TYPE} ElixirDetrendTypes; typedef enum {M_UNDEF = -1, M_NONE, M_MEF, M_SPLIT, M_SINGLE, M_CUBE, M_SLICE, M_MODES, N_MODE} ElixirDetrendModes; - -typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; // these are used as NAN for types of int values @@ -62,28 +86,13 @@ # define IMREG_DIST 0x01 /* image distributed, only imregister-3.0 */ -/* catalog values to be loaded */ -# define LOAD_NONE 0x0000 -# define LOAD_AVES 0x0001 -# define LOAD_MEAS 0x0002 -# define LOAD_MISS 0x0004 -# define LOAD_SECF 0x0008 -# define SKIP_AVES 0x0010 -# define SKIP_MEAS 0x0020 -# define SKIP_MISS 0x0040 -# define SKIP_SECF 0x0080 -# define LOAD_LENSING 0x0100 -# define LOAD_LENSOBJ 0x0200 -# define SKIP_LENSING 0x0400 -# define SKIP_LENSOBJ 0x0800 -# define LOAD_STARPAR 0x1000 -# define SKIP_STARPAR 0x2000 - /* photometry code types */ // # define PHOT_PRI 0x01 -# define PHOT_SEC 0x02 -# define PHOT_DEP 0x03 -# define PHOT_REF 0x04 -# define PHOT_ALT 0x05 /* never stored, only for look-ups */ -# define PHOT_MAG 0x06 /* generic magnitude; never stored */ +typedef enum { + PHOT_SEC = 0x02, + PHOT_DEP = 0x03, + PHOT_REF = 0x04, + PHOT_ALT = 0x05, /* never stored, only for look-ups */ + PHOT_MAG = 0x06, /* generic magnitude; never stored */ +} DVOPhotCodeTypes; /* Image.code values -- these values are 32 bit (as of PS1_V1) */ @@ -238,7 +247,8 @@ char lockstate; char mode; /* what data storage mode is used for disk file? */ - char format; /* what data format is used for disk file? */ + DVOCatFormat format; /* what data format is used for disk file? */ char virtual; /* is table in ftable or vtable? */ - char swapped; /* is table in internal byte-order? */ + char nativeOrder; /* is table in internal byte-order? */ + char scaledValue; /* is table in internal byte-order? */ } FITS_DB; @@ -500,5 +510,5 @@ unsigned short Nmeasure; // number of psf measurements unsigned short Nmissing; // number of missings - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of extended measurements uint32_t measureOffset; // offset to first psf measurement uint32_t missingOffset; // offset to first missing obs @@ -720,10 +730,10 @@ unsigned short Nlensing; // number of lensing measurements unsigned short Nlensobj; // number of lensing measurements - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of galphot measurements int measureOffset; // offset to first psf measurement int missingOffset; // offset to first missing obs int lensingOffset; // offset to first lensing obs int lensobjOffset; // offset to mean lensing data - int extendOffset; // offset to extended object entry + int galphotOffset; // offset to extended object entry int starparOffset; // offset to stellar parameter data float refColorBlue; // color of astrometry ref stars @@ -758,12 +768,13 @@ // lensing data (optional?) - Lensing *lensing; - Lensobj *lensobj; - StarPar *starpar; + Lensing *lensing; + Lensobj *lensobj; + StarPar *starpar; + GalPhot *galphot; int Nsecfilt; /* number of secfilt entries for each average entry */ - off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Nsecf_mem; /* current number of each component in memory */ - off_t Naves_disk, Nmeas_disk, Nmiss_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Nsecf_disk; /* current number of each component on disk */ - off_t Naves_off, Nmeas_off, Nmiss_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Nsecf_off; /* index of first loaded data value */ + off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot, Nsecfilt_mem; /* current number of each component in memory */ + off_t Naverage_disk, Nmeasure_disk, Nmissing_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalphot_disk, Nsecfilt_disk; /* current number of each component on disk */ + off_t Naverage_off, Nmeasure_off, Nmissing_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Ngalphot_off, Nsecfilt_off; /* index of first loaded data value */ // note that we use these for the full-sky relphot analysis @@ -781,10 +792,11 @@ /* pointers to split data files */ - struct Catalog *measure_catalog; /* measure catalog data (split) */ - struct Catalog *missing_catalog; /* missing catalog data (split) */ - struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ - struct Catalog *lensing_catalog; /* lensing catalog data (split) */ - struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ - struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *measure_catalog; /* measure catalog data (split) */ + struct Catalog *missing_catalog; /* missing catalog data (split) */ + struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ + struct Catalog *lensing_catalog; /* lensing catalog data (split) */ + struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ + struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *galphot_catalog; /* galphot catalog data (split) */ unsigned int objID; @@ -793,9 +805,11 @@ /* extra catalog information */ char lockmode; - char catmode; /* storage mode (raw, mef, split, mysql) */ - char catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatMode catmode; /* storage mode (raw, mef, split, mysql) */ + DVOCatFormat catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatCompress catcompress; // output compress mode + DVOCatFlags catflags; /* choices to be loaded */ + int sorted; /* is measure table average-sorted? (NOTE this is an int only because gfits_scan %t requires it) */ - - short catflags; /* choices to be loaded */ + /* pointers for data manipulation */ @@ -946,6 +960,8 @@ int dvo_catalog_save_complete (Catalog *catalog, char VERBOSE); int dvo_catalog_update (Catalog *catalog, char VERBOSE); -int dvo_catalog_catformat (char *catformat); -int dvo_catalog_catmode (char *catmode); +DVOCatFormat dvo_catalog_catformat (char *catformat); +DVOCatMode dvo_catalog_catmode (char *catmode); +DVOCatCompress dvo_catalog_catcompress (char *catcompress); +char *dvo_catalog_compress_string (DVOCatCompress catcompress); void dvo_catalog_test (Catalog *catalog, int halt); @@ -974,22 +990,28 @@ int WriteRawSecFilt (FILE *f, SecFilt *secfilt, off_t Nsecfilt, char format); -DVOTableFormat FtableGetFormat (FTable *ftable); - -Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary); -Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format); -SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format); -Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format); -Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format); -StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format); -int FtableToImage (FTable *ftable, Header *theader, char *format); - -int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary); -int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format); -int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format); -int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format); -int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format); -int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format); -int ImageToFtable (FTable *ftable, Header *theader, char format); -int ImageToVtable (VTable *vtable, Header *theader, char format); +DVOCatFormat FtableGetFormat (FTable *ftable); + +Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeOrder); + +Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeOrder); +Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes); +SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeOrder); +Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeOrder); +Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeOrder); +StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeOrder); +GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeOrder); + +int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format); + +int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative); +int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative); +int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative); +int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative); +int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative); +int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative); +int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative); + +int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format); +int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format); # include "loneos_defs.h" @@ -1023,5 +1045,5 @@ int gfits_table_set_Image (FTable *ftable); int gfits_table_mkheader_Image (Header *header); -Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped); +Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder); /* flatcorr APIs */ @@ -1087,5 +1109,7 @@ void InitHost (HostInfo *host); -HostTableGroup *HostTableGroups (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax); + int HostTableGroupWaitJobsGetIO (HostTableGroup *table, char *file, int lineno, int VERBOSE); @@ -1124,4 +1148,5 @@ void dvo_lensobj_init (Lensobj *lensobj, int toZero); void dvo_starpar_init (StarPar *starpar); +void dvo_galphot_init (GalPhot *galphot); void InitRegionHosts (RegionHostInfo *hosts, int Nhosts, int NHOSTS); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/ps1_v5_defs.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/ps1_v5_defs.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/include/ps1_v5_defs.h (revision 38447) @@ -23,4 +23,7 @@ StarPar_PS1_V5 *StarParInternalTo_PS1_V5 (StarPar *in, off_t Nvalues); +GalPhot *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues); +GalPhot_PS1_V5 *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues); + PhotCode *PhotCode_PS1_V5_To_Internal (PhotCode_PS1_V5 *in, off_t Nvalues); PhotCode_PS1_V5 *PhotCode_Internal_To_PS1_V5 (PhotCode *in, off_t Nvalues); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/AstromOffsetMapIO.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/AstromOffsetMapIO.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/AstromOffsetMapIO.c (revision 38447) @@ -39,5 +39,5 @@ return (NULL); } - AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL); + AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL); if (!map_disk) { fprintf (stderr, "ERROR: failed to read Astrom Offset Map\n"); @@ -78,5 +78,5 @@ ftable.header = &theader; - gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap); + gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap, TRUE); f = fopen (filename, "w"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/HostTable.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/HostTable.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/HostTable.c (revision 38447) @@ -155,5 +155,5 @@ // split a host table into Ngroups, each with a unique set of hosts -HostTableGroup *HostTableGroups (HostTable *table, int *ngroups) { +HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups) { // identify the unique host names and the number of times they each exist @@ -210,4 +210,29 @@ foundHost[k] = TRUE; } + } + } + *ngroups = Ngroups; + return groups; +} + +// split a host table into Ngroups, each with a unique set of hosts +HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax) { + + int i, j; + + int Ngroups = (table->Nhosts % Nmax) ? (int)(table->Nhosts / Nmax + 1) : table->Nhosts / Nmax; + + HostTableGroup *groups = NULL; + ALLOCATE (groups, HostTableGroup, Ngroups); + + // in each group, attempt to add one of each unique host + int k = 0; + for (i = 0; i < Ngroups; i++) { + groups[i].Nhosts = 0; + ALLOCATE (groups[i].hosts, HostInfo *, Nmax); + for (j = 0; j < Nmax; j++) { + groups[i].hosts[groups[i].Nhosts] = &table->hosts[k]; + groups[i].Nhosts ++; + k++; } } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadImages.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadImages.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadImages.c (revision 38447) @@ -65,5 +65,5 @@ } - image = gfits_table_get_Image (&db.ftable, Nimage, &db.swapped); + image = gfits_table_get_Image (&db.ftable, Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadPhotcodesFITS.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 38447) @@ -15,7 +15,7 @@ /* XXX choose more sensible lock timeouts! */ + gfits_db_init (&db); db.timeout = 60.0; db.lockstate = LCK_SOFT; - gfits_db_init (&db); /* does this mean the db is empty, non-existent, or has access errors? */ @@ -43,5 +43,5 @@ # define CONVERT_FORMAT(NAME, FORMAT) \ if (!strcmp (extname, NAME)) { \ - PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.swapped); \ + PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.scaledValue, &db.nativeOrder); \ if (!photcode_input) { \ fprintf (stderr, "ERROR: failed to read photcodes in LoadPhotcodesFITS.c\n"); \ Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/SavePhotcodesFITS.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 38447) @@ -17,7 +17,7 @@ /* XXX choose more sensible lock timeouts! */ + gfits_db_init (&db); db.lockstate = LCK_XCLD; db.timeout = 10.0; - gfits_db_init (&db); /* does this mean the db is empty, non-existent, or has access errors? */ @@ -34,5 +34,5 @@ /* convert FITS format data to internal format (byteswaps & EXTNAME) */ if (!gfits_db_create (&db)) return (FALSE); - if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE); + if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode, TRUE)) return (FALSE); if (!gfits_db_save (&db)) return (FALSE); if (!gfits_db_close (&db)) return (FALSE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog.c (revision 38447) @@ -42,5 +42,5 @@ } -int dvo_catalog_catformat (char *catformat) { +DVOCatFormat dvo_catalog_catformat (char *catformat) { /* set the specified CATFORMAT */ @@ -62,5 +62,5 @@ } -int dvo_catalog_catmode (char *catmode) { +DVOCatMode dvo_catalog_catmode (char *catmode) { /* set the specified CATMODE */ @@ -69,4 +69,49 @@ if (!strcasecmp (catmode, "SPLIT")) return (DVO_MODE_SPLIT); return (DVO_MODE_UNDEF); +} + +DVOCatCompress dvo_catalog_catcompress (char *catcompress) { + + /* set the specified CATMODE */ + if (!strcasecmp (catcompress, "NONE")) return (DVO_COMPRESS_NONE); + if (!strcasecmp (catcompress, "AUTO")) return (DVO_COMPRESS_AUTO); + if (!strcasecmp (catcompress, "NONE_1")) return (DVO_COMPRESS_NONE_1); + if (!strcasecmp (catcompress, "NONE_2")) return (DVO_COMPRESS_NONE_2); + if (!strcasecmp (catcompress, "GZIP_1")) return (DVO_COMPRESS_GZIP_1); + if (!strcasecmp (catcompress, "GZIP_2")) return (DVO_COMPRESS_GZIP_2); + if (!strcasecmp (catcompress, "RICE_1")) return (DVO_COMPRESS_RICE_1); + return (DVO_COMPRESS_NONE); +} + +// we can return a static string here unless we run multiple outputs threads at once +static char compress_string[16]; +char *dvo_catalog_compress_string (DVOCatCompress catcompress) { + + switch (catcompress) { + case DVO_COMPRESS_NONE: + strcpy (compress_string, "NONE"); + break; + case DVO_COMPRESS_AUTO: + strcpy (compress_string, "AUTO"); + break; + case DVO_COMPRESS_GZIP_1: + strcpy (compress_string, "GZIP_1"); + break; + case DVO_COMPRESS_GZIP_2: + strcpy (compress_string, "GZIP_2"); + break; + case DVO_COMPRESS_NONE_1: + strcpy (compress_string, "NONE_1"); + break; + case DVO_COMPRESS_NONE_2: + strcpy (compress_string, "NONE_2"); + break; + case DVO_COMPRESS_RICE_1: + strcpy (compress_string, "RICE_1"); + break; + default: + myAbort ("error in compress option"); + } + return compress_string; } @@ -131,5 +176,5 @@ average->Nlensobj = 0; average->Nstarpar = 0; - average->Nextend = 0; + average->Ngalphot = 0; average->measureOffset = -1; @@ -138,5 +183,5 @@ average->lensobjOffset = -1; average->starparOffset = -1; - average->extendOffset = -1; + average->galphotOffset = -1; average->refColorBlue = NAN; @@ -477,4 +522,36 @@ // init all data, or just catalog data +void dvo_galphot_init (GalPhot *galphot) { + galphot->Xfit = NAN; + galphot->Yfit = NAN; + galphot->mag = NAN; + galphot->magErr = NAN; + galphot->majorAxis = NAN; + galphot->minorAxis = NAN; + galphot->majorAxisErr = NAN; + galphot->minorAxisErr = NAN; + galphot->theta = NAN; + galphot->theta_err = NAN; + galphot->index = NAN; + galphot->chisq = NAN; + galphot->Npix = 0; + + galphot->detID = -1; + galphot->objID = -1; + galphot->catID = -1; + galphot->imageID = -1; + + galphot->photcode = 0; + galphot->modelType = 0; +} + +# define INIT_TABLE(TABLE) \ + catalog[0].TABLE = NULL; \ + catalog[0].N##TABLE = 0; \ + catalog[0].N##TABLE##_disk = 0; \ + catalog[0].N##TABLE##_off = 0; \ + catalog[0].TABLE##_catalog = NULL; + +// init all data, or just catalog data void dvo_catalog_init (Catalog *catalog, int complete) { @@ -484,68 +561,41 @@ catalog[0].filename = NULL; - catalog[0].lockmode = 0; - catalog[0].catmode = 0; - catalog[0].catformat = 0; - catalog[0].catflags = 0; - catalog[0].Nsecfilt = 0; + catalog[0].lockmode = 0; + catalog[0].catmode = DVO_MODE_UNDEF; + catalog[0].catformat = DVO_FORMAT_UNDEF; + catalog[0].catcompress = DVO_COMPRESS_NONE; + catalog[0].catflags = DVO_LOAD_NONE; + catalog[0].Nsecfilt = 0; } gfits_init_header (&catalog[0].header); - + // the following describe the catalog files on disk catalog[0].average = NULL; - catalog[0].measure = NULL; - catalog[0].missing = NULL; + catalog[0].Naverage = 0; + catalog[0].Naverage_disk = 0; + catalog[0].Naverage_off = 0; + // average lacks average_catalog + catalog[0].secfilt = NULL; - - catalog[0].lensing = NULL; - catalog[0].lensobj = NULL; - - catalog[0].starpar = NULL; + catalog[0].Nsecfilt_mem = 0; + catalog[0].Nsecfilt_disk = 0; + catalog[0].Nsecfilt_off = 0; + catalog[0].secfilt_catalog = NULL; + // secfilt uses Nsecfilt_mem not Nsecfilt catalog[0].averageT = NULL; catalog[0].measureT = NULL; + + INIT_TABLE(measure); + INIT_TABLE(missing); + INIT_TABLE(lensing); + INIT_TABLE(lensobj); + INIT_TABLE(starpar); + INIT_TABLE(galphot); catalog[0].objID = 0; catalog[0].catID = 0; catalog[0].sorted = 0; - - catalog[0].Naverage = 0; - catalog[0].Nmeasure = 0; - catalog[0].Nmissing = 0; - catalog[0].Nsecf_mem = 0; - - catalog[0].Nlensing = 0; - catalog[0].Nlensobj = 0; - - catalog[0].Nstarpar = 0; - - catalog[0].Naves_disk = 0; - catalog[0].Nmeas_disk = 0; - catalog[0].Nmiss_disk = 0; - catalog[0].Nsecf_disk = 0; - - catalog[0].Nlensing_disk = 0; - catalog[0].Nlensobj_disk = 0; - - catalog[0].Nstarpar_disk = 0; - - catalog[0].Naves_off = 0; - catalog[0].Nmeas_off = 0; - catalog[0].Nmiss_off = 0; - catalog[0].Nsecf_off = 0; - - catalog[0].Nlensing_off = 0; - catalog[0].Nlensobj_off = 0; - - catalog[0].Nstarpar_off = 0; - - /* pointers to SPLIT data files */ - catalog[0].measure_catalog = NULL; - catalog[0].missing_catalog = NULL; - catalog[0].secfilt_catalog = NULL; - catalog[0].lensing_catalog = NULL; - catalog[0].lensobj_catalog = NULL; - catalog[0].starpar_catalog = NULL; /* pointers for data manipulation */ @@ -579,5 +629,8 @@ if (catalog[0].f == NULL) return (DVO_CAT_OPEN_FAIL); - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { + perror ("fseeko: "); + exit (1); + } return (DVO_CAT_OPEN_OK); } @@ -591,5 +644,6 @@ if (fflush (catalog[0].f)) { - fprintf (stderr, "failed to flush file\n"); + perror ("fflush: "); + fprintf (stderr, "failed to flush file %s\n", catalog[0].filename); return FALSE; } @@ -618,4 +672,5 @@ if (catalog[0].lensobj_catalog) { if (!dvo_catalog_unlock (catalog[0].lensobj_catalog)) { fprintf (stderr, "failed to unlock lensobj\n"); return (0); }} if (catalog[0].starpar_catalog) { if (!dvo_catalog_unlock (catalog[0].starpar_catalog)) { fprintf (stderr, "failed to unlock starpar\n"); return (0); }} + if (catalog[0].galphot_catalog) { if (!dvo_catalog_unlock (catalog[0].galphot_catalog)) { fprintf (stderr, "failed to unlock galphot\n"); return (0); }} } return (1); @@ -860,5 +915,5 @@ catalog[0].secfilt = outsec; catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_mem = Nsecfilt * catalog[0].Naverage; + catalog[0].Nsecfilt_mem = Nsecfilt * catalog[0].Naverage; } return (TRUE); @@ -898,4 +953,9 @@ dvo_catalog_free (catalog[0].starpar_catalog); free (catalog[0].starpar_catalog); + } + if (catalog[0].galphot_catalog) { + free (catalog[0].galphot_catalog[0].filename); + dvo_catalog_free (catalog[0].galphot_catalog); + free (catalog[0].galphot_catalog); } } @@ -925,5 +985,5 @@ if (catalog[0].secfilt) { free (catalog[0].secfilt); - catalog[0].Nsecf_mem = 0; + catalog[0].Nsecfilt_mem = 0; catalog[0].secfilt = NULL; } @@ -943,4 +1003,9 @@ catalog[0].starpar = NULL; } + if (catalog[0].galphot) { + free (catalog[0].galphot); + catalog[0].Ngalphot = 0; + catalog[0].galphot = NULL; + } if (catalog[0].nOwn_t) { free (catalog[0].nOwn_t); catalog[0].nOwn_t = NULL; } if (catalog[0].found_t) { free (catalog[0].found_t); catalog[0].found_t = NULL; } @@ -949,5 +1014,5 @@ /* - mode : items to read (LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF) + mode : items to read (DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT) format : what table structure on disk (INTERNAL, LONEOS, etc, ) style : raw, mef, split, mysql @@ -982,5 +1047,5 @@ // skip empty cpt files - if (primary && !catalog->Naves_disk) { + if (primary && !catalog->Naverage_disk) { return TRUE; } @@ -1064,4 +1129,14 @@ } } + if (catalog[0].galphot_catalog != NULL) { + if (catalog[0].Ngalphot_disk == 0) { + // need to relock (and re-open) file for close elsewhere + status = dvo_catalog_lock (catalog[0].galphot_catalog, lockmode); + } else { + if (!dvo_catalog_backup (catalog[0].galphot_catalog, FALSE)) { + return FALSE; + } + } + } } return TRUE; @@ -1071,5 +1146,5 @@ int dvo_catalog_unlink_backup (Catalog *catalog, int primary) { - if (primary && !catalog->Naves_disk) { + if (primary && !catalog->Naverage_disk) { // skip empty files (empty when read, but output may not be empty) return TRUE; @@ -1120,4 +1195,9 @@ } } + if ((catalog[0].galphot_catalog != NULL) && (catalog[0].Ngalphot_disk > 0)) { + if (!dvo_catalog_unlink_backup (catalog[0].galphot_catalog, FALSE)) { + return FALSE; + } + } } return TRUE; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_create.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 38447) @@ -1,4 +1,6 @@ # include # define DEBUG 0 + +Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename); // create a new dvo catalog file (if split, lock extra files as well?) @@ -9,8 +11,4 @@ // catalog[0].lockmode void dvo_catalog_create (SkyRegion *region, Catalog *catalog) { - - int length; - char *path, *root, *file, *line; - time_t now; if (DEBUG) fprintf (stderr, "new catalog file: %s\n", catalog[0].filename); @@ -29,84 +27,11 @@ if (catalog[0].catmode == DVO_MODE_SPLIT) { - path = pathname (catalog[0].filename); - root = filerootname (catalog[0].filename); - length = strlen(path) + strlen(root) + 6; - - /* define measure catalog file */ - ALLOCATE (catalog[0].measure_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].measure_catalog, TRUE); - - /* create basic data for measure catalog file */ - gfits_create_header (&catalog[0].measure_catalog[0].header); - - ALLOCATE (catalog[0].measure_catalog[0].filename, char, length); - - sprintf (catalog[0].measure_catalog[0].filename, "%s/%s.cpm", path, root); - file = filebasename (catalog[0].measure_catalog[0].filename); - gfits_modify (&catalog[0].header, "MEASURE", "%s", 1, file); - free (file); - - /* define missing catalog file */ - ALLOCATE (catalog[0].missing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].missing_catalog, TRUE); - - /* create basic data for missing catalog file */ - gfits_create_header (&catalog[0].missing_catalog[0].header); - ALLOCATE (catalog[0].missing_catalog[0].filename, char, length); - sprintf (catalog[0].missing_catalog[0].filename, "%s/%s.cpn", path, root); - file = filebasename (catalog[0].missing_catalog[0].filename); - gfits_modify (&catalog[0].header, "MISSING", "%s", 1, file); - free (file); - - /* define secfilt catalog file */ - ALLOCATE (catalog[0].secfilt_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].secfilt_catalog, TRUE); - - /* create basic data for secfilt catalog file */ - gfits_create_header (&catalog[0].secfilt_catalog[0].header); - ALLOCATE (catalog[0].secfilt_catalog[0].filename, char, length); - sprintf (catalog[0].secfilt_catalog[0].filename, "%s/%s.cps", path, root); - file = filebasename (catalog[0].secfilt_catalog[0].filename); - gfits_modify (&catalog[0].header, "SECFILT", "%s", 1, file); - free (file); - - /* define lensing catalog file */ - ALLOCATE (catalog[0].lensing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensing_catalog, TRUE); - - /* create basic data for lensing catalog file */ - gfits_create_header (&catalog[0].lensing_catalog[0].header); - ALLOCATE (catalog[0].lensing_catalog[0].filename, char, length); - sprintf (catalog[0].lensing_catalog[0].filename, "%s/%s.cpx", path, root); - file = filebasename (catalog[0].lensing_catalog[0].filename); - gfits_modify (&catalog[0].header, "LENSING", "%s", 1, file); - free (file); - - /* define lensobj catalog file */ - ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensobj_catalog, TRUE); - - /* create basic data for lensobj catalog file */ - gfits_create_header (&catalog[0].lensobj_catalog[0].header); - ALLOCATE (catalog[0].lensobj_catalog[0].filename, char, length); - sprintf (catalog[0].lensobj_catalog[0].filename, "%s/%s.cpy", path, root); - file = filebasename (catalog[0].lensobj_catalog[0].filename); - gfits_modify (&catalog[0].header, "LENSOBJ", "%s", 1, file); - free (file); - - /* define starpar catalog file */ - ALLOCATE (catalog[0].starpar_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].starpar_catalog, TRUE); - - /* create basic data for starpar catalog file */ - gfits_create_header (&catalog[0].starpar_catalog[0].header); - ALLOCATE (catalog[0].starpar_catalog[0].filename, char, length); - sprintf (catalog[0].starpar_catalog[0].filename, "%s/%s.cpz", path, root); - file = filebasename (catalog[0].starpar_catalog[0].filename); - gfits_modify (&catalog[0].header, "STARPAR", "%s", 1, file); - free (file); - - free (path); - free (root); + catalog[0].measure_catalog = dvo_catalog_create_subcat (catalog, "cpm", "MEASURE"); + catalog[0].missing_catalog = dvo_catalog_create_subcat (catalog, "cpn", "MISSING"); + catalog[0].secfilt_catalog = dvo_catalog_create_subcat (catalog, "cps", "SECFILT"); + catalog[0].lensing_catalog = dvo_catalog_create_subcat (catalog, "cpx", "LENSING"); + catalog[0].lensobj_catalog = dvo_catalog_create_subcat (catalog, "cpy", "LENSOBJ"); + catalog[0].starpar_catalog = dvo_catalog_create_subcat (catalog, "cpz", "STARPAR"); + catalog[0].galphot_catalog = dvo_catalog_create_subcat (catalog, "cpq", "GALPHOT"); // lock the additional split files @@ -136,4 +61,8 @@ exit (2); } + if (dvo_catalog_lock (catalog[0].galphot_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) { + fprintf (stderr, "error with file lock\n"); + exit (2); + } } @@ -150,6 +79,7 @@ /* write creation date in header */ + time_t now; ohana_str_to_time ("now", &now); - line = ohana_sec_to_date (now); + char *line = ohana_sec_to_date (now); gfits_modify (&catalog[0].header, "DATE", "%s", 1, line); free (line); @@ -163,4 +93,5 @@ ALLOCATE (catalog[0].lensobj, Lensobj, 1); ALLOCATE (catalog[0].starpar, StarPar, 1); + ALLOCATE (catalog[0].galphot, GalPhot, 1); /* setup secondary filters to match photcodes: @@ -170,4 +101,35 @@ } +Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename) { + + char *path = pathname (catalog[0].filename); + char *root = filerootname (catalog[0].filename); + int length = strlen(path) + strlen(root) + 6; + + Catalog *subcat; + + /* define subcat catalog file */ + ALLOCATE (subcat, Catalog, 1); + dvo_catalog_init (subcat, TRUE); + + /* create basic data for measure catalog file */ + gfits_create_header (&subcat->header); + + subcat->catcompress = catalog->catcompress; + + ALLOCATE (subcat->filename, char, length); + + sprintf (subcat->filename, "%s/%s.%s", path, root, ext); + + char *file = filebasename (subcat->filename); + gfits_modify (&catalog[0].header, tablename, "%s", 1, file); + + free (file); + free (path); + free (root); + + return subcat; +} + int dvo_catalog_set_range (Catalog *catalog) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_mef.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 38447) @@ -26,5 +26,5 @@ } - // NSTARS, average, Naves_disk + // NSTARS, average, Naverage_disk # define GET_TABLE_SIZES(HFIELD, CFIELD, DFIELD, REQUIRED) \ @@ -41,10 +41,11 @@ /* get the components and sizes from the header */ - GET_TABLE_SIZES ("NSTARS", average, Naves_disk, TRUE); - GET_TABLE_SIZES ("NMEAS", measure, Nmeas_disk, TRUE); - GET_TABLE_SIZES ("NMISS", missing, Nmiss_disk, FALSE); - GET_TABLE_SIZES ("NLENSING", lensing, Nlensing_disk, FALSE); - GET_TABLE_SIZES ("NLENSOBJ", lensobj, Nlensobj_disk, FALSE); - GET_TABLE_SIZES ("NSTARPAR", starpar, Nstarpar_disk, FALSE); + GET_TABLE_SIZES ("NSTARS", average, Naverage_disk, TRUE); + GET_TABLE_SIZES ("NMEAS", measure, Nmeasure_disk, TRUE); + GET_TABLE_SIZES ("NMISS", missing, Nmissing_disk, FALSE); + GET_TABLE_SIZES ("NLENSING", lensing, Nlensing_disk, FALSE); + GET_TABLE_SIZES ("NLENSOBJ", lensobj, Nlensobj_disk, FALSE); + GET_TABLE_SIZES ("NSTARPAR", starpar, Nstarpar_disk, FALSE); + GET_TABLE_SIZES ("NGALPHOT", galphot, Ngalphot_disk, FALSE); /** Nsecfilt is unusual: it does not list the number of data items in the table @@ -53,5 +54,5 @@ if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_disk = Naverage * Nsecfilt; + catalog[0].Nsecfilt_disk = Naverage * Nsecfilt; catalog[0].secfilt = NULL; @@ -70,5 +71,5 @@ /* read Average table data (or skip) */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table average data"); @@ -76,10 +77,10 @@ } /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); - } - catalog[0].Naverage = catalog[0].Naves_disk; - catalog[0].Naves_off = 0; + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE); + if (Naverage != catalog[0].Naverage_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naverage_disk); + } + catalog[0].Naverage = catalog[0].Naverage_disk; + catalog[0].Naverage_off = 0; } else { catalog[0].catformat = FtableGetFormat (&ftable); @@ -88,5 +89,5 @@ ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; - catalog[0].Naves_off = catalog[0].Naves_disk; + catalog[0].Naverage_off = catalog[0].Naverage_disk; } gfits_free_header (&header); @@ -99,15 +100,15 @@ } /* read Measure table data */ - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table measure data"); return (FALSE); } - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); - } - catalog[0].Nmeasure = catalog[0].Nmeas_disk; - catalog[0].Nmeas_off = 0; + catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat, FALSE); + if (Nmeasure != catalog[0].Nmeasure_disk) { + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeasure_disk); + } + catalog[0].Nmeasure = catalog[0].Nmeasure_disk; + catalog[0].Nmeasure_off = 0; } else { Nbytes = gfits_data_size (&header); @@ -115,5 +116,5 @@ ALLOCATE (catalog[0].measure, Measure, 1); catalog[0].Nmeasure = 0; - catalog[0].Nmeas_off = catalog[0].Nmeas_disk; + catalog[0].Nmeasure_off = catalog[0].Nmeasure_disk; } @@ -124,5 +125,5 @@ } /* read Missing table data */ - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table missing data"); @@ -130,14 +131,14 @@ } /* no conversions currently defined */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL); + catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL, NULL); if (!catalog[0].missing) { fprintf (stderr, "ERROR: failed to read missing\n"); exit (2); } - if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); - } - catalog[0].Nmissing = catalog[0].Nmiss_disk; - catalog[0].Nmiss_off = 0; + if (Nmissing != catalog[0].Nmissing_disk) { + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmissing_disk); + } + catalog[0].Nmissing = catalog[0].Nmissing_disk; + catalog[0].Nmissing_off = 0; } else { Nbytes = gfits_data_size (&header); @@ -145,5 +146,5 @@ ALLOCATE (catalog[0].missing, Missing, 1); catalog[0].Nmissing = 0; - catalog[0].Nmiss_off = catalog[0].Nmiss_disk; + catalog[0].Nmissing_off = catalog[0].Nmissing_disk; } @@ -154,5 +155,5 @@ } /* read secfilt table data */ - if (catalog[0].catflags & LOAD_SECF) { + if (catalog[0].catflags & DVO_LOAD_SECFILT) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table secfilt data"); @@ -162,5 +163,5 @@ /* how many entries do we expect from the secfilt table? */ Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage; - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); + catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE); if (Nexpect != Nitems) { fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nexpect, Nitems); @@ -175,7 +176,7 @@ Ntmpfilt = catalog[0].Nsecfilt; Nsecfilt = catalog[0].Nsecfilt + 1; - Ntotal = Nsecfilt * catalog[0].Naves_disk; + Ntotal = Nsecfilt * catalog[0].Naverage_disk; ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); - for (i = 0; i < catalog[0].Naves_disk; i++) { + for (i = 0; i < catalog[0].Naverage_disk; i++) { catalog[0].secfilt[i*Nsecfilt + 0] = primary[i]; for (j = 0; j < Ntmpfilt; j++) { @@ -184,10 +185,10 @@ } catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_disk = Ntotal; + catalog[0].Nsecfilt_disk = Ntotal; free (tmpfilt); free (primary); } - catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; - catalog[0].Nsecf_off = 0; + catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt_disk; + catalog[0].Nsecfilt_off = 0; } else { /* no real need to skip the data array here... */ @@ -197,9 +198,9 @@ free (primary); catalog[0].Nsecfilt ++; - catalog[0].Nsecf_disk = catalog[0].Nsecfilt * catalog[0].Naves_disk; + catalog[0].Nsecfilt_disk = catalog[0].Nsecfilt * catalog[0].Naverage_disk; } ALLOCATE (catalog[0].secfilt, SecFilt, 1); - catalog[0].Nsecf_mem = 0; - catalog[0].Nsecf_off = catalog[0].Nsecf_disk; + catalog[0].Nsecfilt_mem = 0; + catalog[0].Nsecfilt_off = catalog[0].Nsecfilt_disk; } @@ -210,10 +211,10 @@ } /* read Lensing table data */ - if (catalog[0].catflags & LOAD_LENSING) { + if (catalog[0].catflags & DVO_LOAD_LENSING) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table lensing data"); return (FALSE); } - catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat); + catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &catalog[0].Nlensing, &catalog[0].catformat, FALSE); if (Nlensing != catalog[0].Nlensing_disk) { fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensing, catalog[0].Nlensing_disk); @@ -235,10 +236,10 @@ } /* read Lensobj table data */ - if (catalog[0].catflags & LOAD_LENSOBJ) { + if (catalog[0].catflags & DVO_LOAD_LENSOBJ) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table lensobj data"); return (FALSE); } - catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat); + catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE); if (Nlensobj != catalog[0].Nlensobj_disk) { fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensobj, catalog[0].Nlensobj_disk); @@ -260,10 +261,10 @@ } /* read StarPar table data */ - if (catalog[0].catflags & LOAD_STARPAR) { + if (catalog[0].catflags & DVO_LOAD_STARPAR) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table starpar data"); return (FALSE); } - catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat); + catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE); if (Nstarpar != catalog[0].Nstarpar_disk) { fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nstarpar, catalog[0].Nstarpar_disk); @@ -277,4 +278,29 @@ catalog[0].Nstarpar = 0; catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk; + } + + /* read GalPhot table header */ + if (!gfits_fread_header (catalog[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot header"); + return (FALSE); + } + /* read GalPhot table data */ + if (catalog[0].catflags & DVO_LOAD_GALPHOT) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot data"); + return (FALSE); + } + catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &catalog[0].Ngalphot, &catalog[0].catformat, FALSE); + if (Ngalphot != catalog[0].Ngalphot_disk) { + fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Ngalphot, catalog[0].Ngalphot_disk); + } + catalog[0].Ngalphot = catalog[0].Ngalphot_disk; + catalog[0].Ngalphot_off = 0; + } else { + Nbytes = gfits_data_size (&header); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); + ALLOCATE (catalog[0].galphot, GalPhot, 1); + catalog[0].Ngalphot = 0; + catalog[0].Ngalphot_off = catalog[0].Ngalphot_disk; } @@ -330,4 +356,6 @@ gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, catalog[0].Nlensobj); gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, catalog[0].Nstarpar); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, catalog[0].Ngalphot); + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -357,5 +385,5 @@ /* write out Average table (convert to FITS table format) */ - AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary); + AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -370,5 +398,5 @@ /* write out Measure table (convert to FITS table format) */ - MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat); + MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -383,5 +411,5 @@ /* write out Missing table (convert to FITS table format) */ - gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing); + gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -397,5 +425,5 @@ /* write out SecFilt table (convert to FITS table format) */ Nitems = catalog[0].Naverage * Nsecfilt; - SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat); + SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -410,5 +438,5 @@ /* write out Lensing table (convert to FITS table format) */ - LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat); + LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -423,25 +451,31 @@ /* write out Lensobj table (convert to FITS table format) */ - LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat); - if (!gfits_fwrite_Theader (catalog[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catalog[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_table (&ftable); - gfits_free_header (&header); - - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - return (TRUE); + LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat, TRUE); + if (!gfits_fwrite_Theader (catalog[0].f, &header)) { + fprintf (stderr, "can't write table header"); + goto failure; + } + if (!gfits_fwrite_table (catalog[0].f, &ftable)) { + fprintf (stderr, "can't write table data"); + goto failure; + } + gfits_free_table (&ftable); + gfits_free_header (&header); /* write out StarPar table (convert to FITS table format) */ - StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat); + StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat, TRUE); + if (!gfits_fwrite_Theader (catalog[0].f, &header)) { + fprintf (stderr, "can't write table header"); + goto failure; + } + if (!gfits_fwrite_table (catalog[0].f, &ftable)) { + fprintf (stderr, "can't write table data"); + goto failure; + } + gfits_free_table (&ftable); + gfits_free_header (&header); + + /* write out GalPhot table (convert to FITS table format) */ + GalPhotToFtable (&ftable, catalog[0].galphot, catalog[0].Ngalphot, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 38447) @@ -136,5 +136,5 @@ /* read and convert the averages (use a macro to clean this up?) */ /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { catalog[0].average = ReadRawAverage (catalog[0].f, catalog[0].Naverage, catalog[0].catformat, &primary); } else { @@ -145,5 +145,5 @@ /* read and convert the measures (use a macro to clean this up?) */ - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { catalog[0].measure = ReadRawMeasure (catalog[0].f, catalog[0].average, catalog[0].Nmeasure, catalog[0].catformat); } else { @@ -154,5 +154,5 @@ /* read and convert missing */ - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { ALLOCATE (catalog[0].missing, Missing, MAX (catalog[0].Nmissing, 1)); Nitems = catalog[0].Nmissing; @@ -170,5 +170,5 @@ /* read and convert secfilt */ - if (catalog[0].catflags & LOAD_SECF) { + if (catalog[0].catflags & DVO_LOAD_SECFILT) { Nitems = catalog[0].Naverage * catalog[0].Nsecfilt; catalog[0].secfilt = ReadRawSecFilt (catalog[0].f, Nitems, catalog[0].catformat); @@ -191,5 +191,5 @@ } catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_mem = Ntotal; + catalog[0].Nsecfilt_mem = Ntotal; free (primary); } @@ -210,5 +210,5 @@ /* check data integrity */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { for (i = Nmeas = Nmiss = 0; i < catalog[0].Naverage; i++) { Nmeas += catalog[0].average[i].Nmeasure; @@ -226,7 +226,7 @@ /* save the current number so we can do partial updates */ - catalog[0].Naves_disk = catalog[0].Naverage; - catalog[0].Nmeas_disk = catalog[0].Nmeasure; - catalog[0].Nmiss_disk = catalog[0].Nmissing; + catalog[0].Naverage_disk = catalog[0].Naverage; + catalog[0].Nmeasure_disk = catalog[0].Nmeasure; + catalog[0].Nmissing_disk = catalog[0].Nmissing; return (TRUE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 38447) @@ -1,3 +1,4 @@ # include +# define OHANA_MEMCHECK 0 // return options: @@ -6,4 +7,20 @@ // * ok +// utility function to see if we should compress this table) +int output_is_compressed (int start, int Nrows, int Ntotal, DVOCatCompress catcompress) { + int fullWrite = (start == 0) && (Nrows == Ntotal); + if (fullWrite && catcompress) return TRUE; // we want to compress, so do not swap + return FALSE; +} + +int byteswap_varlength_ftable (Header *header, FTable *table) { + int i, Nfields; + if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE; + for (i = 0; i < Nfields; i++) { + if (!gfits_byteswap_varlength_column (table, i+1)) return FALSE; + } + return TRUE; +} + int dvo_catalog_secfilt_to_primary (Catalog *catalog, SecFilt **myPrimary, SecFilt **mySecfilt, int *myNsecfilt) { @@ -20,5 +37,5 @@ // XXX this translation only works if we have loaded / created a matched average/secfilt set - assert (catalog[0].Nsecf_mem == catalog[0].Nsecfilt*catalog[0].Naverage); + assert (catalog[0].Nsecfilt_mem == catalog[0].Nsecfilt*catalog[0].Naverage); Nallfilt = catalog[0].Nsecfilt; @@ -35,5 +52,5 @@ } catalog[0].Nsecfilt --; - catalog[0].Nsecf_mem = catalog[0].Naverage*catalog[0].Nsecfilt; + catalog[0].Nsecfilt_mem = catalog[0].Naverage*catalog[0].Nsecfilt; *myPrimary = primary; @@ -51,5 +68,5 @@ Ntmpfilt = catalog[0].Nsecfilt; - // we do NOT modify Nsecf_disk; this operation only modifies in in-memory values + // we do NOT modify Nsecfilt_disk; this operation only modifies in in-memory values catalog[0].Nsecfilt ++; @@ -57,5 +74,5 @@ Ntotal = Nsecfilt * Naves; - catalog[0].Nsecf_mem = Ntotal; + catalog[0].Nsecfilt_mem = Ntotal; ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); @@ -74,62 +91,12 @@ Matrix matrix; - - // ohana_memcheck_func (TRUE); - - if (0) { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v2", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - // catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - // catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } - if (0) { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - // catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - // catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } -# if (0) - { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v2", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } - { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v3", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } -# endif + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + /* rewind file pointers and truncate (file is still open) */ if (fseeko (catalog->f, 0LL, SEEK_SET)) { perror ("dvo_catalog_save_subset: "); fprintf (stderr, "failed to seek to beginning\n"); + return FALSE; } @@ -151,13 +118,63 @@ gfits_free_matrix (&matrix); + FTable *outtable = ftable; + FTable cmptable; + Header cmpheader; + gfits_init_header (&cmpheader); + gfits_init_table (&cmptable); + cmptable.header = &cmpheader; + + int fullWrite = (start == 0) && (Nrows == Ntotal); + int isCompressed = output_is_compressed (start, Nrows, Ntotal, catalog->catcompress); + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + + if (isCompressed) { + // fprintf (stderr, "compress %s\n", catalog->filename); + char *compressMode = dvo_catalog_compress_string (catalog->catcompress); + // I should test how Ntile affects fpack/funpack and speed + if (!gfits_compress_table (ftable, &cmptable, 1000, compressMode)) { + fprintf (stderr, "compression failure\n"); + return (FALSE); + } + if (!byteswap_varlength_ftable (&cmpheader, &cmptable)) { + fprintf (stderr, "failed to swap varlength column\n"); + return FALSE; + } + if (!gfits_modify (cmptable.header, "DVO_CMP", "%s", 1, compressMode)) { + fprintf (stderr, "can't save compression mode\n"); + return (FALSE); + } + outtable = &cmptable; + } + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + // write the table data - if (!gfits_fwrite_ftable_range (catalog->f, ftable, start, Nrows, Ndisk, Ntotal)) { - fprintf (stderr, "can't write table data\n"); - return (FALSE); - } - fullsize += ftable->datasize + ftable->header->datasize; + if (fullWrite) { + if (!gfits_fwrite_Theader (catalog->f, outtable->header)) { + fprintf (stderr, "can't write table header\n"); + return (FALSE); + } + if (!gfits_fwrite_table (catalog->f, outtable)) { + fprintf (stderr, "can't write table data\n"); + return (FALSE); + } + } else { + if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) { + fprintf (stderr, "can't write table data (range)\n"); + return (FALSE); + } + } + fullsize += outtable->datasize + outtable->header->datasize; + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + + // since we init'ed the structures above, these operations are safe whether or not we compressed the table + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); // if the output file will be completely re-written, truncate to total datasize - if ((start == 0) && (Nrows == Ntotal)) { + if (fullWrite) { int fd = fileno (catalog->f); if (ftruncate (fd, fullsize)) { @@ -192,7 +209,8 @@ // databases created prior to ~2014.07.01 did not have the LENSING, LENSOBJ, STARPAR paths in their headers. // in these cases, we do not try to lock or open the relevant file - if (!strcmp (name, "LENSING")) return (DVO_CAT_OPEN_EMPTY); - if (!strcmp (name, "LENSOBJ")) return (DVO_CAT_OPEN_EMPTY); - if (!strcmp (name, "STARPAR")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "LENSING")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "LENSOBJ")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "STARPAR")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "GALPHOT")) return (DVO_CAT_OPEN_EMPTY); return (DVO_CAT_OPEN_FAIL); } @@ -200,4 +218,7 @@ sprintf (subcat[0].filename, "%s/%s", path, string); free (path); + + // inherit compression mode for this subcat; + subcat->catcompress = catalog->catcompress; /* lock & open catalog file */ @@ -220,5 +241,5 @@ } Nskip = gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nskip, SEEK_CUR); + if (fseeko (subcat[0].f, Nskip, SEEK_CUR)) { perror ("fseeko: "); exit (1); } /* read table header */ @@ -230,4 +251,104 @@ return (DVO_CAT_OPEN_OK); } + +// ftable must already exist and have a valid, loaded header +int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder, char VERBOSE) { + + // fread_ftable_data requires the header + + *nativeOrder = FALSE; + if (!gfits_fread_ftable_data (catalog->f, ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table data\n"); + return FALSE; + } + // NOTE: fread_ftable returns an unswapped table; uncompress swaps the bytes into native order + + if (gfits_extension_is_compressed_table (ftable->header)) { + FTable rawtable; + Header rawheader; + gfits_init_table (&rawtable); + gfits_init_header (&rawheader); + rawtable.header = &rawheader; + + // NOTE: uncompress swaps the data bytes into native order, but needs the varlength columns native + if (!byteswap_varlength_ftable (ftable->header, ftable)) { + fprintf (stderr, "failed to swap varlength column\n"); + return FALSE; + } + if (!gfits_uncompress_table (ftable, &rawtable)) { + if (VERBOSE) fprintf (stderr, "failed to uncompress table\n"); + gfits_free_table (ftable); + return FALSE; + } + + char compressMode[80]; + if (!gfits_scan (ftable->header, "DVO_CMP", "%s", 1, compressMode)) { + strcpy (compressMode, "AUTO"); + } + catalog->catcompress = dvo_catalog_catcompress (compressMode); // if any table is compressed, set all to a compress state? + + // free the buffers + gfits_free_header (ftable->header); + gfits_free_table (ftable); + + // copies values but does not allocate new memory + Header *outheader = ftable->header; + gfits_copy_header_ptr (&rawheader, outheader); + gfits_copy_ftable_ptr (&rawtable, ftable); // this replaces ftable->header + ftable->header = outheader; + *nativeOrder = TRUE; + } + return TRUE; +} + +// LOAD_SUBCAT(measure,MEASURE,Measure) + +# define LOAD_SUBCAT(FIELD, NAME, STRUCT) \ + { \ + Header header; \ + FTable ftable; \ + off_t Nitems; \ + gfits_init_table (&ftable); \ + gfits_init_header (&header); \ + ftable.header = &header; \ + int status = DVO_CAT_OPEN_EMPTY; \ + if (!(catalog[0].catflags & DVO_SKIP_##NAME)) { \ + status = dvo_catalog_open_subcat (catalog, &catalog[0].FIELD##_catalog, ftable.header, #NAME, VERBOSE); \ + if (status == DVO_CAT_OPEN_FAIL) { \ + return (FALSE); \ + } \ + if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].N##FIELD##_disk > 0)) { \ + return (FALSE); \ + } \ + } \ + if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & DVO_LOAD_##NAME)) { \ + char nativeOrder = FALSE; \ + /* read table data */ \ + if (!gfits_fread_uncompressed (catalog[0].FIELD##_catalog, &ftable, &nativeOrder, VERBOSE)) { \ + if (VERBOSE) fprintf (stderr, "can't read table %s data\n", #FIELD); \ + gfits_free_header (&header); \ + return (FALSE); \ + } \ + /* convert data format to internal : returns number of row read in Nvalues */ \ + catalog[0].FIELD = FtableTo##STRUCT (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, nativeOrder); \ + if (Nitems != catalog[0].N##FIELD##_disk) { \ + fprintf (stderr, "Warning: mismatch between N%s in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", #FIELD, Nitems, catalog[0].N##FIELD##_disk); \ + } \ + catalog[0].N##FIELD = catalog[0].N##FIELD##_disk; \ + catalog[0].N##FIELD##_off = 0; \ + } else { \ + if (catalog[0].FIELD##_catalog) { \ + gfits_free_header (&catalog[0].FIELD##_catalog[0].header); \ + } else { \ + ALLOCATE (catalog[0].FIELD##_catalog, Catalog, 1); \ + dvo_catalog_init (catalog[0].FIELD##_catalog, TRUE); \ + } \ + gfits_create_header (&catalog[0].FIELD##_catalog[0].header); \ + ALLOCATE (catalog[0].FIELD, STRUCT, 1); \ + catalog[0].N##FIELD = 0; \ + catalog[0].N##FIELD##_off = catalog[0].N##FIELD##_disk; \ + } \ + gfits_free_header (&header); \ + } int dvo_catalog_load_split (Catalog *catalog, int VERBOSE) { @@ -240,26 +361,20 @@ off_t Nlensobj; off_t Nstarpar; - off_t Nitems; - int status, Nsecfilt; - Header header; - FTable ftable; + off_t Ngalphot; + int Nsecfilt; SecFilt *primary; - /* ftable header storage for below */ - ftable.header = &header; - ftable.buffer = NULL; - header.buffer = NULL; primary = NULL; - status = FALSE; /* get the components from the header - these duplicate information in the split files (NAXIS2) */ - // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR are not (0 if not found) - if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; - if (!gfits_scan (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, &Nlensing)) Nlensing = 0; - if (!gfits_scan (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, &Nlensobj)) Nlensobj = 0; - if (!gfits_scan (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, &Nstarpar)) Nstarpar = 0; + // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR, NGALPHOT are not (0 if not found) + if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; + if (!gfits_scan (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, &Nlensing)) Nlensing = 0; + if (!gfits_scan (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, &Nlensobj)) Nlensobj = 0; + if (!gfits_scan (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, &Nstarpar)) Nstarpar = 0; + if (!gfits_scan (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, &Ngalphot)) Ngalphot = 0; /* the OBJID is a counter that uniquely defines an average entry and never changes. if @@ -276,24 +391,22 @@ /* save the current number so we can do partial updates */ - catalog[0].Naves_disk = Naverage; - catalog[0].Nmeas_disk = Nmeasure; - catalog[0].Nmiss_disk = Nmissing; - catalog[0].Nsecf_disk = Naverage * Nsecfilt; - catalog[0].Nlensing_disk = Nlensing; - catalog[0].Nlensobj_disk = Nlensobj; - catalog[0].Nstarpar_disk = Nstarpar; - - /** Nsecfilt is unusual: it does not list the number of data items in the table - instead, the number of items is Nsecfilt * Naverage. **/ - catalog[0].Nsecfilt = Nsecfilt; + catalog[0].Naverage_disk = Naverage; + catalog[0].Nmeasure_disk = Nmeasure; + catalog[0].Nmissing_disk = Nmissing; + catalog[0].Nsecfilt_disk = Naverage * Nsecfilt; + catalog[0].Nlensing_disk = Nlensing; + catalog[0].Nlensobj_disk = Nlensobj; + catalog[0].Nstarpar_disk = Nstarpar; + catalog[0].Ngalphot_disk = Ngalphot; /* default values, but we will assign these a valid value before we exit (even if empty) */ - catalog[0].average = NULL; - catalog[0].measure = NULL; - catalog[0].missing = NULL; - catalog[0].secfilt = NULL; - catalog[0].lensing = NULL; - catalog[0].lensobj = NULL; - catalog[0].starpar = NULL; + catalog[0].average = NULL; + catalog[0].measure = NULL; + catalog[0].missing = NULL; + catalog[0].secfilt = NULL; + catalog[0].lensing = NULL; + catalog[0].lensobj = NULL; + catalog[0].starpar = NULL; + catalog[0].galphot = NULL; /*** Average Table ***/ @@ -302,268 +415,71 @@ /* move pointer past PHU header -- must be already read (load_catalog) */ Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); - fseeko (catalog[0].f, Nbytes, SEEK_SET); + if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* ftable header storage for below */ + char nativeOrder; + Header header; + FTable ftable; + gfits_init_table (&ftable); + gfits_init_header (&header); + ftable.header = &header; /* read Average table header */ if (!gfits_fread_header (catalog[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read table average header"); + if (VERBOSE) fprintf (stderr, "can't read table average header\n"); return (FALSE); } - if (catalog[0].catflags & LOAD_AVES) { - /* read Average table data : format is irrelevant here */ - if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table average data"); - return (FALSE); - } - /* convert the saved version of the table to the internal version. Old versions of DVO stored + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { + if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder, VERBOSE)) { + if (VERBOSE) fprintf (stderr, "can't read Average table\n"); + gfits_free_header (&header); + return FALSE; + } + + /* convert the disk version of the table to the internal version. Old versions of DVO stored * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: * primary is only used if we read in the secfilt table, otherwise it should be freed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); - } - gfits_free_header (&header); - catalog[0].Naverage = catalog[0].Naves_disk; - catalog[0].Naves_off = 0; + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, nativeOrder); + if (Naverage != catalog[0].Naverage_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naverage_disk); + } + catalog[0].Naverage = catalog[0].Naverage_disk; + catalog[0].Naverage_off = 0; } else { catalog[0].catformat = FtableGetFormat (&ftable); ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; - catalog[0].Naves_off = catalog[0].Naves_disk; - } + catalog[0].Naverage_off = catalog[0].Naverage_disk; + } + gfits_free_header (&header); /*** Measure Table ***/ - if (!(catalog[0].catflags & SKIP_MEAS)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].measure_catalog, ftable.header, "MEASURE", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmeas_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) { - // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK? - /* read Measure table data */ - if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table measure data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nmeasure */ - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); - } - catalog[0].Nmeasure = catalog[0].Nmeas_disk; - catalog[0].Nmeas_off = 0; - } else { - // XXX is it necessary to generate a template header here? - // XXX this is a memory leak, right? - if (catalog[0].measure_catalog) { - gfits_free_header (&catalog[0].measure_catalog[0].header); + + LOAD_SUBCAT(measure,MEASURE,Measure); + LOAD_SUBCAT(missing,MISSING,Missing); + LOAD_SUBCAT(secfilt,SECFILT,SecFilt); + LOAD_SUBCAT(lensing,LENSING,Lensing); + LOAD_SUBCAT(lensobj,LENSOBJ,Lensobj); + LOAD_SUBCAT(starpar,STARPAR,StarPar); + LOAD_SUBCAT(galphot,GALPHOT,GalPhot); + + /** catalog->Nsecfilt is unusual: it does not list the number of data items in the + table instead, the number of items is Nsecfilt * Naverage, and is stored in + Nsecfilt_mem. fix these below **/ + + catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt; + catalog[0].Nsecfilt = Nsecfilt; + + /** some old formats stored one of the secfilt values in the average table. + repair this below **/ + + if (primary) { + if (catalog[0].Nsecfilt_mem) { + dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naverage_disk); } else { - ALLOCATE (catalog[0].measure_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].measure_catalog, TRUE); - } - gfits_create_header (&catalog[0].measure_catalog[0].header); - ALLOCATE (catalog[0].measure, Measure, 1); - catalog[0].Nmeasure = 0; - catalog[0].Nmeas_off = catalog[0].Nmeas_disk; - } - gfits_free_header (&header); - - /*** Missing Table ***/ - status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmiss_disk > 0)) { - return (FALSE); - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) { - /* read Missing table data */ - if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table missing data\n"); - return (FALSE); - } - /* no conversions currently defined : this just does the byte swap */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); - if (!catalog[0].missing) { - fprintf (stderr, "ERROR: failed to read missing\n"); - exit (2); - } - if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); - } - catalog[0].Nmissing = catalog[0].Nmiss_disk; - catalog[0].Nmiss_off = 0; - } else { - // XXX is it necessary to generate a template header here? - gfits_create_header (&catalog[0].missing_catalog[0].header); - ALLOCATE (catalog[0].missing, Missing, 1); - catalog[0].Nmissing = 0; - catalog[0].Nmiss_off = catalog[0].Nmiss_disk; - } - gfits_free_header (ftable.header); - - /*** Secfilt Table ***/ - status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE); - // fprintf (stderr, "secfilt header %s : %zx\n", catalog[0].secfilt_catalog[0].filename, (size_t) catalog[0].secfilt_catalog[0].header.buffer); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nsecf_disk > 0)) { - return (FALSE); - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) { - /* read secfilt table data */ - if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n"); - return (FALSE); - } - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); - if (Nitems != catalog[0].Nsecf_disk) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, catalog[0].Nsecf_disk); - } - catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; - catalog[0].Nsecf_off = 0; - - /* if primary is defined, we were supplied with one additional average magnitude from Average - we need to interleave these magnitudes with the secfilt entries just loaded */ - if (primary != NULL) { - // this modifies catalog.Nsecf_mem,Nsecfilt - dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naves_disk); - } - } else { - if (primary != NULL) { free (primary); catalog[0].Nsecfilt ++; } - gfits_create_header (&catalog[0].secfilt_catalog[0].header); - ALLOCATE (catalog[0].secfilt, SecFilt, 1); - catalog[0].Nsecf_mem = 0; - catalog[0].Nsecf_off = catalog[0].Nsecf_disk; - } - gfits_free_header (ftable.header); - - /*** Lensing Table ***/ - if (!(catalog[0].catflags & SKIP_LENSING)) { - // unless we specify 'skip', we still need to load the Lensing table - status = dvo_catalog_open_subcat (catalog, &catalog[0].lensing_catalog, ftable.header, "LENSING", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the Lensing file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensing_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSING)) { - // only read the Lensing table if the file actually exists - /* read Lensing table data */ - if (!gfits_fread_ftable_data (catalog[0].lensing_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table lensing data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nlensing */ - catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat); - if (Nlensing != catalog[0].Nlensing_disk) { - fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensing, catalog[0].Nlensing_disk); - } - catalog[0].Nlensing = catalog[0].Nlensing_disk; - catalog[0].Nlensing_off = 0; - } else { - if (catalog[0].lensing_catalog) { - gfits_free_header (&catalog[0].lensing_catalog[0].header); - } else { - ALLOCATE (catalog[0].lensing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensing_catalog, TRUE); - } - gfits_create_header (&catalog[0].lensing_catalog[0].header); - ALLOCATE (catalog[0].lensing, Lensing, 1); - catalog[0].Nlensing = 0; - catalog[0].Nlensing_off = catalog[0].Nlensing_disk; - } - gfits_free_header (&header); - - /*** Lensobj Table ***/ - if (!(catalog[0].catflags & SKIP_LENSOBJ)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].lensobj_catalog, ftable.header, "LENSOBJ", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the Lensobj file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensobj_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSOBJ)) { - // only read the Lensobj table if the file actually exists - /* read Lensobj table data */ - if (!gfits_fread_ftable_data (catalog[0].lensobj_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table lensobj data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nlensobj */ - catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat); - if (Nlensobj != catalog[0].Nlensobj_disk) { - fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensobj, catalog[0].Nlensobj_disk); - } - catalog[0].Nlensobj = catalog[0].Nlensobj_disk; - catalog[0].Nlensobj_off = 0; - } else { - if (catalog[0].lensobj_catalog) { - gfits_free_header (&catalog[0].lensobj_catalog[0].header); - } else { - ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensobj_catalog, TRUE); - } - gfits_create_header (&catalog[0].lensobj_catalog[0].header); - ALLOCATE (catalog[0].lensobj, Lensobj, 1); - catalog[0].Nlensobj = 0; - catalog[0].Nlensobj_off = catalog[0].Nlensobj_disk; - } - gfits_free_header (&header); - - /*** StarPar Table ***/ - if (!(catalog[0].catflags & SKIP_STARPAR)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].starpar_catalog, ftable.header, "STARPAR", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the StarPar file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nstarpar_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_STARPAR)) { - // only read the StarPar table if the file actually exists - /* read StarPar table data */ - if (!gfits_fread_ftable_data (catalog[0].starpar_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table starpar data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nstarpar */ - catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat); - if (Nstarpar != catalog[0].Nstarpar_disk) { - fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nstarpar, catalog[0].Nstarpar_disk); - } - catalog[0].Nstarpar = catalog[0].Nstarpar_disk; - catalog[0].Nstarpar_off = 0; - } else { - if (catalog[0].starpar_catalog) { - gfits_free_header (&catalog[0].starpar_catalog[0].header); - } else { - ALLOCATE (catalog[0].starpar_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].starpar_catalog, TRUE); - } - gfits_create_header (&catalog[0].starpar_catalog[0].header); - ALLOCATE (catalog[0].starpar, StarPar, 1); - catalog[0].Nstarpar = 0; - catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk; - } - gfits_free_header (&header); + } return (TRUE); @@ -575,5 +491,5 @@ off_t Nbytes; - off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar; + off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot; Header header; FTable ftable; @@ -587,10 +503,11 @@ /*** Average (& SecFilt) Table ***/ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { /*** load the Average data ***/ /* move pointer past header and matrix -- must be already read (load_catalog) */ Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); - fseeko (catalog[0].f, Nbytes, SEEK_SET); + if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + /* read Average table header */ if (!gfits_fread_header (catalog[0].f, &header)) { @@ -607,5 +524,5 @@ * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: * primary is only used if we read in the secfilt table, otherwise it should be freed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE); if (Naverage != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -614,5 +531,5 @@ gfits_free_header (&header); catalog[0].Naverage = Naverage; - catalog[0].Naves_off = start; + catalog[0].Naverage_off = start; /*** load the secfilt data ***/ @@ -621,5 +538,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Secfilt table header */ @@ -635,10 +552,10 @@ Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt; - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); + catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE); if (Nitems != Nexpect) { fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, Nexpect); } - catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt; - catalog[0].Nsecf_off = start * catalog[0].Nsecfilt; + catalog[0].Nsecfilt_mem = catalog[0].Naverage * catalog[0].Nsecfilt; + catalog[0].Nsecfilt_off = start * catalog[0].Nsecfilt; /* if primary is defined, we were supplied with one additional average magnitude from Average @@ -651,5 +568,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { Catalog *subcat = catalog[0].measure_catalog; @@ -657,5 +574,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Measure table header */ @@ -671,5 +588,5 @@ /* convert data format to internal : returns number of row read in Nmeasure */ - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat); + catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, FALSE); if (Nmeasure != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -678,9 +595,9 @@ gfits_free_header (&header); catalog[0].Nmeasure = Nmeasure; - catalog[0].Nmeas_off = start; + catalog[0].Nmeasure_off = start; } // XXX check the open status of the catalog? - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { Catalog *subcat = catalog[0].missing_catalog; @@ -688,5 +605,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Missing table header */ @@ -702,5 +619,5 @@ /* no conversions currently defined : this just does the byte swap */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); + catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, NULL); if (!catalog[0].missing) { fprintf (stderr, "ERROR: failed to read missing\n"); @@ -712,9 +629,9 @@ gfits_free_header (&header); catalog[0].Nmissing = Nmissing; - catalog[0].Nmiss_off = start; + catalog[0].Nmissing_off = start; } // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_LENSING) { + if (catalog[0].catflags & DVO_LOAD_LENSING) { Catalog *subcat = catalog[0].lensing_catalog; @@ -722,5 +639,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Lensing table header */ @@ -736,5 +653,5 @@ /* convert data format to internal : returns number of row read in Nlensing */ - catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat); + catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &Nlensing, &catalog[0].catformat, FALSE); if (Nlensing != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -747,5 +664,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_LENSOBJ) { + if (catalog[0].catflags & DVO_LOAD_LENSOBJ) { Catalog *subcat = catalog[0].lensobj_catalog; @@ -753,5 +670,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Lensobj table header */ @@ -767,5 +684,5 @@ /* convert data format to internal : returns number of row read in Nlensobj */ - catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat); + catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &Nlensobj, &catalog[0].catformat, FALSE); if (Nlensobj != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -778,5 +695,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_STARPAR) { + if (catalog[0].catflags & DVO_LOAD_STARPAR) { Catalog *subcat = catalog[0].starpar_catalog; @@ -784,5 +701,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read StarPar table header */ @@ -798,5 +715,5 @@ /* convert data format to internal : returns number of row read in Nstarpar */ - catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat); + catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &Nstarpar, &catalog[0].catformat, FALSE); if (Nstarpar != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -808,4 +725,35 @@ } + // XXX check the open status of the catalog + if (catalog[0].catflags & DVO_LOAD_GALPHOT) { + + Catalog *subcat = catalog[0].galphot_catalog; + + /* move pointer past header -- must be already read (load_catalog) */ + Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* read GalPhot table header */ + if (!gfits_fread_header (subcat[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot header"); + return (FALSE); + } + /* read GalPhot table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (subcat[0].f, FALSE, &ftable, start, Nrows)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot data"); + return (FALSE); + } + + /* convert data format to internal : returns number of row read in Ngalphot */ + catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &Ngalphot, &catalog[0].catformat, FALSE); + if (Ngalphot != Nrows) { + // XXX this condition denotes the eof has been reached; not an error or a warning + // fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Ngalphot, Nrows); + } + gfits_free_header (&header); + catalog[0].Ngalphot = Ngalphot; + catalog[0].Ngalphot_off = start; + } + return (TRUE); } @@ -814,11 +762,9 @@ int dvo_catalog_save_split (Catalog *catalog, char VERBOSE) { - off_t Nitems; Header header; FTable ftable; SecFilt *primary, *secfilt; int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; - off_t first, start, Nrows; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; ftable.header = &header; @@ -828,6 +774,6 @@ // skip empty catalogs: it is illegal to have Measures without corresponding Averages - Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); - if (Naves_disk_new == 0) { + Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off); + if (Naverage_disk_new == 0) { if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); return (TRUE); @@ -847,19 +793,22 @@ } - Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); - Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); - Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); + Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off); + Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off); + Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off); Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); + Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off); /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); - gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); - gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); - gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); - gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); + gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); + gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); + gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -868,22 +817,29 @@ /*** Average Table ***/ - if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Naves_off; // first disk row to write - Nrows = catalog[0].Naverage - first; + if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Naverage_off; // first disk row to write + off_t Nrows = catalog[0].Naverage - first; assert (Nrows >= 0); assert (first >= 0); assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); - - /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); + + /* convert internal to external format : also results in a byte-swapped, scaled output + table. Note that ftable is newly allocated. + */ + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress); + + // convert to external table format + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { fprintf (stderr, "failure writing Average table\n"); goto failure; @@ -894,5 +850,5 @@ // even if we do not save the average table, we need to keep the header in sync /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* write table PHU header - always write this out */ @@ -905,8 +861,336 @@ /*** Measure Table ***/ - if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nmeasure_off; // first disk row to write + off_t Nrows = catalog[0].Nmeasure - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nmeasure); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress); + + // convert to external table format + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Measure table + catalog->measure_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { + fprintf (stderr, "trouble writing Measure table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Missing Table ***/ + if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) { + + if (catalog[0].Nmissing_off != 0) { + fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); + goto failure; + } + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (catalog[0].Nmissing_off, catalog[0].Nmissing, Nmissing_disk_new, catalog->catcompress); + + // convert to external table format + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Missing table (must write out entire table) + catalog->missing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble writing Missing Table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Secfilt Table ***/ + if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nsecfilt_off; // first disk row to write + off_t Nitems = catalog[0].Naverage*Nsecfilt; + off_t Nrows = Nitems - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= Nitems); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); + // XXX check these for consistency... + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress); + + // convert to external table format + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out SecFilt table + catalog->secfilt_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { + fprintf (stderr, "failure writing SecFilt table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Lensing Table ***/ + if ((catalog[0].catflags & DVO_LOAD_LENSING) && (catalog[0].lensing != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nlensing_off; // first disk row to write + off_t Nrows = catalog[0].Nlensing - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nlensing); + assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress); + + // convert to external table format + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Lensing table + catalog->lensing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) { + fprintf (stderr, "trouble writing Lensing table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Lensobj Table ***/ + if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nlensobj_off; // first disk row to write + off_t Nrows = catalog[0].Nlensobj - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nlensobj); + assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress); + + // convert to external table format + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Lensobj table + catalog->lensobj_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) { + fprintf (stderr, "trouble writing Lensobj table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** StarPar Table ***/ + if ((catalog[0].catflags & DVO_LOAD_STARPAR) && (catalog[0].starpar != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nstarpar_off; // first disk row to write + off_t Nrows = catalog[0].Nstarpar - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nstarpar); + assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress); + + // convert to external table format + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out StarPar table + catalog->starpar_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { + fprintf (stderr, "trouble writing StarPar table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** GalPhot Table ***/ + if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && (catalog[0].galphot != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Ngalphot_off; // first disk row to write + off_t Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress); + + // convert to external table format + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + catalog->galphot_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /* free temp storage */ + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (TRUE); + + failure: + /* free temp storage */ + gfits_free_header (&header); + gfits_free_table (&ftable); + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (FALSE); +} + +/* update_catalog_split only writes new lines to file. */ +int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { + + off_t Nitems; + Header header; + FTable ftable; + SecFilt *primary, *secfilt; + int Nsecfilt; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; + off_t first, start, Nrows; + + ftable.header = &header; + ftable.buffer = NULL; + header.buffer = NULL; + + // skip empty catalogs: it is illegal to have Measures without corresponding Averages + Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off); + if (Naverage_disk_new == 0) { + if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); + return (TRUE); + } + + // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary + switch (catalog[0].catformat) { + case DVO_FORMAT_ELIXIR: // special case for ELIXIR + case DVO_FORMAT_LONEOS: // special case for LONEOS + dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); + break; + default: + primary = NULL; + secfilt = catalog[0].secfilt; + Nsecfilt = catalog[0].Nsecfilt; + break; + } + + Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off); + Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off); + Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off); + Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); + Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); + Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); + Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off); + + /* make sure header is consistent with data */ + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); + gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); + gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); + gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); + + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); + gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); + + /* in split mode, we can save only part of the data */ + + /*** Average Table ***/ + if (catalog[0].average != NULL) { + + first = catalog[0].Naverage_disk - catalog[0].Naverage_off; // first row to write (memory) + start = catalog[0].Naverage_disk; // first row to write (disk) + Nrows = catalog[0].Naverage - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Naverage); + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); + + /* convert internal to external format */ + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { + fprintf (stderr, "failure writing Average table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } else { + // even if we do not save the average table, we need to keep the header in sync + /* rewind file pointers and truncate (file is still open) */ + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* write table PHU header - always write this out */ + /* XXX EAM : check if disk file size has changed */ + if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { + fprintf (stderr, "can't write primary header"); + goto failure; + } + } + + /*** Measure Table ***/ + if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Nmeas_off; // first disk row to write + start = catalog[0].Nmeasure_off; // first disk row to write Nrows = catalog[0].Nmeasure - first; @@ -914,8 +1198,8 @@ assert (first >= 0); assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); // convert to external table format (note that the block above does not damage or free catalog.average) - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -923,5 +1207,5 @@ // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { fprintf (stderr, "trouble writing Measure table\n"); goto failure; @@ -932,7 +1216,7 @@ /*** Missing Table ***/ - if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) { - - if (catalog[0].Nmiss_off != 0) { + if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) { + + if (catalog[0].Nmissing_off != 0) { fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); goto failure; @@ -940,5 +1224,5 @@ // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -955,9 +1239,9 @@ /*** Secfilt Table ***/ - if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Nsecf_off; // first disk row to write - Nitems = catalog[0].Nsecf_mem; + start = catalog[0].Nsecfilt_off; // first disk row to write + Nitems = catalog[0].Nsecfilt_mem; Nrows = Nitems - first; @@ -965,12 +1249,15 @@ assert (first >= 0); assert (first <= Nitems); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); // XXX check these for consistency... // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { fprintf (stderr, "failure writing SecFilt table\n"); goto failure; @@ -981,5 +1268,5 @@ /*** Lensing Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) { + if ((catalog[0].catflags & DVO_LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) { first = 0; // first row in memory to write @@ -994,5 +1281,5 @@ if (catalog[0].Nlensing) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1010,5 +1297,5 @@ /*** Lensobj Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) { + if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) { first = 0; // first row in memory to write @@ -1023,5 +1310,5 @@ if (catalog[0].Nlensobj) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1039,5 +1326,5 @@ /*** StarPar Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) { + if ((catalog[0].catflags & DVO_LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) { first = 0; // first row in memory to write @@ -1052,5 +1339,5 @@ if (catalog[0].Nstarpar) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1067,4 +1354,33 @@ } + /*** GalPhot Table (optional, do not save if not loaded) ***/ + if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && catalog[0].galphot_catalog->f && catalog[0].galphot) { + + first = 0; // first row in memory to write + start = catalog[0].Ngalphot_off; // first disk row to write + Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + if (catalog[0].Ngalphot) { + // convert to external table format (note that the block above does not damage or free catalog.average) + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + } + /* free temp storage */ if (primary != NULL) { @@ -1074,5 +1390,5 @@ return (TRUE); -failure: + failure: /* free temp storage */ gfits_free_header (&header); @@ -1088,10 +1404,9 @@ int dvo_catalog_save_split_complete (Catalog *catalog, char VERBOSE) { - off_t Nitems; Header header; FTable ftable; SecFilt *primary, *secfilt; int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; off_t first, start, Nrows; @@ -1102,17 +1417,17 @@ // skip empty catalogs: it is illegal to have Measures without corresponding Averages - if (catalog[0].Naves_off > 0) { + if (catalog[0].Naverage_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Average) was loaded\n"); goto failure; } - if (catalog[0].Nmeas_off > 0) { + if (catalog[0].Nmeasure_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Measure) was loaded\n"); goto failure; } - if (catalog[0].Nmiss_off > 0) { + if (catalog[0].Nmissing_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Missing) was loaded\n"); goto failure; } - if (catalog[0].Nsecf_off > 0) { + if (catalog[0].Nsecfilt_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Secfilt) was loaded\n"); goto failure; @@ -1130,7 +1445,11 @@ goto failure; } - - Naves_disk_new = catalog[0].Naverage; - if (Naves_disk_new == 0) { + if (catalog[0].Ngalphot_off > 0) { + fprintf (stderr, "ERROR: only partial catalog (GalPhot) was loaded\n"); + goto failure; + } + + Naverage_disk_new = catalog[0].Naverage; + if (Naverage_disk_new == 0) { if (VERBOSE) fprintf (stderr, "resulting catalog is empty; delete it\n"); // unlink (); @@ -1151,19 +1470,21 @@ } - Nmeas_disk_new = catalog[0].Nmeasure; - Nmiss_disk_new = catalog[0].Nmissing; - Nsecf_disk_new = catalog[0].Naverage*Nsecfilt; + Nmeasure_disk_new = catalog[0].Nmeasure; + Nmissing_disk_new = catalog[0].Nmissing; + Nsecfilt_disk_new = catalog[0].Naverage*Nsecfilt; Nlensing_disk_new = catalog[0].Nlensing; Nlensobj_disk_new = catalog[0].Nlensobj; Nstarpar_disk_new = catalog[0].Nstarpar; + Ngalphot_disk_new = catalog[0].Ngalphot; /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -1172,8 +1493,8 @@ /*** Average Table ***/ - if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Naves_off; // first disk row to write + start = catalog[0].Naverage_off; // first disk row to write Nrows = catalog[0].Naverage - first; @@ -1181,13 +1502,13 @@ assert (first >= 0); assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { fprintf (stderr, "failure writing Average table\n"); goto failure; @@ -1198,5 +1519,5 @@ // even if we do not save the average table, we need to keep the header in sync /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* write table PHU header - always write this out */ @@ -1209,8 +1530,8 @@ /*** Measure Table ***/ - if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nmeas_off; // first disk row to write + if (catalog[0].measure != NULL) { + + first = catalog[0].Nmeasure_disk - catalog[0].Nmeasure_off; // first row in memory to write + start = catalog[0].Nmeasure_disk; // first disk row to write Nrows = catalog[0].Nmeasure - first; @@ -1218,8 +1539,9 @@ assert (first >= 0); assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); // convert to external table format - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.measure have averef correctly set up? + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1227,5 +1549,5 @@ // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { fprintf (stderr, "trouble writing Measure table\n"); goto failure; @@ -1235,8 +1557,12 @@ } + /* missing table CANNOT be written unsorted, thus it is always written + out in full */ + /*** Missing Table ***/ - if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) { - - if (catalog[0].Nmiss_off != 0) { + + if (catalog[0].missing != NULL) { + + if (catalog[0].Nmissing_off != 0) { fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); goto failure; @@ -1244,5 +1570,5 @@ // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1259,22 +1585,22 @@ /*** Secfilt Table ***/ - if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nsecf_off; // first disk row to write - Nitems = catalog[0].Naverage*Nsecfilt; - Nrows = Nitems - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= Nitems); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); - // XXX check these for consistency... + if (catalog[0].secfilt != NULL) { + + first = catalog[0].Nsecfilt_disk - catalog[0].Nsecfilt_off; // first row in memory to write + start = catalog[0].Nsecfilt_disk; // first disk row to write + Nrows = catalog[0].Nsecfilt_mem - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { fprintf (stderr, "failure writing SecFilt table\n"); goto failure; @@ -1285,8 +1611,8 @@ /*** Lensing Table ***/ - if ((catalog[0].catflags & LOAD_LENSING) && (catalog[0].lensing != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nlensing_off; // first disk row to write + if (catalog[0].lensing != NULL) { + + first = catalog[0].Nlensing_disk - catalog[0].Nlensing_off; // first row in memory to write + start = catalog[0].Nlensing_disk; // first disk row to write Nrows = catalog[0].Nlensing - first; @@ -1297,5 +1623,6 @@ // convert to external table format - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.lensing have averef correctly set up? + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1312,8 +1639,8 @@ /*** Lensobj Table ***/ - if ((catalog[0].catflags & LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nlensobj_off; // first disk row to write + if (catalog[0].lensobj != NULL) { + + first = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off; // first row in memory to write + start = catalog[0].Nlensobj_disk; // first disk row to write Nrows = catalog[0].Nlensobj - first; @@ -1324,5 +1651,6 @@ // convert to external table format - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.lensobj have averef correctly set up? + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1339,8 +1667,8 @@ /*** StarPar Table ***/ - if ((catalog[0].catflags & LOAD_STARPAR) && (catalog[0].starpar != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nstarpar_off; // first disk row to write + if (catalog[0].starpar != NULL) { + + first = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off; // first row in memory to write + start = catalog[0].Nstarpar_disk; // first disk row to write Nrows = catalog[0].Nstarpar - first; @@ -1351,5 +1679,6 @@ // convert to external table format - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.starpar have averef correctly set up? + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1359,4 +1688,32 @@ if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { fprintf (stderr, "trouble writing StarPar table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** GalPhot Table ***/ + if (catalog[0].galphot != NULL) { + + first = catalog[0].Ngalphot_disk - catalog[0].Ngalphot_off; // first row in memory to write + start = catalog[0].Ngalphot_disk; // first disk row to write + Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + // convert to external table format + // XXX does catalog.galphot have averef correctly set up? + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); goto failure; } @@ -1370,279 +1727,8 @@ free (secfilt); } + return (TRUE); -failure: - /* free temp storage */ - gfits_free_header (&header); - gfits_free_table (&ftable); - if (primary != NULL) { - free (primary); - free (secfilt); - } - return (FALSE); -} - -/* update_catalog_split only writes new lines to file. */ -int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { - - Header header; - FTable ftable; - SecFilt *primary, *secfilt; - int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; - off_t first, start, Nrows; - - ftable.header = &header; - ftable.buffer = NULL; - header.buffer = NULL; - - // skip empty catalogs: it is illegal to have Measures without corresponding Averages - Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); - if (Naves_disk_new == 0) { - if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); - return (TRUE); - } - - // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary - switch (catalog[0].catformat) { - case DVO_FORMAT_ELIXIR: // special case for ELIXIR - case DVO_FORMAT_LONEOS: // special case for LONEOS - dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); - break; - default: - primary = NULL; - secfilt = catalog[0].secfilt; - Nsecfilt = catalog[0].Nsecfilt; - break; - } - - Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); - Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); - Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); - Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); - Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); - Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); - - /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); - gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); - gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); - gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); - gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); - gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); - gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); - - /* in split mode, we can save only part of the data */ - - /*** Average Table ***/ - if (catalog[0].average != NULL) { - - first = catalog[0].Naves_disk - catalog[0].Naves_off; // first row to write (memory) - start = catalog[0].Naves_disk; // first row to write (disk) - Nrows = catalog[0].Naverage - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); - - /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { - fprintf (stderr, "failure writing Average table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } else { - // even if we do not save the average table, we need to keep the header in sync - /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); - - /* write table PHU header - always write this out */ - /* XXX EAM : check if disk file size has changed */ - if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - } - - /*** Measure Table ***/ - if (catalog[0].measure != NULL) { - - first = catalog[0].Nmeas_disk - catalog[0].Nmeas_off; // first row in memory to write - start = catalog[0].Nmeas_disk; // first disk row to write - Nrows = catalog[0].Nmeasure - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); - - // convert to external table format - // XXX does catalog.measure have averef correctly set up? - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { - fprintf (stderr, "trouble writing Measure table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /* missing table CANNOT be written unsorted, thus it is always written - out in full */ - - /*** Missing Table ***/ - - if (catalog[0].missing != NULL) { - - if (catalog[0].Nmiss_off != 0) { - fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); - goto failure; - } - - // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Missing table (must write out entire table) - if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { - fprintf (stderr, "trouble writing Missing Table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Secfilt Table ***/ - if (catalog[0].secfilt != NULL) { - - first = catalog[0].Nsecf_disk - catalog[0].Nsecf_off; // first row in memory to write - start = catalog[0].Nsecf_disk; // first disk row to write - Nrows = catalog[0].Nsecf_mem - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); - - // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); - - // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { - fprintf (stderr, "failure writing SecFilt table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Lensing Table ***/ - if (catalog[0].lensing != NULL) { - - first = catalog[0].Nlensing_disk - catalog[0].Nlensing_off; // first row in memory to write - start = catalog[0].Nlensing_disk; // first disk row to write - Nrows = catalog[0].Nlensing - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nlensing); - assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off); - - // convert to external table format - // XXX does catalog.lensing have averef correctly set up? - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Lensing table - if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) { - fprintf (stderr, "trouble writing Lensing table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Lensobj Table ***/ - if (catalog[0].lensobj != NULL) { - - first = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off; // first row in memory to write - start = catalog[0].Nlensobj_disk; // first disk row to write - Nrows = catalog[0].Nlensobj - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nlensobj); - assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off); - - // convert to external table format - // XXX does catalog.lensobj have averef correctly set up? - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Lensobj table - if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) { - fprintf (stderr, "trouble writing Lensobj table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** StarPar Table ***/ - if (catalog[0].starpar != NULL) { - - first = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off; // first row in memory to write - start = catalog[0].Nstarpar_disk; // first disk row to write - Nrows = catalog[0].Nstarpar - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nstarpar); - assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off); - - // convert to external table format - // XXX does catalog.starpar have averef correctly set up? - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out StarPar table - if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { - fprintf (stderr, "trouble writing StarPar table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - - return (TRUE); - -failure: + failure: /* free temp storage */ if (primary != NULL) { @@ -1662,5 +1748,5 @@ /* XXX EAM : update is not efficient. MeasureToFtable should only - convert the new rows (Nmeas_disk to Nmeasure). the resulting + convert the new rows (Nmeasure_disk to Nmeasure). the resulting table represents the end rows of the ftable. we need to define the vtable based on the ftable, but with Ny = Nmeasure */ @@ -1668,11 +1754,11 @@ - // * convert to an ftable - // * optionally write the PHU header/matrix - // * advance to the start of the output data block: - // ** Nx * catalog[0].Nmeas_off - // * write out the ftable data block - // * if Nmeas_off + Nmeasure >= Nmeas_disk, update padding - // ** start = Nmeas_off - // ** Nrows = Nmeasure - +// * convert to an ftable +// * optionally write the PHU header/matrix +// * advance to the start of the output data block: +// ** Nx * catalog[0].Nmeasure_off +// * write out the ftable data block +// * if Nmeasure_off + Nmeasure >= Nmeasure_disk, update padding +// ** start = Nmeasure_off +// ** Nrows = Nmeasure + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert.c (revision 38447) @@ -11,7 +11,7 @@ constructions below **/ -DVOTableFormat FtableGetFormat (FTable *ftable) { - - DVOTableFormat format; +DVOCatFormat FtableGetFormat (FTable *ftable) { + + DVOCatFormat format; char extname[80]; @@ -52,5 +52,5 @@ /*** Average / FTable conversion functions ***/ -Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary) { +Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeBytes) { Average *average; @@ -98,5 +98,5 @@ if (!strcmp (extname, NAME)) { \ Average_##TYPE *tmpAverage; \ - tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL); \ + tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL, &nativeBytes); \ if (!tmpAverage) { \ fprintf (stderr, "ERROR: failed to read averages\n"); \ @@ -109,5 +109,5 @@ if (!strcmp (extname, "DVO_AVERAGE")) { - average = gfits_table_get_Average (ftable, Naverage, NULL); + average = gfits_table_get_Average (ftable, Naverage, NULL, &nativeBytes); if (!average) { fprintf (stderr, "ERROR: failed to read averages\n"); @@ -139,18 +139,18 @@ } -int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary) { - -# define FORMAT_CASE(FORMAT, TYPE) \ - case DVO_FORMAT_##FORMAT: { \ - Average_##TYPE *tmpAverage; \ +int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative) { + +# define FORMAT_CASE(FORMAT, TYPE) \ + case DVO_FORMAT_##FORMAT: { \ + Average_##TYPE *tmpAverage; \ tmpAverage = AverageInternalTo_##TYPE (average, Naverage, primary); \ - gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage); \ - free (tmpAverage); \ - break; } - + gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage, swapFromNative); \ + free (tmpAverage); \ + break; } + /* convert from the internal format */ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Average (ftable, average, Naverage); + gfits_table_set_Average (ftable, average, Naverage, swapFromNative); break; } @@ -180,5 +180,6 @@ // FtableToMeasure needs the Average since old formats stored measure.dR,dD only -Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format) { +// other FtableToFOO conversions accept the average argument for macro construction +Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeBytes) { Measure *measure; @@ -226,10 +227,10 @@ if (!strcmp (extname, NAME)) { \ Measure_##TYPE *tmpMeasure; \ - tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL); \ - if (!tmpMeasure) { \ - fprintf (stderr, "ERROR: failed to read measures\n"); \ - exit (2); \ - } \ - myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \ + tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL, &nativeBytes); \ + if (!tmpMeasure) { \ + fprintf (stderr, "ERROR: failed to read measures\n"); \ + exit (2); \ + } \ + myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \ measure = Measure_##TYPE##_ToInternal (average, tmpMeasure, *Nmeasure); \ free (tmpMeasure); \ @@ -238,5 +239,5 @@ if (!strcmp (extname, "DVO_MEASURE")) { - measure = gfits_table_get_Measure (ftable, Nmeasure, NULL); + measure = gfits_table_get_Measure (ftable, Nmeasure, NULL, &nativeBytes); if (!measure) { fprintf (stderr, "ERROR: failed to read measures\n"); @@ -269,5 +270,5 @@ // MeasureToFtable needs the Average since old formats stored measure.dR,dD only -int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format) { +int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE, ABS_COORDS) \ @@ -276,5 +277,5 @@ myAssert (ABS_COORDS || average, "conversion from internal needs average table"); \ tmpMeasure = MeasureInternalTo_##TYPE (average, measure, Nmeasure); \ - gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure); \ + gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure, swapFromNative); \ free (tmpMeasure); \ break; } @@ -283,5 +284,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Measure (ftable, measure, Nmeasure); + gfits_table_set_Measure (ftable, measure, Nmeasure, swapFromNative); break; } @@ -308,7 +309,83 @@ } +/*** Missing / FTable conversion functions ***/ + +Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes) { + + Missing *missing; + char extname[80]; + + /* convert to the internal format */ + if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) { + fprintf (stderr, "EXTNAME missing for missing table\n"); + return (FALSE); + } + +# define SKIPPING_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + fprintf (stderr, "ERROR: format %s not defined for missing, skipping\n", NAME); \ + *Nmissing = 0; \ + return NULL; \ + } + +# define CONVERT_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + Missing_##TYPE *tmpMissing; \ + tmpMissing = gfits_table_get_Missing_##TYPE (ftable, Nmissing, NULL, &nativeBytes); \ + if (!tmpMissing) { \ + fprintf (stderr, "ERROR: failed to read missing\n"); \ + exit (2); \ + } \ + missing = Missing_##TYPE##_ToInternal (tmpMissing, *Nmissing); \ + free (tmpMissing); \ + *format = DVO_FORMAT_##FORMAT; \ + return (missing); } + + // XXX the structure is here for future expansion, but no transformations are currently defined + if (TRUE) { + missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes); + if (!missing) { + fprintf (stderr, "ERROR: failed to read missing\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (missing); + } + + if (!strcmp (extname, "DVO_MISSING")) { + missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes); + if (!missing) { + fprintf (stderr, "ERROR: failed to read missing\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (missing); + } + + SKIPPING_FORMAT ("DVO_MISSING_PS1_SIM", PS1_SIM, PS1_SIM); + SKIPPING_FORMAT ("DVO_MISSING_PS1_REF", PS1_REF, PS1_REF); + SKIPPING_FORMAT ("DVO_MISSING_ELIXIR", ELIXIR, Elixir); + SKIPPING_FORMAT ("DVO_MISSING_LONEOS", LONEOS, Loneos); + SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0); + SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_1", PS1_DEV_1, PS1_DEV_1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_2", PS1_DEV_2, PS1_DEV_2); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V1", PS1_V1, PS1_V1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V2", PS1_V2, PS1_V2); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V3", PS1_V3, PS1_V3); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V4", PS1_V4, PS1_V4); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V5", PS1_V5, PS1_V5); +# undef CONVERT_FORMAT +# undef SKIPPING_FORMAT + + fprintf (stderr, "table format unknown: %s\n", extname); + + *Nmissing = 0; + return (NULL); +} + /*** SecFilt / FTable conversion functions ***/ -SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format) { +SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) { SecFilt *secfilt; @@ -339,5 +416,5 @@ if (!strcmp (extname, NAME)) { \ SecFilt_##TYPE *tmpSecFilt; \ - tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL); \ + tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL, &nativeBytes); \ if (!tmpSecFilt) { \ fprintf (stderr, "ERROR: failed to read secfilts\n"); \ @@ -350,5 +427,5 @@ if (!strcmp (extname, "DVO_SECFILT")) { - secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL); + secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL, &nativeBytes); if (!secfilt) { fprintf (stderr, "ERROR: failed to read secfilts\n"); @@ -380,5 +457,5 @@ } -int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format) { +int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -386,5 +463,5 @@ SecFilt_##TYPE *tmpSecFilt; \ tmpSecFilt = SecFiltInternalTo_##TYPE (secfilt, Nsecfilt); \ - gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt); \ + gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt, swapFromNative); \ free (tmpSecFilt); \ break; } @@ -393,5 +470,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt); + gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt, swapFromNative); break; } @@ -420,5 +497,5 @@ /*** Lensing / FTable conversion functions ***/ -Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format) { +Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) { Lensing *lensing; @@ -433,5 +510,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 128)) { Lensing_PS1_V5_R0 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -445,5 +522,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) { Lensing_PS1_V5_R1 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -457,5 +534,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 144)) { Lensing_PS1_V5_R2 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -477,5 +554,5 @@ if (!strcmp (extname, NAME)) { \ Lensing_##TYPE *tmpLensing; \ - tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL); \ + tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL, &nativeBytes); \ if (!tmpLensing) { \ fprintf (stderr, "ERROR: failed to read lensings\n"); \ @@ -488,5 +565,5 @@ if (!strcmp (extname, "DVO_LENSING")) { - lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL); + lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL, &nativeBytes); if (!lensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -520,5 +597,5 @@ // LensingToFtable needs the Average since old formats stored lensing.dR,dD only -int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format) { +int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -526,5 +603,5 @@ Lensing_##TYPE *tmpLensing; \ tmpLensing = LensingInternalTo_##TYPE (lensing, Nlensing); \ - gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing); \ + gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing, swapFromNative); \ free (tmpLensing); \ break; } @@ -533,5 +610,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Lensing (ftable, lensing, Nlensing); + gfits_table_set_Lensing (ftable, lensing, Nlensing, swapFromNative); break; } @@ -559,5 +636,5 @@ /*** Lensobj / FTable conversion functions ***/ -Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format) { +Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) { Lensobj *lensobj; @@ -572,5 +649,5 @@ if (!strcmp (extname, "DVO_LENSOBJ_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) { Lensobj_PS1_V5_R0 *tmpLensobj; - tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL); + tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL, &nativeBytes); if (!tmpLensobj) { fprintf (stderr, "ERROR: failed to read lensobjs\n"); @@ -592,5 +669,5 @@ if (!strcmp (extname, NAME)) { \ Lensobj_##TYPE *tmpLensobj; \ - tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL); \ + tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL, &nativeBytes); \ if (!tmpLensobj) { \ fprintf (stderr, "ERROR: failed to read lensobjs\n"); \ @@ -603,5 +680,5 @@ if (!strcmp (extname, "DVO_LENSOBJ")) { - lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL); + lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL, &nativeBytes); if (!lensobj) { fprintf (stderr, "ERROR: failed to read lensobjs\n"); @@ -635,5 +712,5 @@ // LensobjToFtable needs the Average since old formats stored lensobj.dR,dD only -int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format) { +int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -641,5 +718,5 @@ Lensobj_##TYPE *tmpLensobj; \ tmpLensobj = LensobjInternalTo_##TYPE (lensobj, Nlensobj); \ - gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj); \ + gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj, swapFromNative); \ free (tmpLensobj); \ break; } @@ -648,5 +725,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Lensobj (ftable, lensobj, Nlensobj); + gfits_table_set_Lensobj (ftable, lensobj, Nlensobj, swapFromNative); break; } @@ -674,5 +751,5 @@ /*** StarPar / FTable conversion functions ***/ -StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format) { +StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) { StarPar *starpar; @@ -695,5 +772,5 @@ if (!strcmp (extname, NAME)) { \ StarPar_##TYPE *tmpStarPar; \ - tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL); \ + tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL, &nativeBytes); \ if (!tmpStarPar) { \ fprintf (stderr, "ERROR: failed to read starpar\n"); \ @@ -706,5 +783,5 @@ if (!strcmp (extname, "DVO_STARPAR")) { - starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL); + starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL, &nativeBytes); if (!starpar) { fprintf (stderr, "ERROR: failed to read starpar\n"); @@ -738,5 +815,5 @@ // StarParToFtable needs the Average since old formats stored starpar.dR,dD only -int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format) { +int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -744,5 +821,5 @@ StarPar_##TYPE *tmpStarPar; \ tmpStarPar = StarParInternalTo_##TYPE (starpar, Nstarpar); \ - gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar); \ + gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar, swapFromNative); \ free (tmpStarPar); \ break; } @@ -751,5 +828,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_StarPar (ftable, starpar, Nstarpar); + gfits_table_set_StarPar (ftable, starpar, Nstarpar, swapFromNative); break; } @@ -776,9 +853,112 @@ } +/*** GalPhot / FTable conversion functions ***/ + +GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeBytes) { + + GalPhot *galphot; + char extname[80]; + + /* convert to the internal format */ + if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) { + fprintf (stderr, "EXTNAME missing for galphot table\n"); + return (FALSE); + } + +# define SKIPPING_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + fprintf (stderr, "ERROR: format %s not defined for galphot, skipping\n", NAME); \ + *Ngalphot = 0; \ + return NULL; \ + } + +# define CONVERT_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + GalPhot_##TYPE *tmpGalPhot; \ + tmpGalPhot = gfits_table_get_GalPhot_##TYPE (ftable, Ngalphot, NULL, &nativeBytes); \ + if (!tmpGalPhot) { \ + fprintf (stderr, "ERROR: failed to read galphots\n"); \ + exit (2); \ + } \ + galphot = GalPhot_##TYPE##_ToInternal (tmpGalPhot, *Ngalphot); \ + free (tmpGalPhot); \ + *format = DVO_FORMAT_##FORMAT; \ + return (galphot); } + + if (!strcmp (extname, "DVO_GALPHOT")) { + galphot = gfits_table_get_GalPhot (ftable, Ngalphot, NULL, &nativeBytes); + if (!galphot) { + fprintf (stderr, "ERROR: failed to read galphots\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (galphot); + } + + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_REF", PS1_REF, PS1_REF); + SKIPPING_FORMAT ("DVO_GALPHOT_ELIXIR", ELIXIR, Elixir); + SKIPPING_FORMAT ("DVO_GALPHOT_LONEOS", LONEOS, Loneos); + SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0); + SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_1", PS1_DEV_1, PS1_DEV_1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_2", PS1_DEV_2, PS1_DEV_2); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V1", PS1_V1, PS1_V1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V2", PS1_V2, PS1_V2); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V3", PS1_V3, PS1_V3); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V4", PS1_V4, PS1_V4); + CONVERT_FORMAT ("DVO_GALPHOT_PS1_V5", PS1_V5, PS1_V5); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_SIM", PS1_SIM, PS1_SIM); +# undef CONVERT_FORMAT +# undef SKIPPING_FORMAT + + fprintf (stderr, "table format unknown: %s\n", extname); + + *Ngalphot = 0; + return (NULL); +} + +// GalPhotToFtable needs the Average since old formats stored galphot.dR,dD only +int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative) { + +# define FORMAT_CASE(FORMAT, TYPE) \ + case DVO_FORMAT_##FORMAT: { \ + GalPhot_##TYPE *tmpGalPhot; \ + tmpGalPhot = GalPhotInternalTo_##TYPE (galphot, Ngalphot); \ + gfits_table_set_GalPhot_##TYPE (ftable, tmpGalPhot, Ngalphot, swapFromNative); \ + free (tmpGalPhot); \ + break; } + + /* convert from the internal format */ + switch (format) { + case DVO_FORMAT_INTERNAL: { + gfits_table_set_GalPhot (ftable, galphot, Ngalphot, swapFromNative); + break; } + +// FORMAT_CASE (PS1_REF, PS1_REF); +// FORMAT_CASE (ELIXIR, Elixir); +// FORMAT_CASE (LONEOS, Loneos); +// FORMAT_CASE (PANSTARRS_DEV_0, Panstarrs_DEV_0); +// FORMAT_CASE (PANSTARRS_DEV_1, Panstarrs_DEV_1); +// FORMAT_CASE (PS1_DEV_1, PS1_DEV_1); +// FORMAT_CASE (PS1_DEV_2, PS1_DEV_2); +// FORMAT_CASE (PS1_V1, PS1_V1); +// FORMAT_CASE (PS1_V2, PS1_V2); +// FORMAT_CASE (PS1_V3, PS1_V3); +// FORMAT_CASE (PS1_V4, PS1_V4); + FORMAT_CASE (PS1_V5, PS1_V5); +# undef FORMAT_CASE + + default: + fprintf (stderr, "table format unknown (galphot)\n"); + return (FALSE); + } + return (TRUE); +} + /*** Image Conversions ***/ // I have loaded the disk db table and now I want to convert to the internal format // (Image structure), but I onyl -int FtableToImage (FTable *ftable, Header *theader, char *format) { +int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format) { off_t Nimage; @@ -788,5 +968,5 @@ if (*format == DVO_FORMAT_ELIXIR) { // special case for ELIXIR Image_Elixir *tmpimage; - tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL); + tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL, NULL); if (!tmpimage) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -814,5 +994,5 @@ Image_##TYPE *tmpimage; \ *format = DVO_FORMAT_##FORMAT; \ - tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL); \ + tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL, NULL); \ if (!tmpimage) { \ fprintf (stderr, "ERROR: failed to read images\n"); \ @@ -849,5 +1029,5 @@ } -int ImageToFtable (FTable *ftable, Header *theader, char format) { +int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format) { off_t Nimage; @@ -860,5 +1040,5 @@ tmpImage = ImageInternalTo_##TYPE ((Image *) ftable[0].buffer, Nimage); \ free (ftable[0].buffer); \ - gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage); \ + gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage, TRUE); \ free (tmpImage); \ break; } @@ -889,5 +1069,5 @@ } -int ImageToVtable (VTable *vtable, Header *theader, char format) { +int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format) { off_t i, Nrow, Nimage; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 38447) @@ -118,5 +118,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Nextend; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -152,5 +152,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Nextend = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 38447) @@ -138,5 +138,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -173,5 +173,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 38447) @@ -143,5 +143,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -182,5 +182,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 38447) @@ -143,5 +143,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -184,5 +184,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 38447) @@ -164,5 +164,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -208,5 +208,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -690,5 +690,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38447) @@ -196,5 +196,5 @@ out[i].Nlensobj = in[i].Nlensobj; out[i].Nstarpar = in[i].Nstarpar; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -203,5 +203,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -264,5 +264,5 @@ out[i].Nlensobj = in[i].Nlensobj; out[i].Nstarpar = in[i].Nstarpar; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -271,5 +271,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -975,4 +975,68 @@ out[i].objID = in[i].objID ; out[i].catID = in[i].catID ; + } + return (out); +} + +GalPhot *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues) { + + off_t i; + GalPhot *out; + + ALLOCATE_ZERO (out, GalPhot, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_galphot_init (&out[i]); + + out[i].Xfit = in[i].Xfit; + out[i]. Yfit = in[i]. Yfit; + out[i]. mag = in[i]. mag; + out[i]. magErr = in[i]. magErr; + out[i]. majorAxis = in[i]. majorAxis; + out[i]. minorAxis = in[i]. minorAxis; + out[i]. majorAxisErr = in[i]. majorAxisErr; + out[i]. minorAxisErr = in[i]. minorAxisErr; + out[i]. theta = in[i]. theta; + out[i]. theta_err = in[i]. theta_err; + out[i]. index = in[i]. index; + out[i]. chisq = in[i]. chisq; + out[i]. Npix = in[i]. Npix; + out[i]. objID = in[i]. objID; + out[i]. catID = in[i]. catID; + out[i]. detID = in[i]. detID; + out[i]. imageID = in[i]. imageID; + out[i]. photcode = in[i]. photcode; + out[i]. modelType = in[i]. modelType; + } + return (out); +} + +GalPhot_PS1_V5 *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues) { + + off_t i; + GalPhot_PS1_V5 *out; + + ALLOCATE_ZERO (out, GalPhot_PS1_V5, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].Xfit = in[i].Xfit; + out[i].Yfit = in[i].Yfit; + out[i].mag = in[i].mag; + out[i].magErr = in[i].magErr; + out[i].majorAxis = in[i].majorAxis; + out[i].minorAxis = in[i].minorAxis; + out[i].majorAxisErr = in[i].majorAxisErr; + out[i].minorAxisErr = in[i].minorAxisErr; + out[i].theta = in[i].theta; + out[i].theta_err = in[i].theta_err; + out[i].index = in[i].index; + out[i].chisq = in[i].chisq; + out[i].Npix = in[i].Npix; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].photcode = in[i].photcode; + out[i].modelType = in[i].modelType; } return (out); @@ -1234,5 +1298,5 @@ out[i].Nlensing = in[i].Nlensing; out[i].Nlensobj = in[i].Nlensobj; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -1241,5 +1305,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -1302,10 +1366,10 @@ SWAP_BYTE (110); // NLENSING SWAP_BYTE (112); // NLENSOBJ - SWAP_BYTE (114); // NEXTEND + SWAP_BYTE (114); // NGALPHOT SWAP_WORD (116); // OFF_MEASURE SWAP_WORD (120); // OFF_MISSING SWAP_WORD (124); // OFF_LENSING SWAP_WORD (128); // OFF_LENSOBJ - SWAP_WORD (132); // OFF_EXTEND + SWAP_WORD (132); // OFF_GALPHOT SWAP_WORD (136); // OFF_PARAMS SWAP_WORD (140); // REF_COLOR_BLUE Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image.c (revision 38447) @@ -19,5 +19,4 @@ db[0].lockstate = lockstate; db[0].timeout = timeout; - gfits_db_init (db); if (!gfits_db_lock (db, filename)) { @@ -89,5 +88,6 @@ // so when we later do a gfits_copy_ftable, we stomp on bad memory FtableToImage (&db[0].ftable, &db[0].theader, &db[0].format); - db[0].swapped = TRUE; /* table has internal byte-order */ + db[0].nativeOrder = TRUE; /* table has internal byte-order */ + db[0].scaledValue = TRUE; /* table has internal byte-order */ return (TRUE); } @@ -100,5 +100,6 @@ /* convert from internal to requested external format */ ImageToFtable (&db[0].ftable, &db[0].theader, db[0].format); - db[0].swapped = FALSE; + db[0].nativeOrder = FALSE; + db[0].scaledValue = FALSE; /* write data in appropriate mode */ @@ -149,5 +150,6 @@ /* theader is modified to match output format */ ImageToVtable (&db[0].vtable, &db[0].theader, db[0].format); - db[0].swapped = FALSE; + db[0].scaledValue = FALSE; + db[0].nativeOrder = FALSE; /* write data in appropriate mode */ @@ -363,5 +365,5 @@ /* return internal structure representation */ -Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped) { +Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder) { int Ncols; @@ -377,9 +379,15 @@ data = (Image *) ftable[0].buffer; - if (!swapped) { - myAbort ("invalid to call this without suppying 'swapped'"); - } - if (*swapped == FALSE) { - myAbort ("invalid for table to be swapped"); + if (!scaledValue) { + myAbort ("invalid to call this without suppying 'scaledValue'"); + } + if (*scaledValue == FALSE) { + myAbort ("invalid for table NOT to be scaledValue"); + } + if (!nativeOrder) { + myAbort ("invalid to call this without suppying 'nativeOrder'"); + } + if (*nativeOrder == FALSE) { + myAbort ("invalid for table NOT to be nativeOrder"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image_raw.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image_raw.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_image_raw.c (revision 38447) @@ -107,5 +107,7 @@ db[0].theader.Naxis[1] = Nimage; db[0].ftable.datasize = gfits_data_size (&db[0].theader); - db[0].swapped = FALSE; /* table does not have internal byte-order */ + + db[0].nativeOrder = FALSE; /* table does not have internal byte-order */ + db[0].scaledValue = FALSE; /* table does not have internal byte-order */ return (TRUE); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_util.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_util.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/dvo_util.c (revision 38447) @@ -127,5 +127,5 @@ } - dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.swapped); + dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.scaledValue, &dvoConfig->imageDB.nativeOrder); if (!dvoConfig->images) { fprintf (stderr, "problem loading images\n"); @@ -207,5 +207,5 @@ Catalog catalog; catalog.filename = skylist->filename[reg]; - catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF; + catalog.catflags = GetMeasures ? DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT : DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; catalog.Nsecfilt = 0; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/fits_db.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/fits_db.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/fits_db.c (revision 38447) @@ -6,22 +6,18 @@ db[0].f = NULL; db[0].filename = NULL; + db[0].dbstate = 0; + db[0].lockstate = 0; + db[0].mode = 0; + db[0].format = 0; + db[0].virtual = 0; + db[0].nativeOrder = 0; + db[0].scaledValue = 0; -# if (1) gfits_init_header (&db->header); gfits_init_matrix (&db->matrix); gfits_init_header (&db->theader); - gfits_init_table (&db->ftable); + gfits_init_table (&db->ftable); db->ftable.header = &db->theader; gfits_init_vtable (&db->vtable); -# else - db[0].header.buffer = NULL; - db[0].matrix.buffer = NULL; - db[0].theader.buffer = NULL; - db[0].ftable.buffer = NULL; - db[0].ftable.header = &db[0].theader; - db[0].vtable.header = NULL; - db[0].vtable.buffer = NULL; - db[0].vtable.row = NULL; -# endif return (TRUE); } @@ -97,5 +93,6 @@ return (FALSE); } - db[0].swapped = FALSE; /* table does not have internal byte-order */ + db[0].nativeOrder = FALSE; /* table does not have internal byte-order */ + db[0].scaledValue = FALSE; /* table has not been scaled by BZERO,BSCALE */ return (TRUE); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/flatcorr_io.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/flatcorr_io.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/flatcorr_io.c (revision 38447) @@ -49,5 +49,5 @@ return (NULL); } - flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL); + flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL, NULL); if (!flatcorrTable->image) { fprintf (stderr, "ERROR: failed to read Flat Correction Images\n"); @@ -64,5 +64,5 @@ return (NULL); } - flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL); + flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL, NULL); if (!flatcorrTable->corr) { fprintf (stderr, "ERROR: failed to read Flat Corrections\n"); @@ -93,5 +93,5 @@ for (j = 0; j < flatcorrTable->image[i].Nx; j++) { ALLOCATE (flatcorrTable->offset[i][j], float, flatcorrTable->image[i].Ny); - memset (flatcorrTable->offset[i][j], 0.0, flatcorrTable->image[i].Ny*sizeof(float)); + memset (flatcorrTable->offset[i][j], 0, flatcorrTable->image[i].Ny*sizeof(float)); } } @@ -161,8 +161,8 @@ ftableImage.header = &theaderImage; - gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage); + gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage, TRUE); ftableCorr.header = &theaderCorr; - gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr); + gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr, TRUE); f = fopen (filename, "w"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/skyregion_io.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/skyregion_io.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libdvo/src/skyregion_io.c (revision 38447) @@ -40,5 +40,5 @@ ALLOCATE (skytable, SkyTable, 1); memset (skytable->hosts, 0, 80); - skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL); + skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL, NULL); if (!skytable[0].regions) { fprintf (stderr, "ERROR: failed to read sky regions\n"); @@ -99,5 +99,5 @@ gfits_print (&header, "HOSTS", "%s", 1, skytable->hosts); - gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions); + gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions, TRUE); f = fopen (filename, "w"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/Makefile =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/Makefile (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/Makefile (revision 38447) @@ -1,5 +1,5 @@ default: install help: - @echo "make options: install libfits man clean dist" + @echo "make options: install libfits man clean dist test test.verbose test.clean" include ../../Makefile.System @@ -13,4 +13,6 @@ INC = $(HOME)/include MAN = $(HOME)/doc +TESTDIR = $(HOME)/test +TESTBIN = $(HOME)/test include ../../Makefile.Common @@ -19,4 +21,18 @@ FULL_CPPFLAGS = $(BASE_CPPFLAGS) -I$(EXT) FULL_LDFLAGS = $(BASE_LDFLAGS) -lohana + +TEST_CFLAGS = $(BASE_CFLAGS) +TEST_CPPFLAGS = $(BASE_CPPFLAGS) +TEST_LDFLAGS = $(BASE_LDFLAGS) -lFITS -lohana -ltap_ohana + + +TESTXTRA = tcomptiming +TESTPROG = zlib ricetest imagecomp tablecomp +$(TESTPROG) : % : $(TESTBIN)/% +$(TESTXTRA) : % : $(TESTBIN)/% +test: $(TESTPROG) + for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS); done +test.verbose: $(TESTPROG) + for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS) -v; done install: $(DESTLIB)/libFITS.a $(DESTLIB)/libFITS.$(DLLTYPE) $(DESTMAN)/fits.1 @@ -37,14 +53,16 @@ MATRIX_OBJ = \ -$(MATR)/F_add_M_value.$(ARCH).o $(MATR)/F_add_M.$(ARCH).o \ -$(MATR)/F_create_M.$(ARCH).o $(MATR)/F_divide_M.$(ARCH).o \ -$(MATR)/F_free_M.$(ARCH).o $(MATR)/F_copy_M.$(ARCH).o \ -$(MATR)/F_write_M.$(ARCH).o $(MATR)/F_get_M_value.$(ARCH).o \ -$(MATR)/F_multiply_M.$(ARCH).o $(MATR)/F_read_M.$(ARCH).o \ -$(MATR)/F_insert_M.$(ARCH).o $(MATR)/F_set_M_value.$(ARCH).o \ -$(MATR)/F_convert_format.$(ARCH).o $(MATR)/F_read_segment.$(ARCH).o \ -$(MATR)/F_read_portion.$(ARCH).o $(MATR)/F_load_M.$(ARCH).o \ -$(MATR)/F_matrix.$(ARCH).o $(MATR)/F_compress_M.$(ARCH).o \ -$(MATR)/F_uncompress_data.$(ARCH).o +$(MATR)/F_add_M_value.$(ARCH).o $(MATR)/F_add_M.$(ARCH).o \ +$(MATR)/F_create_M.$(ARCH).o $(MATR)/F_divide_M.$(ARCH).o \ +$(MATR)/F_free_M.$(ARCH).o $(MATR)/F_copy_M.$(ARCH).o \ +$(MATR)/F_write_M.$(ARCH).o $(MATR)/F_get_M_value.$(ARCH).o \ +$(MATR)/F_multiply_M.$(ARCH).o $(MATR)/F_read_M.$(ARCH).o \ +$(MATR)/F_insert_M.$(ARCH).o $(MATR)/F_set_M_value.$(ARCH).o \ +$(MATR)/F_convert_format.$(ARCH).o $(MATR)/F_read_segment.$(ARCH).o \ +$(MATR)/F_read_portion.$(ARCH).o $(MATR)/F_load_M.$(ARCH).o \ +$(MATR)/F_matrix.$(ARCH).o $(MATR)/F_compress_utils.$(ARCH).o \ +$(MATR)/F_compress_M.$(ARCH).o $(MATR)/F_compress_data.$(ARCH).o \ +$(MATR)/F_uncompress_M.$(ARCH).o $(MATR)/F_uncompress_data.$(ARCH).o \ +$(MATR)/F_swap_raw.$(ARCH).o TABLE_OBJ = \ @@ -55,5 +73,6 @@ $(TABL)/F_set_column.$(ARCH).o $(TABL)/F_get_column.$(ARCH).o \ $(TABL)/F_table_row.$(ARCH).o $(TABL)/F_free_T.$(ARCH).o \ -$(TABL)/F_table_varlength.$(ARCH).o $(TABL)/F_copy_T.$(ARCH).o +$(TABL)/F_table_varlength.$(ARCH).o $(TABL)/F_copy_T.$(ARCH).o \ +$(TABL)/F_compress_T.$(ARCH).o $(TABL)/F_uncompress_T.$(ARCH).o EXTERN_OBJ = \ Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/doc/notes-compress.txt =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/doc/notes-compress.txt (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/doc/notes-compress.txt (revision 38447) @@ -1,2 +1,20 @@ + +20150525 : + +uncompression steps: + + * read in a table: gfits_fread_ftable_data (f, &ftable, FALSE); (no byte swap) + * uncompress the image: gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable); + * get ZIMAGE from heaer & delete + * get ZCMPTYPE + * get ZBITPIX, ZNAXIS, ZNAXISnn + * get ZTILEn (if missing ZTILE1 = Nx, others 1) + * get ZNAME, ZVAL + * ZSIMPLE, ZTENSION, ZEXTEND, ZBLOCKED, ZPCOUNT, ZGCOUNT, etc + * ZSCALE, ZBLANK, OBLANK, ZZERO + + * data is stored in fits table COMPRESSED_DATA (1PB, 1PI, 1PJ) + +------------------------------------------------ TFORMn : rPt(e_max) Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/extern/gzip.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/extern/gzip.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/extern/gzip.c (revision 38447) @@ -2,4 +2,5 @@ # include # include +# define EXTRA_VERBOSE 0 /** the two functions in this file re-implement the uncompress function of zlib. @@ -10,35 +11,35 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.3, July 18th, 2005 - - Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - - - The data format used by the zlib library is described by RFCs (Request for - Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt - (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). + version 1.2.3, July 18th, 2005 + + Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + The data format used by the zlib library is described by RFCs (Request for + Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt + (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). */ /* - The 'zlib' compression library provides in-memory compression and + The 'zlib' compression library provides in-memory compression and decompression functions, including integrity checks of the uncompressed data. This version of the library supports only one compression method @@ -46,5 +47,5 @@ stream interface. - Compression can be done in a single step if the buffers are large + Compression can be done in a single step if the buffers are large enough (for example if an input file is mmap'ed), or can be done by repeated calls of the compression function. In the latter case, the @@ -52,21 +53,21 @@ (providing more output space) before each call. - The compressed data format used by default by the in-memory functions is + The compressed data format used by default by the in-memory functions is the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped around a deflate stream, which is itself documented in RFC 1951. - The library also supports reading and writing files in gzip (.gz) format + The library also supports reading and writing files in gzip (.gz) format with an interface similar to that of stdio using the functions that start with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. - - The zlib format was designed to be compact and fast for use in memory + This library can optionally read and write gzip streams in memory as well. + + The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- file compression on file systems, has a larger header than zlib to maintain directory information, and uses a different, slower check method than zlib. - The library does not install any signal handler. The decoder checks + The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash even in case of corrupted input. @@ -116,37 +117,120 @@ } -// XXX ??? is cfitsio writing data which is incompatible with zlib ??? -int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) -{ - z_stream stream; - int err; - - stream.next_in = (Bytef*)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; - - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - - // MAX_WBITS = 15 - err = inflateInit2(&stream, -15); - if (err != Z_OK) return err; - - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - assert (stream.total_out <= *destLen); - *destLen = stream.total_out; - - err = inflateEnd(&stream); - return err; +# define ESCAPE(RET) { fprintf (stderr, "gzip error in %s @ %s:%d\n", __func__, __FILE__, __LINE__); return (RET); } + +int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { + + z_stream stream; + int i, err; + + stream.next_in = (Bytef*)source; + stream.avail_in = (uInt)sourceLen; + + /* Check for source > 64K on 16-bit machine: */ + if ((uLong)stream.avail_in != sourceLen) ESCAPE (Z_BUF_ERROR); + + stream.next_out = dest; + stream.avail_out = (uInt)*destLen; // allocated space + if ((uLong)stream.avail_out != *destLen) ESCAPE (Z_BUF_ERROR); + + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + + if (EXTRA_VERBOSE) { + fprintf (stderr, "inp cmp: "); + for (i = 0; i < sourceLen; i++) { + fprintf (stderr, "0x%02hhx ", source[i]); + } + fprintf (stderr, "\n"); + } + + // the '1' is the compression level: make this a user argument + // NOTE: cfitsio uses a fixed value of 1 + // I am using deflateInit2 because cfitsio expects gzip, not just zlib + // windowBits = 31 = (15 + 16) = (2^15 window bits) + (create a gzip stream) + err = deflateInit2(&stream, 1, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + if (0) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", stream.avail_in, stream.avail_out, (int) stream.total_out); + + if (err != Z_OK) ESCAPE(err); + + // XXX this is written to do the compression in a single pass. it could be re-done to have + // the compression occur in a series of steps + err = deflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + err = deflateEnd(&stream); + ESCAPE (Z_BUF_ERROR); + } + + if (EXTRA_VERBOSE) { + fprintf (stderr, "out cmp: "); + for (i = 0; i < stream.total_out; i++) { + fprintf (stderr, "0x%02hhx ", dest[i]); + } + fprintf (stderr, "\n"); + } + + assert (stream.total_out <= *destLen); + *destLen = stream.total_out; + + err = deflateEnd(&stream); + return err; } + +// NOTE: CFITSIO uses gzip format, not just zlib +int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { + + z_stream stream; + int i, err; + + stream.next_in = (Bytef*)source; + stream.avail_in = (uInt)sourceLen; + + /* Check for source > 64K on 16-bit machine: */ + if ((uLong)stream.avail_in != sourceLen) ESCAPE (Z_BUF_ERROR); + + stream.next_out = dest; + stream.avail_out = (uInt)*destLen; + if ((uLong)stream.avail_out != *destLen) ESCAPE (Z_BUF_ERROR); + + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + + if (EXTRA_VERBOSE) { + fprintf (stderr, "inp unc: "); + for (i = 0; i < sourceLen; i++) { + fprintf (stderr, "0x%02hhx ", source[i]); + } + fprintf (stderr, "\n"); + } + + // windowBits = 47 = (15 + 32) = (2^15 window bits) + (test for either gzip or zlib + // streams). NOTE: modern versions of zlib (version > 1.2.3.4) allow for windowBits = 0 + // (or 16 or 32) to use the compressed stream windowBits value. IPP cluster still has + // zlib version = 1.2.3, so we cannot use this feature. + err = inflateInit2(&stream, 47); + if (err != Z_OK) ESCAPE (err); + + err = inflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + inflateEnd(&stream); + if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) ESCAPE (Z_DATA_ERROR); + ESCAPE(err); + } + + if (EXTRA_VERBOSE) { + fprintf (stderr, "out unc: "); + for (i = 0; i < stream.total_out; i++) { + fprintf (stderr, "0x%02hhx ", dest[i]); + } + fprintf (stderr, "\n"); + } + + assert (stream.total_out <= *destLen); + *destLen = stream.total_out; + + err = inflateEnd(&stream); + return err; +} + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_copy_H.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_copy_H.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_copy_H.c (revision 38447) @@ -30,2 +30,30 @@ return (TRUE); } + +int gfits_copy_header_ptr (Header *in, Header *out) { + + int i; + + if (!in) return FALSE; + if (!out) return FALSE; + + out[0].simple = in[0].simple; + out[0].unsign = in[0].unsign; + out[0].extend = in[0].extend; + out[0].bitpix = in[0].bitpix; + + out[0].Naxes = in[0].Naxes; + for (i = 0; i < FT_MAX_NAXES; i++) + out[0].Naxis[i] = in[0].Naxis[i]; + + out[0].datasize = in[0].datasize; + + out[0].pcount = in[0].pcount; + out[0].gcount = in[0].gcount; + out[0].bzero = in[0].bzero; + out[0].bscale = in[0].bscale; + + out[0].buffer = in[0].buffer; + + return (TRUE); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_modify.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_modify.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/header/F_modify.c (revision 38447) @@ -48,5 +48,5 @@ strncpy (comment, qe, p + 80 - qe); } - gfits_pad_ending (comment, ' ', 82); /* comment must contain spaces to the end */ + gfits_pad_ending (comment, 0x20, 82); /* comment must contain spaces to the end */ /* write the numeric modes */ @@ -134,5 +134,5 @@ strncpy (comment, qe, p + 80 - qe); } - gfits_pad_ending (comment, ' ', 82); /* comment must contain spaces to the end */ + gfits_pad_ending (comment, 0x20, 82); /* comment must contain spaces to the end */ /* write the boolean mode */ @@ -166,5 +166,5 @@ if (!ptr) goto invalid; strncpy (comment, ptr, 80); - gfits_pad_ending (comment, ' ', 82); /* comment must contain spaces to the end */ + gfits_pad_ending (comment, 0x20, 82); /* comment must contain spaces to the end */ snprintf (string, 81, "%-8s= %s / %-s", field, data, comment); /* this will keep the original line, but truncate the comment */ @@ -296,15 +296,23 @@ */ +// XXX this is an absurd patch on a gcc bug: memset has a problem is value != 0 and N is constant +void myMemset (char *ptr, int value, size_t N) { + + char *p = ptr; + + size_t i = 0; + for (i = 0; i < N; i++, p++) { + *p = value; + } +} + /* fill 'line' with Nbyte space from first NULL to last byte with value */ void gfits_pad_ending (char *line, char value, int Nbyte) { - - char *p; - int N; line[Nbyte-1] = 0; - p = line + strlen (line); - N = MAX (Nbyte - strlen (line) - 1, 0); - memset (p, value, N); + char *p = line + strlen (line); + size_t N = MAX (Nbyte - strlen (line) - 1, 0); + myMemset (p, value, N); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/include/gfitsio.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/include/gfitsio.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/include/gfitsio.h (revision 38447) @@ -73,4 +73,5 @@ off_t datasize; // size of the buffer (including block padding at the end) off_t validsize; // size of the valid portion of the table (< validsize if file is short) + off_t heap_start; // byte offset to start of HEAP } FTable; @@ -85,12 +86,32 @@ } VTable; -typedef struct { - int maxlen; // max size of all table rows - int nbytes; // number of bytes per column element - int Nstart; // byte offset of this column relative to first column - off_t heap_start; // byte offset to start of HEAP - char format; // data format character (one of: XLABIJEDCM) - char mode; // access data pointer type (one of P or Q) +// description / metadata needed for a single varlength column. A varlength column has a +// 2-element array giving the (length,offset) of data in the heap. Below, 'real data' are +// the values in the heap; 'metadata' are the values in the main table. +typedef struct { + int column; // field number (of the metadata) + char format; // format character for real data (one of: XLABIJKEDCM) + char mode; // format character for metadata (one of P [float] or Q [double]) + int maxlen; // max size of all table rows for this column + int nbytes; // number of bytes per real data column element + int offset; // byte offset of the metadata column relative to first column } VarLengthColumn; + +typedef struct { + char ttype[16]; // TTYPE field of original table + char ttype_cmt[80]; // comment associated with TTYPE + char tunit[16]; // TUNIT field of original table + char tunit_cmt[80]; // comment associated with TUNIT + char tformat[16]; // TFORM field of original table + char tformat_cmt[80]; // comment associated with TFORM + char zctype[80]; // compression type for this field + VarLengthColumn zdef; // description of the output variable length column + + char datatype[80]; // named data type associated with ttype (eg, 18J -> int, 18D -> double) + int Nvalues; // number of values per row (eg 18J -> 18 int values) + int pixsize; // number of bytes per value of this row (eg, 18J -> 4 bytes) + int rowsize; // number of bytes for a full row (Nvalues * pixsize) + int offset; // byte offset of the real data column relative to first column +} TableField; # ifndef PROTO @@ -106,4 +127,5 @@ char *gfits_keyword_end PROTO((char *line)); int gfits_copy_header PROTO((Header *in, Header *out)); +int gfits_copy_header_ptr PROTO((Header *in, Header *out)); int gfits_create_header PROTO((Header *header)); int gfits_delete PROTO((Header *header, char *field, int N)); @@ -117,6 +139,6 @@ int gfits_modify PROTO((Header *header, char *field, char *mode, int N,...)) OHANA_FORMAT(printf, 3, 5); int gfits_print PROTO((Header *header, char *field, char *mode, int N,...)) OHANA_FORMAT(printf, 3, 5); -int gfits_modify_alt PROTO((Header *header, char *field, char *mode, int N,...)); -int gfits_print_alt PROTO((Header *header, char *field, char *mode, int N,...)); +int gfits_modify_alt PROTO((Header *header, char *field, char *mode, int N,...)); // do not use a FORMAT: non-standard fmt chars +int gfits_print_alt PROTO((Header *header, char *field, char *mode, int N,...)); // do not use a FORMAT: non-standard fmt chars int gfits_find_Xheader PROTO((FILE *f, Header *header, char *extname)); int gfits_read_Xheader PROTO((char *filename, Header *header, int N)); @@ -124,5 +146,5 @@ int gfits_save_header PROTO((FILE *f, Header *header)); int gfits_scan PROTO((Header *header, char *field, char *mode, int N,...)) OHANA_FORMAT(scanf, 3, 5); -int gfits_scan_alt PROTO((Header *header, char *field, char *mode, int N,...)); +int gfits_scan_alt PROTO((Header *header, char *field, char *mode, int N,...)); // do not use a FORMAT: non-standard fmt chars int gfits_set_unsign_mode PROTO((int mode)); int gfits_stripwhite PROTO((char *string)); @@ -133,4 +155,5 @@ off_t gfits_data_min_size PROTO((Header *header)); off_t gfits_data_pad_size PROTO((off_t rawsize)); +off_t gfits_heap_start PROTO((Header *header)); int gfits_extended_to_primary PROTO((Header *header, int simple, char *comment)); int gfits_primary_to_extended PROTO((Header *header, char *exttype, char *comment)); @@ -166,12 +189,36 @@ void gfits_set_matrix_value PROTO((Matrix *matrix, off_t x, off_t y, double value)); int gfits_write_matrix PROTO((char *filename, Matrix *matrix)); + int gfits_uncompress_image PROTO((Header *header, Matrix *matrix, FTable *ftable)); -int gfits_uncompress_data PROTO((char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize)); -int gfits_distribute_data PROTO((Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero)); -int gfits_byteswap_zdata PROTO((char *zdata, int Nzdata, int bitpix)); -int gfits_extension_is_compressed PROTO((Header *header)); +int gfits_uncompress_data PROTO((char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int Nout_alloc, int out_pixsize)); +int gfits_distribute_data PROTO((Matrix *matrix, char *data, int Ndata, int bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero)); + +int gfits_compress_data PROTO((char *zdata, int *Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *rawdata, int Nraw, int raw_pixsize, int Nx, int Ny)); +int gfits_compress_image PROTO((Header *header, Matrix *matrix, FTable *ftable, int *Ztile, char *zcmptype)); +int gfits_collect_data PROTO((Matrix *matrix, char *raw, int Nraw, int raw_pixsize, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero)); + +int gfits_copy_keywords_compress PROTO((Header *srchead, Header *tgthead)); +int gfits_swap_raw PROTO((Matrix *matrix)); + off_t gfits_tile_size PROTO((Matrix *matrix, int *otile, int *ztile)); +int gfits_imtile_maxsize PROTO((Matrix *matrix, int *ztile)); +int gfits_imtile_count PROTO((Matrix *matrix, int *ztile, int *ntile)); +int gfits_imtile_start PROTO((Matrix *matrix, int *otile)); +int gfits_imtile_next PROTO((Matrix *matrix, int *ztile, int *ntile, int *otile)); + +int gfits_extension_is_compressed_image PROTO((Header *header)); +int gfits_extension_is_compressed_table PROTO((Header *header)); +int gfits_byteswap_zdata PROTO((char *zdata, int Nzdata, int pixsize)); +int gfits_compressed_data_pixsize PROTO((char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions)); int gfits_uncompressed_data_pixsize PROTO((char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions)); -int gfits_vartable_heap_pixsize PROTO((char format)); +int gfits_uncompressed_data_bitpix PROTO((char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions)); +int gfits_vartable_heap_pixsize PROTO((char format)); +int gfits_cmptype_valid PROTO((char *cmptype)); + +int gfits_varlength_column_add_data PROTO((FTable *table, char *data, off_t Ndata, int row, VarLengthColumn *column)); +void *gfits_varlength_column_pointer PROTO((FTable *ftable, VarLengthColumn *column, off_t row, off_t *length)); +int gfits_varlength_column_define PROTO((FTable *ftable, VarLengthColumn *def, int column)); +int gfits_varlength_column_finish PROTO((FTable *ftable, VarLengthColumn *def)); +int gfits_byteswap_varlength_column PROTO((FTable *ftable, int column)); /******************************* Table functions *************/ @@ -182,4 +229,5 @@ char *gfits_table_print PROTO((FTable *ftable,...)); int gfits_add_rows PROTO((FTable *ftable, char *data, off_t Nrow, off_t Nbytes)); +int gfits_set_table_rows PROTO((Header *header, FTable *table, off_t Nrows)); int gfits_bintable_format PROTO((char *format, char *type, int *Nval, int *Nbytes)); int gfits_create_table PROTO((Header *header, FTable *ftable)); @@ -198,7 +246,9 @@ int gfits_fwrite_ftable_range PROTO((FILE *f, FTable *table, off_t start, off_t Nrows, off_t Ndisk, off_t Ntotal)); int gfits_get_bintable_column PROTO((Header *header, FTable *table, char *label, void **data)); +int gfits_get_bintable_column_raw PROTO((Header *header, FTable *table, char *label, void **data, char nativeOrder)); int gfits_get_bintable_column_type PROTO((Header *header, char *label, char *type, int *Nval)); +void *gfits_get_bintable_column_data PROTO((Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol)); int gfits_get_bintable_column_type_by_N PROTO((Header *header, int N, char *type, int *Nval)); -void *gfits_get_bintable_column_data PROTO((Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol)); +void *gfits_get_bintable_column_data_raw PROTO((Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol, char nativeOrder)); int gfits_get_table_column PROTO((Header *header, FTable *table, char *label, void **data)); int gfits_get_table_column_type PROTO((Header *header, char *label, char *type)); @@ -206,5 +256,5 @@ int gfits_read_table PROTO((char *filename, FTable *ftable)); int gfits_set_bintable_column PROTO((Header *header, FTable *table, char *label, void *data, off_t Nrow)); -int gfits_set_bintable_column_reformat PROTO((Header *header, FTable *table, char *label, char *intype, void *data, off_t Nrow)); +int gfits_set_bintable_column_reformat PROTO((Header *header, FTable *table, char *label, char *intype, void *data, off_t Nrow, char nativeOrder)); int gfits_set_table_column PROTO((Header *header, FTable *table, char *label, void *data, off_t Nrow)); int gfits_table_column PROTO((FTable *ftable, char *field, char *mode,...)) OHANA_FORMAT(printf, 3, 4); @@ -218,4 +268,5 @@ int gfits_copy_ftable PROTO((FTable *in, FTable *out)); int gfits_copy_vtable PROTO((VTable *in, VTable *out)); +int gfits_copy_ftable_ptr PROTO((FTable *in, FTable *out)); int gfits_create_Theader PROTO((Header *header, char *type)); @@ -226,6 +277,11 @@ int gfits_write_Theader PROTO((char *filename, Header *header)); -int gfits_varlength_column_define PROTO((FTable *ftable, VarLengthColumn *def, int column)); -void *gfits_varlength_column_pointer PROTO((FTable *ftable, VarLengthColumn *column, off_t row, off_t *length)); +int gfits_compress_table PROTO((FTable *srctable, FTable *tgttable, int ztilelen, char *zcmptype)); +int gfits_uncompress_table PROTO((FTable *srctable, FTable *tgttable)); + +// XXX EAM for testing +int gfits_dump_raw_table (FTable *table, char *message); +int gfits_dump_cmp_table (FTable *table, char *message); + #endif /* FITSIO */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_M.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_M.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_M.c (revision 38447) @@ -2,313 +2,306 @@ # include # include - -# define ESCAPE { \ - if (ztile != NULL) free (ztile); \ - if (optname != NULL) { \ - for (j = 0; j < Noptions; j++) { \ - free (optname[j]); \ - free (optvalue[j]); \ - } \ - free (optname); \ - free (optvalue); \ - } \ - if (out != NULL) free (out); \ - if (otile != NULL) free (otile); \ - if (ntile != NULL) free (ntile); \ - return (FALSE); } - -# define MOD_KEYWORD(ZNAME,NAME,TYPE,IN,OUT) { \ - if (gfits_scan (header, ZNAME, TYPE, 1, IN)) { \ - gfits_modify (header, NAME, TYPE, 1, OUT); \ - } \ - gfits_delete (header, ZNAME, 1); } - -# define MOD_KEYWORD_ALT(ZNAME,NAME,TYPE,IN,OUT) { \ - if (gfits_scan_alt (header, ZNAME, TYPE, 1, IN)) { \ - gfits_modify_alt (header, NAME, TYPE, 1, OUT); \ - } \ - gfits_delete (header, ZNAME, 1); } - -# define MOD_KEYWORD_REQUIRED(ZNAME,NAME,TYPE,IN,OUT) { \ - if (!gfits_scan (header, ZNAME, TYPE, 1, IN)) ESCAPE; \ - gfits_delete (header, ZNAME, 1); \ - gfits_modify (header, NAME, TYPE, 1, OUT); } - -int gfits_uncompress_image (Header *header, Matrix *matrix, FTable *ftable) { - - off_t Nzdata, Nzrows, zcol; - int i, j, status, zimage, Nout, max_tile_size; - char cmptype[80]; - char zaxis[10], naxis[10], key[10], word[81], exttype[81], checksum[81], datasum[81]; - int Noptions, NOPTIONS, zblank, oblank; - VarLengthColumn zdef; - float zscale, zzero; - - int zdata_pixsize, odata_pixsize; - - int *ztile = NULL; +# define VERBOSE_DUMP 0 + +int gfits_collect_gzp2 (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero); + +// the user needs to specify the various compression options: +// ztile[i] (may be NULL) +// zcmptype + +// steps to convert an image to a compressed table: +// * determine the number of tiles (from ztile[] and image size) +// * construct an empty table with the right dimensions (1 column, Ntile rows) +// * loop over tiles +// * extract tile data to a buffer +// * compress the buffer data +// * insert in table heap +// * update header keywords + +// we do not try to support compression of an image with an associated table (not valid) + +# define ESCAPE { fprintf (stderr, "error in %s @ line %d\n", __func__, __LINE__); goto escape; } + +int gfits_compress_image (Header *header, Matrix *matrix, FTable *ftable, int *Ztile, char *zcmptype) { + + int i; + + // pointers to track the tile data int *otile = NULL; int *ntile = NULL; - char **optname = NULL; - char **optvalue = NULL; - char *out = NULL; + char *raw = NULL; char *zdata = NULL; - Noptions = 0; - - // is ZIMAGE present? - // NOTE target of %t must be int length - status = gfits_scan_alt (ftable->header, "ZIMAGE", "%t", 1, &zimage); - if (!status || !zimage) ESCAPE; - - // copy original header to output header - gfits_copy_header (ftable->header, header); - - // delete ZIMAGE from output header - gfits_delete (header, "ZIMAGE", 1); - - // extract compression-specific keywords, update header as needed. - if (!gfits_scan (header, "ZCMPTYPE", "%s", 1, cmptype)) ESCAPE; - gfits_delete (header, "ZCMPTYPE", 1); - - MOD_KEYWORD_REQUIRED ("ZBITPIX", "BITPIX", "%d", &header->bitpix, header->bitpix); - MOD_KEYWORD_REQUIRED ("ZNAXIS", "NAXIS", "%d", &header->Naxes, header->Naxes); + int *ztile = Ztile; + + Header *theader = ftable->header; + + if (!gfits_cmptype_valid(zcmptype)) ESCAPE; + + // determine the number of tiles (from ztile[] and image size) + if (!ztile) { + ALLOCATE (ztile, int, matrix->Naxes); + ztile[0] = matrix->Naxis[0]; + for (i = 1; i < matrix->Naxes; i++) ztile[i] = 1; + } + ALLOCATE (ntile, int, matrix->Naxes); + ALLOCATE (otile, int, matrix->Naxes); + + int Ntile = gfits_imtile_count (matrix, ztile, ntile); + + // creates an empty (Naxes = 2, Naxis[i] = 0) table header with XTENSION = BINTABLE, EXTNAME = COMPRESSED_IMAGE + if (!gfits_create_table_header (theader, "BINTABLE", "COMPRESSED_IMAGE")) ESCAPE; + + // by using "P" format, we are limited to 32bit pointers (2GB heap) + // XXX how is the "B" format used? + if (!gfits_define_bintable_column (theader, "1PB(0)", "COMPRESSED_DATA", "compressed image data", "none", 1.0, 0.0)) ESCAPE; + if (!gfits_delete (theader, "TUNIT1", 1)) ESCAPE; + + // allocates the default data array (ftable->buffer) + if (!gfits_create_table (theader, ftable)) ESCAPE; + + // copy original header to output header (XXX this needs to be finished) + if (!gfits_copy_keywords_compress (header, theader)) ESCAPE; + + // this allocates the pointer data array for the full set of tiles (NOT the heap) + char *tmpbuffer = NULL; + ALLOCATE_ZERO (tmpbuffer, char, Ntile); + if (!gfits_set_bintable_column (theader, ftable, "COMPRESSED_DATA", tmpbuffer, Ntile)) ESCAPE; + free (tmpbuffer); + + // add ZIMAGE from output header + if (!gfits_modify_alt (theader, "ZIMAGE", "%t", 1, TRUE)) ESCAPE; + + // define compression-specific keywords, update header as needed. + if (!gfits_modify (theader, "ZCMPTYPE", "%s", 1, zcmptype)) ESCAPE; + + // define compression-specific keywords, update header as needed. + if (!strcasecmp(zcmptype, "GZIP_1") || !strcasecmp(zcmptype, "GZIP_2")) { + if ((header[0].bitpix == -32) || (header[0].bitpix == -64)) { + if (!gfits_modify (theader, "ZQUANTIZ", "%s", 1, "NONE")) ESCAPE; + } + } + + if (!gfits_modify_alt (theader, "ZSIMPLE", "%t", 1, TRUE)) ESCAPE; + // gfits_modify (header, "ZTENSION", "%s", 1, exttype); + + // supply the Z* header values from the source image header + if (!gfits_modify (theader, "ZBITPIX", "%d", 1, header[0].bitpix)) ESCAPE; + + if (!gfits_modify (theader, "ZNAXIS", "%d", 1, header[0].Naxes)) ESCAPE; + + char keyword[11]; + for (i = 0; i < header[0].Naxes; i++) { + snprintf (keyword, 10, "ZNAXIS%d", i + 1); + if (!gfits_modify (theader, keyword, OFF_T_FMT, 1, header[0].Naxis[i])) ESCAPE; + } for (i = 0; i < header->Naxes; i++) { - snprintf (zaxis, 10, "ZNAXIS%d", i + 1); - snprintf (naxis, 10, "NAXIS%d", i + 1); - MOD_KEYWORD_REQUIRED (zaxis, naxis, OFF_T_FMT, &header->Naxis[i], header->Naxis[i]); - } - - // set up the tile sizes : ztile is the default size of the tile in the Nth dimension - // the actual tile size may be smaller at the edge of a dimension. if the ZTILEn - // entries are not found, default to [Nx,1,1,...] - ALLOCATE (ztile, int, header->Naxes); - if (!gfits_scan (header, "ZTILE1", "%d", 1, &ztile[0])) { - ztile[0] = header->Naxis[0]; - for (i = 1; i < header->Naxes; i++) { - ztile[i] = 1; - } - } else { - gfits_delete (header, "ZTILE1", 1); - for (i = 1; i < header->Naxes; i++) { - snprintf (key, 10, "ZTILE%d", i + 1); - if (!gfits_scan (header, key, "%d", 1, &ztile[i])) ESCAPE; - gfits_delete (header, key, 1); - } - } + snprintf (keyword, 10, "ZTILE%d", i + 1); + if (!gfits_modify (theader, keyword, "%d", 1, ztile[i])) ESCAPE; + } + + if (!gfits_modify (theader, "BSCALE", "%lf", 1, header[0].bscale)) ESCAPE; + if (!gfits_modify (theader, "BZERO", "%lf", 1, header[0].bzero)) ESCAPE; + + // other keywords to define: + // ZEXTEND + // ZBLOCKED + // ZHECKSUM + // ZDATASUM + // PCOUNT -> ZPCOUNT + // GCOUNT -> ZGCOUNT + + float zscale = 1.0; + // gfits_modify (header, "ZSCALE", "%f", 1, zscale); + + float zzero = 0.0; + // gfits_modify (header, "ZZERO", "%f", 1, zzero); + + int zblank = 32767; + // gfits_modify (header, "ZBLANK", "%d", 1, zblank); + + int oblank = 32767; + // gfits_modify (header, "BLANK", "%d", 1, oblank); + +# if (0) + // need to set ZNAMEnn and ZVALnn if they are defined and used // search for algorithm-specific keywords. these are used to control compression options // note the difference between the keywords (1 indexed) and the variables (0 indexed) - NOPTIONS = 10; - ALLOCATE (optname, char *, NOPTIONS); - ALLOCATE (optvalue, char *, NOPTIONS); - for (Noptions = 0; TRUE; Noptions++) { - snprintf (key, 10, "ZNAME%d", Noptions + 1); - if (!gfits_scan (header, key, "%s", 1, word)) break; - gfits_delete (header, key, 1); - optname[Noptions] = strcreate (word); - - snprintf (key, 10, "ZVAL%d", Noptions + 1); - if (!gfits_scan (header, key, "%s", 1, word)) ESCAPE; - gfits_delete (header, key, 1); - optvalue[Noptions] = strcreate (word); - - if (Noptions == NOPTIONS - 1) { - NOPTIONS += 10; - REALLOCATE (optname, char *, NOPTIONS); - REALLOCATE (optvalue, char *, NOPTIONS); - } - } - - // XXX get ZMASKCMP - int zsimple, have_zsimple; - int have_ztension; - - // NOTE target of %t must be int length - have_zsimple = gfits_scan_alt (header, "ZSIMPLE", "%t", 1, &zsimple); - have_ztension = gfits_scan (header, "ZTENSION", "%s", 1, exttype); - - // this is a very bogus case: we cannot have both keywords - if (have_zsimple && have_ztension) ESCAPE; - - // if neither are present, we have an image that is not really following the standard: - // assume it is a PHU - if (!have_zsimple && !have_ztension) { - header->simple = TRUE; - gfits_extended_to_primary (header, header->simple, "Image data"); - - MOD_KEYWORD_ALT ("ZEXTEND", "EXTEND", "%t", &header->extend, header->extend); - MOD_KEYWORD_ALT ("ZBLOCKED", "BLOCKED", "%t", &header->extend, header->extend); - } - - // have_zsimple : image comes from a PHU - if (have_zsimple) { - header->simple = zsimple; - gfits_delete (header, "ZSIMPLE", 1); - gfits_extended_to_primary (header, header->simple, "Image data"); - - MOD_KEYWORD_ALT ("ZEXTEND", "EXTEND", "%t", &header->extend, header->extend); - MOD_KEYWORD_ALT ("ZBLOCKED", "BLOCKED", "%t", &header->extend, header->extend); - } - - // have_ztension : image comes from an extension - if (have_ztension) { - gfits_delete (header, "ZTENSION", 1); - gfits_modify_extended (header, exttype, "Image extension"); - - // we may have an uncompressed PCOUNT / GCOUNT value, otherwise set to 0,1 - if (gfits_scan (header, "ZPCOUNT", "%d", 1, &header->pcount)) { - gfits_delete (header, "ZPCOUNT", 1); - gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + for (i = 0; i < Noptions; i++) { + snprintf (key, 10, "ZNAME%d", i + 1); + if (!gfits_modify (header, key, "%s", 1, optname[i])) break; + + snprintf (key, 10, "ZVAL%d", i + 1); + if (!gfits_modify (header, key, "%s", 1, optvalue[i])) break; + } +# endif + + VarLengthColumn zdef; + if (!gfits_varlength_column_define (ftable, &zdef, 1)) ESCAPE; + + // pixel sizes and tile sizes: + // * We have Ntile tiles, each of Nx * Ny * Nz ... tile pixels + // -> A pixel in the tile (in the image) has size tile_pixsize = f(BITPIX) + // * We copy the pixels in the tile to a continuous buffer to be compressed. + // During this copy, we can scale the data (eg, floats -> ints), in which + // case the raw buffer may have a different pixel size (and type) than the + // image tile + // -> A pixel in the raw buffer is f(zcmptype,BITPIX): + // -- GZIP_1 : raw_pixsize = tile_pixsize (no scaling is performed) + // -- GZIP_2 : raw_pixsize = tile_pixsize (no scaling is performed) + // -- RICE_1 : raw pixels must be integers. raw_pixsize is user specified? (ZNAMEn,ZVALn : BYTEPIX = 1,2,4,8) + // -- PLIO_1 : raw_pixsize = 2 bytes + // -- HCOMPRESS_1 : raw pixels must be integers. + + // size (in pixels) of the largest tile + int max_tile_size = gfits_imtile_maxsize (matrix, ztile); + + // size of a pixel in the raw pixel buffer (before compression) + int raw_pixsize = gfits_uncompressed_data_pixsize (zcmptype, header[0].bitpix, NULL, NULL, 0); + int raw_bitpix = gfits_uncompressed_data_bitpix (zcmptype, header[0].bitpix, NULL, NULL, 0); + int cmp_pixsize = gfits_compressed_data_pixsize (zcmptype, header[0].bitpix, NULL, NULL, 0); + + // define compression-specific keywords, update header as needed. + if (!strcasecmp(zcmptype, "RICE_1") || !strcasecmp(zcmptype, "RICE_ONE")) { + if (!gfits_modify (theader, "ZNAME1", "%s", 1, "BLOCKSIZE")) ESCAPE; + if (!gfits_modify (theader, "ZVAL1", "%d", 1, 32)) ESCAPE; + if (!gfits_modify (theader, "ZNAME2", "%s", 1, "BYTEPIX")) ESCAPE; + if (!gfits_modify (theader, "ZVAL2", "%d", 1, raw_pixsize)) ESCAPE; + } + + // size of a pixel in the image tile -- this is probably not needed +# if VERBOSE_DUMP + int tile_pixsize = abs(header[0].bitpix) / 8; + fprintf (stderr, "raw_pixsize: %d, cmp_pixsize: %d, tile_pixsize: %d, raw_bitpix: %d\n", raw_pixsize, cmp_pixsize, tile_pixsize, raw_bitpix); +# endif + + // allocate the buffer for compression work + int Nzdata_alloc = raw_pixsize*max_tile_size + 100; + ALLOCATE (raw, char, raw_pixsize*max_tile_size); + ALLOCATE (zdata, char, Nzdata_alloc); + + // init the otile[] counters + if (!gfits_imtile_start (matrix, otile)) ESCAPE; + + // compress the data : copy into a tile, compress the tile, then add to the output table + // each tile -> 1 row of the output table + for (i = 0; i < Ntile; i++) { + + // size of the current tile in pixels + int Nraw = gfits_tile_size (matrix, otile, ztile); + + // copy the raw pixels from their native matrix locations to the temporary output buffer + // for float -> int scaling by zscale, zzero may be applied + if (!strcasecmp (zcmptype, "GZIP_2") || !strcasecmp (zcmptype, "NONE_2")) { + if (!gfits_collect_gzp2 (matrix, raw, Nraw, raw_bitpix, otile, oblank, ztile, zblank, zscale, zzero)) ESCAPE; } else { - header->pcount = 0; - gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); - } - if (gfits_scan (header, "ZGCOUNT", "%d", 1, &header->gcount)) { - gfits_delete (header, "ZGCOUNT", 1); - gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); - } else { - header->pcount = 1; - gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); - } - } else { - header->pcount = 0; - header->gcount = 1; - gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); - gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); - } - - MOD_KEYWORD ("ZHECKSUM", "CHECKSUM", "%s", checksum, checksum); - MOD_KEYWORD ("ZDATASUM", "DATASUM", "%s", datasum, datasum); - - zscale = 1; - gfits_scan (header, "ZSCALE", "%f", 1, &zscale); - - zblank = 32767; - gfits_scan (header, "ZBLANK", "%d", 1, &zblank); - - oblank = 32767; - gfits_scan (header, "BLANK", "%d", 1, &oblank); - - zzero = 0; - gfits_scan (header, "ZZERO", "%f", 1, &zzero); - - // find the COMPRESSED_DATA column (format should be 1PB, 1PI, 1PJ) - for (i = 1; TRUE; i++) { - snprintf (key, 10, "TTYPE%d", i); - if (!gfits_scan (ftable->header, key, "%s", 1, word)) ESCAPE; - if (!strcmp (word, "COMPRESSED_DATA")) break; - } - zcol = i; - - if (!gfits_varlength_column_define (ftable, &zdef, zcol)) ESCAPE; - gfits_delete (header, "TFIELDS", 1); - snprintf (key, 10, "TTYPE"OFF_T_FMT, zcol); - gfits_delete (header, key, 1); - snprintf (key, 10, "TFORM"OFF_T_FMT, zcol); - gfits_delete (header, key, 1); - - // create the output image - gfits_create_matrix (header, matrix); - - // counters for tile numbers - ALLOCATE (otile, int, matrix->Naxes); - ALLOCATE (ntile, int, matrix->Naxes); - max_tile_size = 1; - for (i = 0; i < matrix->Naxes; i++) { - otile[i] = 0; - ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]); - max_tile_size *= ztile[i]; - } - - // this takes place in three major steps: - // 1) read the table data : zdef.format tells the size of the varlength heap element - // 2) uncompress the data : pixel size depends on compression method - // 3) distribute data to the tiles : output pixel size depends on header.bitpix - - // heapdata -> zdata -> odata -> idata - - // heapdata.pixsize : zdef.format - // zdata.pixsize : depends on compression - // odata.pixsize : depends on compression - // idata.pixsize : header.bitpix - - // size of an element in the vartable heap section - zdata_pixsize = gfits_vartable_heap_pixsize (zdef.format); - - // size of a pixel in the final image (not needed) - // idata_pixsize = abs(header[0].bitpix) / 8; - - // size of a pixel in the output from the decompression routine - odata_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions); - ALLOCATE (out, char, odata_pixsize*max_tile_size); - - off_t row; - - // uncompress the data - Nzrows = ftable->header->Naxis[1]; - for (row = 0; row < Nzrows; row++) { - - // expected output size for this tile - Nout = gfits_tile_size (matrix, otile, ztile); - - zdata = gfits_varlength_column_pointer (ftable, &zdef, row, &Nzdata); - if (!zdata) return (FALSE); - - // XXX not certain this is the correct place to do the swap (or if zdata_pixsize is - // the correct size to guide the swap) - - if (strcasecmp(cmptype, "GZIP_1")) { - if (!gfits_byteswap_zdata (zdata, Nzdata, zdata_pixsize)) return (FALSE); - } - - // gfits_uncompress_data uncompresses from zdata to the temporary output buffer which must be allocated - // XXX the tile must not be > 2GB - if (!gfits_uncompress_data ((char *)zdata, Nzdata, cmptype, optname, optvalue, Noptions, out, &Nout, odata_pixsize)) return (FALSE); - - // copy the uncompressed pixels into their correct locations - if (!gfits_distribute_data (matrix, odata_pixsize, out, Nout, otile, oblank, ztile, zblank, zscale, zzero)) return (FALSE); - - // update the tile counters, carrying to the next dimension if needed - for (j = 0; j < matrix->Naxes; j++) { - otile[j] ++; - if (otile[j] == ntile[j]) { - otile[j] = 0; - } else { - break; - } - } - } + if (!gfits_collect_data (matrix, raw, Nraw, raw_bitpix, otile, oblank, ztile, zblank, zscale, zzero)) ESCAPE; + } + + if (VERBOSE_DUMP && (i == 0)) { + int k; + fprintf (stderr, "cmp mat: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", matrix->buffer[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + fprintf (stderr, "cmp raw: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", raw[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + // gzip compresses bytes which are in BIG-ENDIAN order + // Nraw is number of pixels + if (!strcasecmp(zcmptype, "GZIP_1")) { + if (!gfits_byteswap_zdata (raw, Nraw * raw_pixsize, raw_pixsize)) ESCAPE; + } + int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression + + if (VERBOSE_DUMP && (i == 0)) { + int k; + fprintf (stderr, "cmp swp: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", raw[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + fprintf (stderr, "Nzdata: %d -> ", Nzdata); + } + + // optname, optvalue = NULL, Noptions = 0 : defaults for RICE_1, HCOMPRESS_1 + + if (!gfits_compress_data (zdata, &Nzdata, zcmptype, NULL, NULL, 0, raw, Nraw, raw_pixsize, ztile[1], ztile[0])) ESCAPE; + if (VERBOSE_DUMP && (i == 0)) { + int k; + fprintf (stderr, "%d\n", Nzdata); + fprintf (stderr, "cmp SWP: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", zdata[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + // GZIP_1 is swapped before compression; GZIP_2 implies swapping. All other + // compression modes require swapping after compression (compresssion & decompression + // operate on native ENDIAN) + if (strcasecmp(zcmptype, "NONE_2") && // strcasecmp(zcmptype, "NONE") && + strcasecmp(zcmptype, "GZIP_1") && + strcasecmp(zcmptype, "GZIP_2") && + strcasecmp(zcmptype, "RICE_1") && + strcasecmp(zcmptype, "RICE_ONE")) { + // Nzdata is number of bytes + if (!gfits_byteswap_zdata (zdata, Nzdata, cmp_pixsize)) ESCAPE; + } + + if (VERBOSE_DUMP && (i == 0)) { + int k; + fprintf (stderr, "cmp dat: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", zdata[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + if (!gfits_varlength_column_add_data (ftable, zdata, Nzdata, i, &zdef)) ESCAPE; + + // update the otile[] counters, carrying to the next dimension if needed + gfits_imtile_next (matrix, ztile, ntile, otile); + } + + if (!gfits_varlength_column_finish (ftable, &zdef)) ESCAPE; + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "cmp tbl: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", ftable->buffer[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + free (raw); + free (zdata); + + free (otile); + free (ntile); + + if (!Ztile) free (ztile); return (TRUE); + +escape: + FREE (raw); + FREE (zdata); + + FREE (otile); + FREE (ntile); + + if (!Ztile) FREE (ztile); + return FALSE; } -// gfits_varlength returns a pointer to a chunk of Nzdata of data elements starting at zdata. - -// XXX need to have an API to send the user data - -// XXX need to byte-swap the table column; this needs to be worked out more clearly -// in the APIs: do the table read / column extract functions swap or not?. we have -// put the byte-swapping in gifts_varlength_column_pointer, but this is fairly weak. - -// XXX inconsistency between data element size between compressed data, uncompressed data, -// and output pixel data..... - -// true sizes of this tile (in pixels) -off_t gfits_tile_size (Matrix *matrix, int *otile, int *ztile) { - - off_t i, Npixels, Ndimen; - - Npixels = 1; - for (i = 0; i < matrix->Naxes; i++) { - Ndimen = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]); - Npixels *= Ndimen; - } - - return (Npixels); +int gfits_copy_keywords_compress (Header *srchead, Header *tgthead) { + + int i; + + for (i = 0; i < srchead->datasize; i += FT_LINE_LENGTH) { + } + return TRUE; } -// bitpix is the input data size/type -int gfits_distribute_data (Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) { +// raw_pixsize is the bytes / pixel for the input matrix +// place the raw image bytes for the current tile into the tile buffer +// * otile is the counter for the current output tile +// * ztile gives the size of the i-th dimension of the current tile +// * raw_bitpix defines the size and type of the raw buffer +int gfits_collect_data (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) { int i, j, start, offset, coord, Nline; @@ -316,9 +309,7 @@ int *Ztile = NULL; - ALLOCATE (counter, int, matrix->Naxes); - ALLOCATE (Ztile, int, matrix->Naxes); - - // counter for current row in tile to copy - // true sizes of this tile (in pixels) + ALLOCATE (counter, int, matrix->Naxes); // counter for current row in tile to copy + ALLOCATE (Ztile, int, matrix->Naxes); // true sizes of this tile (in pixels) + for (i = 0; i < matrix->Naxes; i++) { counter[i] = 0; @@ -332,8 +323,8 @@ } - // double check reported size (needs the Isize) - assert (Ndata == Ztile[0]*Nline); - - // set the starting point of the tile: + // confirm the tile size: + assert (Nraw == Ztile[0]*Nline); + + // set the starting point of the tile in the matrix buffer // start = otile[0]*ztile[0] + otile[1]*ztile[1]*Naxis[0] + otile[2]*ztile[2]*Naxis[0]*Naxis[1] + ...; // start = otile[0]*ztile[0] + Naxis[0]*(otile[1]*ztile[1] + Naxis[1]*(otile[2]*ztile[2] + ...)); @@ -347,52 +338,82 @@ offset = 0; + int directCopy = (zzero == 0.0) && (zscale == 1.0); + // we need to set up switches for all of the possible combinations: // this macro is used at the inner switch to run the actual loop -# define SCALE_AND_DIST(TYPE, SIZE) { \ - TYPE *Optr = (TYPE *) &matrix->buffer[SIZE*(offset + start)]; \ - for (j = 0; j < Ztile[0]; j++, Iptr++, Optr++) { \ - if (*Iptr == zblank) { \ - *Optr = oblank; \ - } else { \ - *Optr = *Iptr * zscale + zzero; \ +# define SCALE_AND_DIST_INT_PRINT(TYPE, SIZE, INTYPE) { \ + TYPE *TILEptr = (TYPE *) &matrix->buffer[SIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (*TILEptr == oblank) { \ + *RAWptr = zblank; \ + } else { \ + /* *RAWptr = directCopy ? *TILEptr : (*TILEptr - zzero) / zscale; */ \ + *RAWptr = *TILEptr; \ + fprintf (stderr, "col: %s to %s : %08x : %08x\n", #TYPE, #INTYPE, *TILEptr, *RAWptr); \ } } } - // this macro sets up the outer switch and calls above macro with all output bitpix options -# define SETUP_INSIZE(TYPE, SIZE) { \ - TYPE *Iptr = (TYPE *) &data[i*SIZE*Ztile[0]]; \ - switch (matrix->bitpix) { \ - case 8: \ - SCALE_AND_DIST (char, 1); \ - break; \ - case 16: \ - SCALE_AND_DIST (short, 2); \ - break; \ - case 32: \ - SCALE_AND_DIST (int, 4); \ - break; \ - case -32: \ - SCALE_AND_DIST (float, 4); \ - break; \ - case -64: \ - SCALE_AND_DIST (double, 8); \ - break; \ - default: \ - abort(); \ + // we need to set up switches for all of the possible combinations: + // this macro is used at the inner switch to run the actual loop +# define SCALE_AND_DIST_INT(OTYPE, OSIZE) { \ + OTYPE *TILEptr = (OTYPE *) &matrix->buffer[OSIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (*TILEptr == oblank) { \ + *RAWptr = zblank; \ + } else { \ + if (directCopy) { \ + *RAWptr = *TILEptr; \ + } else { \ + *RAWptr = (*TILEptr - zzero) / zscale; \ + } } } } + + // we need to set up switches for all of the possible combinations: + // this macro is used at the inner switch to run the actual loop +# define SCALE_AND_DIST_FLOAT(OTYPE, OSIZE) { \ + OTYPE *TILEptr = (OTYPE *) &matrix->buffer[OSIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (!isfinite(*TILEptr)) { \ + *RAWptr = zblank; \ + } else { \ + if (directCopy) { \ + *RAWptr = *TILEptr; \ + } else { \ + *RAWptr = (*TILEptr - zzero) / zscale; \ + } } } } + + // this macro sets up the outer switch and calls above the macro with all RAW buffer bitpix options + // +# define SETUP_RAWSIZE_PRINT(TYPE, SIZE) { \ + TYPE *RAWptr = (TYPE *) &raw[i*SIZE*Ztile[0]]; \ + switch (matrix->bitpix) { \ + case 8: SCALE_AND_DIST_INT (char, 1); break; \ + case 16: SCALE_AND_DIST_INT (short, 2); break; \ + case 32: SCALE_AND_DIST_INT_PRINT (int, 4, TYPE); break; \ + case -32: SCALE_AND_DIST_FLOAT (float, 4); break; \ + case -64: SCALE_AND_DIST_FLOAT (double, 8); break; \ + default: abort(); \ + } } + + // this macro sets up the outer switch and calls above the macro with all RAW buffer bitpix options + // +# define SETUP_RAWSIZE(ITYPE, ISIZE) { \ + ITYPE *RAWptr = (ITYPE *) &raw[i*ISIZE*Ztile[0]]; \ + switch (matrix->bitpix) { \ + case 8: SCALE_AND_DIST_INT (char, 1); break; \ + case 16: SCALE_AND_DIST_INT (short, 2); break; \ + case 32: SCALE_AND_DIST_INT (int, 4); break; \ + case -32: SCALE_AND_DIST_FLOAT (float, 4); break; \ + case -64: SCALE_AND_DIST_FLOAT (double, 8); break; \ + default: abort(); \ } } // loop over lines in the tile for (i = 0; i < Nline; i++) { - switch (bitpix) { - case 1: - SETUP_INSIZE (char, 1); - break; - case 2: - SETUP_INSIZE (short, 2); - break; - case 4: - SETUP_INSIZE (int, 4); - break; - default: - abort(); + switch (raw_bitpix) { + case 8: SETUP_RAWSIZE (char, 1); break; + case 16: SETUP_RAWSIZE (short, 2); break; + case 32: SETUP_RAWSIZE (int, 4); break; + case -32: SETUP_RAWSIZE (float, 4); break; + case -64: SETUP_RAWSIZE (double, 8); break; + default: abort(); } @@ -421,123 +442,107 @@ free (counter); free (Ztile); + return (TRUE); } -int gfits_byteswap_zdata (char *zdata, int Nzdata, int pixsize) { - -# define DOSWAP(A,B) { char tmp = A; A = B; B = tmp; } - -# ifdef BYTE_SWAP - - int i; - - switch (pixsize) { - case 1: - break; - - case 2: - for (i = 0; i < 2*Nzdata; i+=2) { - DOSWAP (zdata[i+0], zdata[i+1]); +// GZIP_2 organizes data in bytes by significance: (A1 A2 A3 A4), (B1 B2 B3 B4) -> (A1 B1), (A2, B2), (A3, B3), (A4, B4) +// NOTE that this organization depends on the machine endian state. +// raw_pixsize is the bytes / pixel for the input matrix +// place the raw image bytes for the current tile into the tile buffer +// * otile is the counter for the current output tile +// * ztile gives the size of the i-th dimension of the current tile +// * raw_bitpix defines the size and type of the raw buffer +int gfits_collect_gzp2 (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) { + + int i, j, k; + int *counter = NULL; + int *Ztile = NULL; + + ALLOCATE (counter, int, matrix->Naxes); // counter for current row in tile to copy + ALLOCATE (Ztile, int, matrix->Naxes); // true sizes of this tile (in pixels) + + for (i = 0; i < matrix->Naxes; i++) { + counter[i] = 0; + Ztile[i] = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]); + } + + // number of lines in the tile (in pixels) + int Nline = 1; + int Npix = matrix->Naxis[0]; + for (i = 1; i < matrix->Naxes; i++) { + Nline *= Ztile[i]; + Npix *= matrix->Naxis[i]; + } + + // confirm the tile size: + assert (Nraw == Ztile[0]*Nline); + + // set the starting point of the tile in the matrix buffer + // start = otile[0]*ztile[0] + otile[1]*ztile[1]*Naxis[0] + otile[2]*ztile[2]*Naxis[0]*Naxis[1] + ...; + // start = otile[0]*ztile[0] + Naxis[0]*(otile[1]*ztile[1] + Naxis[1]*(otile[2]*ztile[2] + ...)); + int start = otile[matrix->Naxes-1]*ztile[matrix->Naxes-1]; + for (i = matrix->Naxes - 2; i >= 0; i--) { + int coord = otile[i]*ztile[i]; + start = start*matrix->Naxis[i] + coord; + } + + int size = 0; + switch (raw_bitpix) { + case 8: size = 1; break; + case 16: size = 2; break; + case 32: size = 4; break; + case -32: size = 4; break; + case -64: size = 8; break; + default: abort(); + } + + // pixel offset in output array relative to tile start + int offset = 0; + + static int pass = 0; + for (k = 0; k < size; k++) { + for (i = 0; i < Nline; i++) { +# ifdef BYTE_SWAP + char *srcptr = &matrix->buffer[size*(offset + start) + (size - k - 1)]; +# else + char *srcptr = &matrix->buffer[size*(offset + start) + k]; +# endif + char *rawptr = &raw[i*Ztile[0] + k*Nraw]; + for (j = 0; j < Ztile[0]; j++, srcptr += size, rawptr ++) { + if (FALSE && (i == 0) && (j < 4) && (pass == 0)) { + fprintf (stderr, "0x%02hhx -> 0x%02hhx\n", (int)(srcptr - matrix->buffer), (int)(rawptr - raw)); + } + *rawptr = *srcptr; + // myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun"); + // myAssert (rawptr - raw < Nraw*size, "memory overrun"); } - break; - - case 4: - for (i = 0; i < 4*Nzdata; i+=4) { - DOSWAP (zdata[i+0], zdata[i+3]); - DOSWAP (zdata[i+1], zdata[i+2]); + + // update the counters, carrying to the next dimension if needed + for (j = 1; j < matrix->Naxes; j++) { + counter[j] ++; + if (counter[j] == Ztile[j]) { + counter[j] = 0; + } else { + break; + } } - break; - - case 8: - for (i = 0; i < 8*Nzdata; i+=8) { - DOSWAP (zdata[i+0], zdata[i+7]); - DOSWAP (zdata[i+1], zdata[i+6]); - DOSWAP (zdata[i+2], zdata[i+5]); - DOSWAP (zdata[i+3], zdata[i+4]); - } - break; - } -# endif + if (j == matrix->Naxes) assert (i == Nline - 1); // we should be done here... + + // Naxes = 3 + // offset = counter[1]*matrix->Naxis[0] + counter[2]*matrix->Naxis[0]*matrix->Naxis[1] + + // offset = matrix->Naxis[0]*(counter[1] + matrix->Naxis[1]*(counter[2] + matrix->Naxis[2]*...)) + + // determine the offset of the next line relative to the start position + offset = counter[matrix->Naxes - 1]; + for (j = matrix->Naxes - 2; j >= 0; j--) { + offset = offset*matrix->Naxis[j] + counter[j]; + } + } + } + pass ++; + + free (counter); + free (Ztile); + return (TRUE); } - -int gfits_extension_is_compressed (Header *header) { - - int has_extension, has_extname, has_zimage, zimage; - char extname[80], extension[80]; - - has_extension = gfits_scan (header, "XTENSION", "%s", 1, extension); - has_extname = gfits_scan (header, "EXTNAME", "%s", 1, extname); - has_zimage = gfits_scan_alt (header, "ZIMAGE", "%t", 1, &zimage); - - if (has_extension && !strcmp (extension, "IMAGE")) return (FALSE); - if (has_zimage && zimage) return (TRUE); - if (has_extname) { - if (!strcmp (extname, "COMPRESSED_IMAGE")) return (TRUE); - } - - return (FALSE); -} - -int gfits_compressed_is_primary (Header *header) { - - int zimage = FALSE; - // int ztension = FALSE; - - int has_zimage = gfits_scan_alt (header, "ZIMAGE", "%t", 1, &zimage); - // int has_ztension = gfits_scan_alt (header, "ZTENSION", "%t", 1, &ztension); - - if (has_zimage && zimage) return (TRUE); - - return (FALSE); -} - -int gfits_uncompressed_data_pixsize (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) { - - int i, Nbyte; - - if (!strcasecmp(cmptype, "GZIP_1")) { - if (out_bitpix == 8) return (1); - if (out_bitpix == 16) return (2); - if (out_bitpix == 32) return (4); - if (out_bitpix == -32) return (4); - if (out_bitpix == -64) return (8); - return (1); - } - if (!strcasecmp(cmptype, "RICE_1")) { - - // if BYTEPIX option is specified, use that for Nbyte - for (i = 0; i < Noptions; i++) { - if (!strcmp(optname[i], "BYTEPIX")) { - Nbyte = atoi (optvalue[i]); - return (Nbyte); - } - } - - return (4); - } - if (!strcasecmp(cmptype, "PLIO_1")) { - return (4); - } - if (!strcasecmp(cmptype, "HCOMPRESS_1")) { - if (out_bitpix == 8) return (4); - if (out_bitpix == 16) return (4); - return (8); - } - return (0); -} - -int gfits_vartable_heap_pixsize (char format) { - - if (format == 'B') { - return (1); - } - if (format == 'I') { - return (2); - } - if (format == 'J') { - return (4); - } - fprintf (stderr, "invalid size for compressed data: %c\n", format); - abort (); -} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_data.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_data.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_data.c (revision 38447) @@ -0,0 +1,159 @@ +# include +# include +# include + +/* functions defined in ricecomp.c */ +int fits_rcomp(int a[], int nx, unsigned char *c, int clen, int nblock); +int fits_rcomp_short(short a[], int nx, unsigned char *c, int clen, int nblock); +int fits_rcomp_byte(char a[], int nx, unsigned char *c, int clen, int nblock); +int fits_rdecomp (unsigned char *c, int clen, unsigned int array[], int nx, int nblock); +int fits_rdecomp_short (unsigned char *c, int clen, unsigned short array[], int nx, int nblock); +int fits_rdecomp_byte (unsigned char *c, int clen, unsigned char array[], int nx, int nblock); + +/* functions defined in fits_hcompress.c */ +# define LONGLONG long long +int fits_hcompress(int *a, int ny, int nx, int scale, char *output, long *nbytes, int *status); +int fits_hcompress64(LONGLONG *a, int ny, int nx, int scale, char *output, long *nbytes, int *status); + +/* functions defined in fits_hdeccompress.c */ +int fits_hdecompress(unsigned char *input, int smooth, int *a, int *ny, int *nx, int *scale, int *status); +int fits_hdecompress64(unsigned char *input, int smooth, LONGLONG *a, int *ny, int *nx, int *scale, int *status); + +/* functions defined in pliocomp.c */ +int pl_p2li (int *pxsrc, int xs, short *lldst, int npix); +int pl_l2pi (short *ll_src, int xs, int *px_dst, int npix); + +/* functions defined in gzip.c */ +int gfits_gz_stripheader (unsigned char *data, int *ndata); +int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); +int gfits_compress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); + +# define ESCAPE(RET) { fprintf (stderr, "gzip error in %s @ %s:%d\n", __func__, __FILE__, __LINE__); return (RET); } + +// XXX I'm putting Nx and Ny on the argument list -- only used by hcompress +// with a little work, these could go on the option list +int gfits_compress_data (char *zdata, int *Nzdata, char *cmptype, + char **optname, char **optvalue, int Nopt, + char *rawdata, int Nrawpix, int rawpix_size, + int Nx, int Ny) { + + int Nout; + int status; + + // do not actually compress : this is used for testing + if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1") || !strcasecmp(cmptype, "NONE_2")) { + unsigned long Nbytes = Nrawpix * rawpix_size; + + memcpy (zdata, rawdata, Nbytes); + *Nzdata = Nbytes; + return (TRUE); + } + + // GZIP_1 uses inflate / deflate, GZIP_2 does as well, but operates on a buffer with + // a different byte organization (gfits_distribute_gzp2) + if (!strcasecmp(cmptype, "GZIP_1") || !strcasecmp(cmptype, "GZIP_2")) { + unsigned long Nbytes = Nrawpix * rawpix_size; + + // the data must be byteswapped before compression begins + // if (!gfits_byteswap_zdata (rawdata, Nrawpix, rawpix_size)) return (FALSE); + + // Nzdata is the size of the allocated buffer before the function is called, returns the actual size + // Nbytes is the number of bytes in the raw data buffer + uLongf destLen = *Nzdata; + status = gfits_compress ((Bytef *) zdata, &destLen, (Bytef *) rawdata, Nbytes); + *Nzdata = destLen; + if (status != Z_OK) ESCAPE(FALSE); + return (TRUE); + } + + if (!strcasecmp(cmptype, "RICE_1") || !strcasecmp(cmptype, "RICE_ONE")) { + int i, blocksize; + // look for the BLOCKSIZE + blocksize = 32; + for (i = 0; i < Nopt; i++) { + if (!strcmp(optname[i], "BLOCKSIZE")) { + blocksize = atoi (optvalue[i]); + if ((blocksize != 16) && (blocksize != 32)) { + fprintf (stderr, "RICE blocksize is not valid: %d (%s = %s)\n", blocksize, optname[i], optvalue[i]); + return (FALSE); + } + } + } + + // Ninsum += Nzdata; + // Noutsum += *Nout; + // fprintf (stderr, "%d comp bytes; %d uncomp 'pixels', totals: %d %d\n", Nzdata, Npix, Ninsum, Noutsum); + + int k; + + switch (rawpix_size) { + case 4: + // rice compression from the CFITSIO source tree + Nout = fits_rcomp ((int *) rawdata, Nrawpix, (unsigned char *) zdata, *Nzdata, blocksize); + break; + + case 2: + // Nout is the number of bytes in the compressed buffer + Nout = fits_rcomp_short ((short *) rawdata, Nrawpix, (unsigned char *) zdata, *Nzdata, blocksize); + + if (0) { + fprintf (stderr, "Nout: %d, Nrawpix: %d\n", Nout, Nrawpix); + fprintf (stderr, "cmp inp: "); + for (k = 0; k < Nout; k++) { fprintf (stderr, "%02hhx", zdata[k]); if (k % 2) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + break; + + case 1: + Nout = fits_rcomp_byte ((char *) rawdata, Nrawpix, (unsigned char *) zdata, *Nzdata, blocksize); + break; + + default: + fprintf (stderr, "invalid output pixel size %d\n", rawpix_size); + *Nzdata = 0; + return (FALSE); + } + if (Nout < 0) { + fprintf (stderr, "error in rice decompression\n"); + *Nzdata = 0; + return (FALSE); + } + *Nzdata = Nout; + return TRUE; + } + + if (!strcasecmp(cmptype, "PLIO_1")) { + // note the fortan starting point: zdata is decremented at start + Nout = pl_p2li ((int *) rawdata, 1, (short *) zdata, Nrawpix); + *Nzdata = Nout; + if (!Nout) return (FALSE); + return (TRUE); + } + + if (!strcasecmp(cmptype, "HCOMPRESS_1")) { + long Nbytes = Nrawpix * rawpix_size; + int scale = 0; + status = 0; + // call hdecompress without smoothing + + if (rawpix_size == 4) { + fits_hcompress ((int *) rawdata, Nx, Ny, scale, (char *) zdata, &Nbytes, &status); + } + if (rawpix_size == 8) { + fits_hcompress64 ((long long *) rawdata, Nx, Ny, scale, (char *) zdata, &Nbytes, &status); + } + + if (status) { + fprintf (stderr, "error in hdecompress\n"); + return (FALSE); + } + // fprintf (stderr, "decompression yields image %d x %d (scale: %d)\n", Nx, Ny, scale); + + *Nzdata = Nbytes / rawpix_size; + return (TRUE); + } + + fprintf (stderr, "unknown compression %s\n", cmptype); + return (FALSE); +} + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_utils.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_utils.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_compress_utils.c (revision 38447) @@ -0,0 +1,344 @@ +# include +# include +# include + +// gfits_varlength returns a pointer to a chunk of Nzdata of data elements starting at zdata. + +// XXX need to have an API to send the user data + +// XXX need to byte-swap the table column; this needs to be worked out more clearly +// in the APIs: do the table read / column extract functions swap or not?. we have +// put the byte-swapping in gifts_varlength_column_pointer, but this is fairly weak. + +// XXX inconsistency between data element size between compressed data, uncompressed data, +// and output pixel data..... + +// need a structure to describe tiles, current tile? + +typedef struct { + int *ztile; // max size of a tile (edge tiles may be smaller) + int *otile; // counter for current tile (eg, for (2,3,0) otile[0] = 2, otile[1] = 3, otile[2] = 0) + int *ntile; // number of tiles in dimension [i] +} ImageTile; + +// advance the otile counter. +int gfits_imtile_next (Matrix *matrix, int *ztile, int *ntile, int *otile) { + + int i; + + // update the tile counters, carrying to the next dimension if needed + for (i = 0; i < matrix->Naxes; i++) { + otile[i] ++; + if (otile[i] == ntile[i]) { + if (i == matrix->Naxes - 1) return FALSE; + otile[i] = 0; + } else { + return TRUE; + } + } + return TRUE; +} + +// how many tiles needed for this image (given ztile[]) +int gfits_imtile_start (Matrix *matrix, int *otile) { + + int i; + + // update the tile counters, carrying to the next dimension if needed + for (i = 0; i < matrix->Naxes; i++) { + otile[i] = 0; + } + return TRUE; +} + +// how many tiles needed for this image (given ztile[]) +int gfits_imtile_count (Matrix *matrix, int *ztile, int *ntile) { + + int i; + + int Ntiles = 1; + for (i = 0; i < matrix->Naxes; i++) { + ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]); + Ntiles *= ntile[i]; + } + + return (Ntiles); +} + +// how many tiles needed for this image (given ztile[]) +int gfits_imtile_maxsize (Matrix *matrix, int *ztile) { + + int i; + + int max_tile_size = 1; + for (i = 0; i < matrix->Naxes; i++) { + max_tile_size *= ztile[i]; + } + + return (max_tile_size); +} + +// true sizes of this tile (in pixels) +off_t gfits_tile_size (Matrix *matrix, int *otile, int *ztile) { + + off_t i, Npixels, Ndimen; + + Npixels = 1; + for (i = 0; i < matrix->Naxes; i++) { + Ndimen = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]); + Npixels *= Ndimen; + } + + return (Npixels); +} + +int gfits_byteswap_zdata (char *zdata, int Nzdata, int pixsize) { + +# define DOSWAP(A,B) { char tmp = A; A = B; B = tmp; } + +# ifdef BYTE_SWAP + + int i; + + // fprintf (stderr, "swapping %d bytes in pix of size %d bytes...\n", Nzdata, pixsize); + switch (pixsize) { + case 1: + break; + + case 2: + for (i = 0; i < Nzdata; i+=2) { + DOSWAP (zdata[i+0], zdata[i+1]); + } + break; + + case 4: + for (i = 0; i < Nzdata; i+=4) { + DOSWAP (zdata[i+0], zdata[i+3]); + DOSWAP (zdata[i+1], zdata[i+2]); + } + break; + + case 8: + for (i = 0; i < Nzdata; i+=8) { + DOSWAP (zdata[i+0], zdata[i+7]); + DOSWAP (zdata[i+1], zdata[i+6]); + DOSWAP (zdata[i+2], zdata[i+5]); + DOSWAP (zdata[i+3], zdata[i+4]); + } + break; + } +# endif + return (TRUE); +} + +int gfits_extension_is_compressed_image (Header *header) { + + int has_extension, has_extname, has_zimage, zimage; + char extname[80], extension[80]; + + has_extension = gfits_scan (header, "XTENSION", "%s", 1, extension); + has_extname = gfits_scan (header, "EXTNAME", "%s", 1, extname); + has_zimage = gfits_scan_alt (header, "ZIMAGE", "%t", 1, &zimage); + + if (has_extension && !strcmp (extension, "IMAGE")) return (FALSE); + if (has_zimage && zimage) return (TRUE); + if (has_extname) { + if (!strcmp (extname, "COMPRESSED_IMAGE")) return (TRUE); + } + + return (FALSE); +} + +int gfits_extension_is_compressed_table (Header *header) { + + int has_ztable, ztable; + has_ztable = gfits_scan_alt (header, "ZTABLE", "%t", 1, &ztable); + if (has_ztable && ztable) return (TRUE); + return (FALSE); +} + +int gfits_compressed_is_primary (Header *header) { + + int zimage = FALSE; + // int ztension = FALSE; + + int has_zimage = gfits_scan_alt (header, "ZIMAGE", "%t", 1, &zimage); + // int has_ztension = gfits_scan_alt (header, "ZTENSION", "%t", 1, &ztension); + + if (has_zimage && zimage) return (TRUE); + + return (FALSE); +} + +int gfits_cmptype_valid (char *cmptype) { + + if (!strcasecmp(cmptype, "NONE")) return TRUE; + if (!strcasecmp(cmptype, "NONE_1")) return TRUE; // do not compress, but shuffle a la GZIP_1 + if (!strcasecmp(cmptype, "NONE_2")) return TRUE; // do not compress, but shuffle a la GZIP_2 + if (!strcasecmp(cmptype, "GZIP_1")) return TRUE; + if (!strcasecmp(cmptype, "GZIP_2")) return TRUE; + if (!strcasecmp(cmptype, "PLIO_1")) return TRUE; + if (!strcasecmp(cmptype, "RICE_1")) return TRUE; + if (!strcasecmp(cmptype, "RICE_ONE")) return TRUE; + if (!strcasecmp(cmptype, "HCOMPRESS_1")) return TRUE; + return FALSE; +} + +int gfits_uncompressed_data_pixsize (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) { + + int i, Nbyte; + + // GZIP_1, GZIP_2 use matched input & output pixels + if (!strcasecmp(cmptype, "GZIP_1") || + !strcasecmp(cmptype, "GZIP_2") || + !strcasecmp(cmptype, "NONE_2") || + !strcasecmp(cmptype, "NONE_1") || + !strcasecmp(cmptype, "NONE")) { + if (out_bitpix == 8) return (1); + if (out_bitpix == 16) return (2); + if (out_bitpix == 32) return (4); + if (out_bitpix == -32) return (4); + if (out_bitpix == -64) return (8); + return (1); + } + + // RICE_1 must operate on integer pixels + if (!strcasecmp(cmptype, "RICE_1") || + !strcasecmp(cmptype, "RICE_ONE")) { + // if BYTEPIX option is specified, use that for Nbyte + for (i = 0; i < Noptions; i++) { + if (!strcmp(optname[i], "BYTEPIX")) { + Nbyte = atoi (optvalue[i]); + return (Nbyte); + } + } + if (out_bitpix == 8) return (1); + if (out_bitpix == 16) return (2); + if (out_bitpix == 32) return (4); + return (4); + } + + // PLIO_1 always results in 4-byte ints (not 2 byte ints?) + if (!strcasecmp(cmptype, "PLIO_1")) { + return (4); + } + + if (!strcasecmp(cmptype, "HCOMPRESS_1")) { + if (out_bitpix == 8) return (4); + if (out_bitpix == 16) return (4); + return (8); + } + return (0); +} + +int gfits_compressed_data_pixsize (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) { + + int i, Nbyte; + + // NONE should be swapped to mimic swapping types + if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1")) { + if (out_bitpix == 8) return (1); + if (out_bitpix == 16) return (2); + if (out_bitpix == 32) return (4); + if (out_bitpix == -32) return (4); + if (out_bitpix == -64) return (8); + return (1); + } + + // GZIP_1, GZIP_2 should not be swapped after compression + if (!strcasecmp(cmptype, "GZIP_1") || + !strcasecmp(cmptype, "GZIP_2") || + !strcasecmp(cmptype, "NONE_2")) { + if (out_bitpix == 8) return (1); + if (out_bitpix == 16) return (1); + if (out_bitpix == 32) return (1); + if (out_bitpix == -32) return (1); + if (out_bitpix == -64) return (1); + return (1); + } + + // RICE_1 must operate on integer pixels + if (!strcasecmp(cmptype, "RICE_1") || + !strcasecmp(cmptype, "RICE_ONE")) { + // if BYTEPIX option is specified, use that for Nbyte + for (i = 0; i < Noptions; i++) { + if (!strcmp(optname[i], "BYTEPIX")) { + Nbyte = atoi (optvalue[i]); + return (Nbyte); + } + } + if (out_bitpix == 8) return (1); + if (out_bitpix == 16) return (2); + if (out_bitpix == 32) return (4); + return (4); + } + + // PLIO_1 always results in 4-byte ints (not 2 byte ints?) + if (!strcasecmp(cmptype, "PLIO_1")) { + return (4); + } + + if (!strcasecmp(cmptype, "HCOMPRESS_1")) { + if (out_bitpix == 8) return (4); + if (out_bitpix == 16) return (4); + return (8); + } + return (0); +} + +int gfits_uncompressed_data_bitpix (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) { + + int i, Nbyte; + + // GZIP_1, GZIP_2 use matched input & output pixels + if (!strcasecmp(cmptype, "GZIP_1") || + !strcasecmp(cmptype, "GZIP_2") || + !strcasecmp(cmptype, "NONE_1") || + !strcasecmp(cmptype, "NONE_2") || + !strcasecmp(cmptype, "NONE")) { + return (out_bitpix); + } + + // RICE_1 must operate on integer pixels + if (!strcasecmp(cmptype, "RICE_1") || + !strcasecmp(cmptype, "RICE_ONE")) { + // if BYTEPIX option is specified, use that for Nbyte + for (i = 0; i < Noptions; i++) { + if (!strcmp(optname[i], "BYTEPIX")) { + Nbyte = atoi (optvalue[i]); + return (Nbyte*8); + } + } + if (out_bitpix == 8) return (out_bitpix); + if (out_bitpix == 16) return (out_bitpix); + if (out_bitpix == 32) return (out_bitpix); + return (32); + } + + // PLIO_1 always results in 4-byte ints (not 2 byte ints?) + if (!strcasecmp(cmptype, "PLIO_1")) { + return (32); + } + + if (!strcasecmp(cmptype, "HCOMPRESS_1")) { + if (out_bitpix == 8) return (32); + if (out_bitpix == 16) return (32); + return (64); + } + return (0); +} + +int gfits_vartable_heap_pixsize (char format) { + + if (format == 'B') { + return (1); + } + if (format == 'I') { + return (2); + } + if (format == 'J') { + return (4); + } + fprintf (stderr, "invalid size for compressed data: %c\n", format); + abort (); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_matrix.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_matrix.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_matrix.c (revision 38447) @@ -2,29 +2,5 @@ # include -off_t gfits_data_size (Header *header) { - - int i; - off_t Nrec, size; - - if (header[0].Naxes == 0) return (0); - - size = abs(header[0].bitpix / 8); - - for (i = 0; i < header[0].Naxes; i++) - size *= header[0].Naxis[i]; - - // XXX do I multiply this times gcount? - size += header[0].pcount; - - /* round up to next complete block */ - if (size % FT_RECORD_SIZE) { - Nrec = 1 + (int) (size / FT_RECORD_SIZE); - size = FT_RECORD_SIZE * Nrec; - } - - return (size); -} - -off_t gfits_data_min_size (Header *header) { +off_t gfits_heap_start (Header *header) { int i; @@ -38,6 +14,13 @@ size *= header[0].Naxis[i]; - // XXX do I multiply this times gcount? + return (size); +} + +off_t gfits_data_min_size (Header *header) { + + off_t size = gfits_heap_start (header); size += header[0].pcount; + + // XXX what do I do with gcount? return (size); @@ -46,11 +29,9 @@ off_t gfits_data_pad_size (off_t rawsize) { - off_t Nrec, size; - - size = rawsize; + off_t size = rawsize; /* round up to next complete block */ if (rawsize % FT_RECORD_SIZE) { - Nrec = 1 + (int) (rawsize / FT_RECORD_SIZE); + off_t Nrec = 1 + (int) (rawsize / FT_RECORD_SIZE); size = FT_RECORD_SIZE * Nrec; } @@ -58,2 +39,10 @@ return (size); } + +off_t gfits_data_size (Header *header) { + off_t rawsize = gfits_data_min_size(header); + off_t size = gfits_data_pad_size (rawsize); + + return (size); +} + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_swap_raw.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_swap_raw.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_swap_raw.c (revision 38447) @@ -0,0 +1,46 @@ +# include +# include +# define DOSWAP(A,B) { char tmp = A; A = B; B = tmp; } + +/*********************** fits convert format ***********************************/ +/* this function is safe in the number of axes (can even be 0) */ +int gfits_swap_raw (Matrix *matrix) { + + off_t i; + + int bytes_per_pixel = (abs(matrix[0].bitpix) / 8); + + off_t Npixels = gfits_npix_matrix (matrix); + + char *byte = matrix[0].buffer; + + switch (bytes_per_pixel) { + case 1: + break; + case 2: + for (i = 0; i < Npixels; i++) { + DOSWAP(byte[0], byte[1]); + byte += 2; + } + break; + case 4: + for (i = 0; i < Npixels; i++) { + DOSWAP(byte[0], byte[3]); + DOSWAP(byte[1], byte[2]); + byte += 4; + } + break; + case 8: + for (i = 0; i < Npixels; i++) { + DOSWAP(byte[0], byte[7]); + DOSWAP(byte[1], byte[6]); + DOSWAP(byte[2], byte[5]); + DOSWAP(byte[3], byte[4]); + byte += 8; + } + break; + default: + myAbort ("invalid bytes_per_pixel"); + } + return TRUE; +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_M.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_M.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_M.c (revision 38447) @@ -0,0 +1,603 @@ +# include +# include +# include +# define VERBOSE_DUMP 0 + +int gfits_distribute_gzp2 (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero); + +# define ESCAPE { \ + fprintf (stderr, "error in %s @ line %d\n", __func__, __LINE__); \ + if (ztile != NULL) free (ztile); \ + if (optname != NULL) { \ + for (j = 0; j < Noptions; j++) { \ + free (optname[j]); \ + free (optvalue[j]); \ + } \ + free (optname); \ + free (optvalue); \ + } \ + if (out != NULL) free (out); \ + if (otile != NULL) free (otile); \ + if (ntile != NULL) free (ntile); \ + return (FALSE); } + +# define MOD_KEYWORD(ZNAME,NAME,TYPE,IN,OUT) { \ + if (gfits_scan (header, ZNAME, TYPE, 1, IN)) { \ + gfits_modify (header, NAME, TYPE, 1, OUT); \ + } \ + gfits_delete (header, ZNAME, 1); } + +# define MOD_KEYWORD_ALT(ZNAME,NAME,TYPE,IN,OUT) { \ + if (gfits_scan_alt (header, ZNAME, TYPE, 1, IN)) { \ + gfits_modify_alt (header, NAME, TYPE, 1, OUT); \ + } \ + gfits_delete (header, ZNAME, 1); } + +# define MOD_KEYWORD_REQUIRED(ZNAME,NAME,TYPE,IN,OUT) { \ + if (!gfits_scan (header, ZNAME, TYPE, 1, IN)) ESCAPE; \ + gfits_delete (header, ZNAME, 1); \ + gfits_modify (header, NAME, TYPE, 1, OUT); } + +int gfits_uncompress_image (Header *header, Matrix *matrix, FTable *ftable) { + + off_t Nzdata, Nzrows, zcol; + int i, j, status, zimage, Nout, max_tile_size; + char cmptype[80]; + char zaxis[10], naxis[10], key[10], word[81], exttype[81], checksum[81], datasum[81]; + int Noptions, NOPTIONS, zblank, oblank; + VarLengthColumn zdef; + float zscale, zzero; + + int *ztile = NULL; + int *otile = NULL; + int *ntile = NULL; + char **optname = NULL; + char **optvalue = NULL; + char *out = NULL; + char *zdata = NULL; + + Noptions = 0; + + // is ZIMAGE present? + // NOTE target of %t must be int length + status = gfits_scan_alt (ftable->header, "ZIMAGE", "%t", 1, &zimage); + if (!status || !zimage) ESCAPE; + + // copy original header to output header + gfits_copy_header (ftable->header, header); + + // delete ZIMAGE from output header + gfits_delete (header, "ZIMAGE", 1); + + // extract compression-specific keywords, update header as needed. + if (!gfits_scan (header, "ZCMPTYPE", "%s", 1, cmptype)) ESCAPE; + gfits_delete (header, "ZCMPTYPE", 1); + + MOD_KEYWORD_REQUIRED ("ZBITPIX", "BITPIX", "%d", &header->bitpix, header->bitpix); + MOD_KEYWORD_REQUIRED ("ZNAXIS", "NAXIS", "%d", &header->Naxes, header->Naxes); + + for (i = 0; i < header->Naxes; i++) { + snprintf (zaxis, 10, "ZNAXIS%d", i + 1); + snprintf (naxis, 10, "NAXIS%d", i + 1); + MOD_KEYWORD_REQUIRED (zaxis, naxis, OFF_T_FMT, &header->Naxis[i], header->Naxis[i]); + } + + // set up the tile sizes : ztile is the default size of the tile in the Nth dimension + // the actual tile size may be smaller at the edge of a dimension. if the ZTILEn + // entries are not found, default to [Nx,1,1,...] + ALLOCATE (ztile, int, header->Naxes); + if (!gfits_scan (header, "ZTILE1", "%d", 1, &ztile[0])) { + ztile[0] = header->Naxis[0]; + for (i = 1; i < header->Naxes; i++) { + ztile[i] = 1; + } + } else { + gfits_delete (header, "ZTILE1", 1); + for (i = 1; i < header->Naxes; i++) { + snprintf (key, 10, "ZTILE%d", i + 1); + if (!gfits_scan (header, key, "%d", 1, &ztile[i])) ESCAPE; + gfits_delete (header, key, 1); + } + } + + // search for algorithm-specific keywords. these are used to control compression options + // note the difference between the keywords (1 indexed) and the variables (0 indexed) + NOPTIONS = 10; + ALLOCATE (optname, char *, NOPTIONS); + ALLOCATE (optvalue, char *, NOPTIONS); + for (Noptions = 0; TRUE; Noptions++) { + snprintf (key, 10, "ZNAME%d", Noptions + 1); + if (!gfits_scan (header, key, "%s", 1, word)) break; + gfits_delete (header, key, 1); + optname[Noptions] = strcreate (word); + + snprintf (key, 10, "ZVAL%d", Noptions + 1); + if (!gfits_scan (header, key, "%s", 1, word)) ESCAPE; + gfits_delete (header, key, 1); + optvalue[Noptions] = strcreate (word); + + if (Noptions == NOPTIONS - 1) { + NOPTIONS += 10; + REALLOCATE (optname, char *, NOPTIONS); + REALLOCATE (optvalue, char *, NOPTIONS); + } + } + + // XXX get ZMASKCMP + int zsimple, have_zsimple; + int have_ztension; + + // NOTE target of %t must be int length + have_zsimple = gfits_scan_alt (header, "ZSIMPLE", "%t", 1, &zsimple); + have_ztension = gfits_scan (header, "ZTENSION", "%s", 1, exttype); + + // this is a very bogus case: we cannot have both keywords + if (have_zsimple && have_ztension) ESCAPE; + + // if neither are present, we have an image that is not really following the standard: + // assume it is a PHU + if (!have_zsimple && !have_ztension) { + header->simple = TRUE; + gfits_extended_to_primary (header, header->simple, "Image data"); + + MOD_KEYWORD_ALT ("ZEXTEND", "EXTEND", "%t", &header->extend, header->extend); + MOD_KEYWORD_ALT ("ZBLOCKED", "BLOCKED", "%t", &header->extend, header->extend); + } + + // have_zsimple : image comes from a PHU + if (have_zsimple) { + header->simple = zsimple; + gfits_delete (header, "ZSIMPLE", 1); + gfits_extended_to_primary (header, header->simple, "Image data"); + + MOD_KEYWORD_ALT ("ZEXTEND", "EXTEND", "%t", &header->extend, header->extend); + MOD_KEYWORD_ALT ("ZBLOCKED", "BLOCKED", "%t", &header->extend, header->extend); + } + + // have_ztension : image comes from an extension + if (have_ztension) { + gfits_delete (header, "ZTENSION", 1); + gfits_modify_extended (header, exttype, "Image extension"); + + // we may have an uncompressed PCOUNT / GCOUNT value, otherwise set to 0,1 + if (gfits_scan (header, "ZPCOUNT", "%d", 1, &header->pcount)) { + gfits_delete (header, "ZPCOUNT", 1); + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + } else { + header->pcount = 0; + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + } + if (gfits_scan (header, "ZGCOUNT", "%d", 1, &header->gcount)) { + gfits_delete (header, "ZGCOUNT", 1); + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } else { + header->pcount = 1; + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } + } else { + header->pcount = 0; + header->gcount = 1; + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } + + MOD_KEYWORD ("ZHECKSUM", "CHECKSUM", "%s", checksum, checksum); + MOD_KEYWORD ("ZDATASUM", "DATASUM", "%s", datasum, datasum); + + zscale = 1; + gfits_scan (header, "ZSCALE", "%f", 1, &zscale); + + zblank = 32767; + gfits_scan (header, "ZBLANK", "%d", 1, &zblank); + + oblank = 32767; + gfits_scan (header, "BLANK", "%d", 1, &oblank); + + zzero = 0; + gfits_scan (header, "ZZERO", "%f", 1, &zzero); + + // find the COMPRESSED_DATA column (format should be 1PB, 1PI, 1PJ) + // is it required that this be the only column? + for (i = 1; TRUE; i++) { + snprintf (key, 10, "TTYPE%d", i); + if (!gfits_scan (ftable->header, key, "%s", 1, word)) ESCAPE; + if (!strcmp (word, "COMPRESSED_DATA")) break; + } + zcol = i; + + if (!gfits_varlength_column_define (ftable, &zdef, zcol)) ESCAPE; + gfits_delete (header, "TFIELDS", 1); + snprintf (key, 10, "TTYPE"OFF_T_FMT, zcol); + gfits_delete (header, key, 1); + snprintf (key, 10, "TFORM"OFF_T_FMT, zcol); + gfits_delete (header, key, 1); + + // create the output image + gfits_create_matrix (header, matrix); + + // counters for tile numbers + ALLOCATE (otile, int, matrix->Naxes); + ALLOCATE (ntile, int, matrix->Naxes); + max_tile_size = 1; + for (i = 0; i < matrix->Naxes; i++) { + otile[i] = 0; + ntile[i] = (matrix->Naxis[i] % ztile[i]) ? (matrix->Naxis[i] / ztile[i] + 1) : (matrix->Naxis[i] / ztile[i]); + max_tile_size *= ztile[i]; + + // ztile[i] is the default (or max) tile size in the i-th dimension + // ntile[i] is the number of tiles in the i-th dimension + // otile[i] is the current output tile counter in the i-th dimension + } + + // this takes place in three major steps: + // 1) read the table data : zdef.format tells the size of the varlength heap element + // 2) uncompress the data : pixel size depends on compression method + // 3) distribute data to the tiles : output pixel size depends on header.bitpix + + // heapdata -> zdata -> odata -> idata + + // heapdata.pixsize : zdef.format + // zdata.pixsize : depends on compression + // odata.pixsize : depends on compression + // idata.pixsize : header.bitpix + + // size of an element in the vartable heap section + // zdata_pixsize = gfits_vartable_heap_pixsize (zdef.format); XXX + + // size of a pixel in the output from the decompression routine + int cmp_pixsize = gfits_compressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions); + int raw_bitpix = gfits_uncompressed_data_bitpix (cmptype, header[0].bitpix, optname, optvalue, Noptions); + int raw_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions); + + // size of a pixel in the final image (not needed) + // int tile_pixsize = abs(header[0].bitpix) / 8; + // fprintf (stderr, "raw_pixsize: %d, cmp_pixsize: %d, tile_pixsize: %d, raw_bitpix: %d\n", raw_pixsize, cmp_pixsize, tile_pixsize, raw_bitpix); + + int Nout_alloc = raw_pixsize*max_tile_size; + ALLOCATE (out, char, Nout_alloc); + + off_t row; + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "unc tbl: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", ftable->buffer[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + // uncompress the data + Nzrows = ftable->header->Naxis[1]; + for (row = 0; row < Nzrows; row++) { + + // expected output size for this tile + Nout = raw_pixsize*gfits_tile_size (matrix, otile, ztile); + + zdata = gfits_varlength_column_pointer (ftable, &zdef, row, &Nzdata); + if (!zdata) ESCAPE; + + if (VERBOSE_DUMP && (row == 0)) { + int k; + fprintf (stderr, "unc dat: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", zdata[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + if (strcasecmp(cmptype, "NONE_2") && // strcasecmp(cmptype, "NONE") && + strcasecmp(cmptype, "GZIP_1") && + strcasecmp(cmptype, "GZIP_2") && + strcasecmp(cmptype, "RICE_1") && + strcasecmp(cmptype, "RICE_ONE")) { + // Nzdata is number of bytes + if (!gfits_byteswap_zdata (zdata, Nzdata, cmp_pixsize)) ESCAPE; + } + + if (VERBOSE_DUMP && (row == 0)) { + int k; + fprintf (stderr, "unc SWP: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", zdata[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + fprintf (stderr, "Nout : %d -> ", Nout); + } + + // gfits_uncompress_data uncompresses from zdata to the temporary output buffer which must be allocated + // XXX the tile must not be > 2GB + // Nout going in is number of bytes + if (!gfits_uncompress_data ((char *)zdata, Nzdata, cmptype, optname, optvalue, Noptions, out, &Nout, Nout_alloc, raw_pixsize)) ESCAPE; + + if (VERBOSE_DUMP && (row == 0)) { + int k; + fprintf (stderr, "%d\n", Nout); + fprintf (stderr, "unc swp: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", out[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + if (!strcasecmp(cmptype, "GZIP_1")) { + // Nout is number of pixels + if (!gfits_byteswap_zdata (out, Nout * raw_pixsize, raw_pixsize)) ESCAPE; + } + + if (VERBOSE_DUMP && (row == 0)) { + int k; + fprintf (stderr, "unc raw: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", out[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + // copy the uncompressed pixels into their correct locations + if (!strcasecmp(cmptype, "GZIP_2") || !strcasecmp(cmptype, "NONE_2")) { + if (!gfits_distribute_gzp2 (matrix, out, Nout, raw_bitpix, otile, oblank, ztile, zblank, zscale, zzero)) ESCAPE; + } else { + if (!gfits_distribute_data (matrix, out, Nout, raw_bitpix, otile, oblank, ztile, zblank, zscale, zzero)) ESCAPE; + } + + if (VERBOSE_DUMP && (row == 0)) { + int k; + fprintf (stderr, "unc mat: "); + for (k = 0; k < 64; k++) { fprintf (stderr, "%02hhx", matrix->buffer[k]); if (k % 4 == 3) fprintf (stderr, " "); } + fprintf (stderr, "\n"); + } + + // update the tile counters, carrying to the next dimension if needed + for (j = 0; j < matrix->Naxes; j++) { + otile[j] ++; + if (otile[j] == ntile[j]) { + otile[j] = 0; + } else { + break; + } + } + } + + FREE (ztile); + if (optname != NULL) { + for (j = 0; j < Noptions; j++) { + FREE (optname[j]); + FREE (optvalue[j]); + } + FREE (optname); + FREE (optvalue); + } + FREE (out); + FREE (otile); + FREE (ntile); + return (TRUE); +} + +// bitpix is the input data size/type +int gfits_distribute_data (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) { + + int i, j; + int *counter = NULL; + int *Ztile = NULL; + + ALLOCATE (counter, int, matrix->Naxes); + ALLOCATE (Ztile, int, matrix->Naxes); + + // counter for current row in tile to copy + // true sizes of this tile (in pixels) + for (i = 0; i < matrix->Naxes; i++) { + counter[i] = 0; + Ztile[i] = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]); + } + + // number of lines in the tile (in pixels) + int Nline = 1; + for (i = 1; i < matrix->Naxes; i++) { + Nline *= Ztile[i]; + } + + // double check reported size (needs the Isize) + assert (Nraw == Ztile[0]*Nline); + + // set the starting point of the tile: + // start = otile[0]*ztile[0] + otile[1]*ztile[1]*Naxis[0] + otile[2]*ztile[2]*Naxis[0]*Naxis[1] + ...; + // start = otile[0]*ztile[0] + Naxis[0]*(otile[1]*ztile[1] + Naxis[1]*(otile[2]*ztile[2] + ...)); + int start = otile[matrix->Naxes-1]*ztile[matrix->Naxes-1]; + for (i = matrix->Naxes - 2; i >= 0; i--) { + int coord = otile[i]*ztile[i]; + start = start*matrix->Naxis[i] + coord; + } + + // pixel offset in output array relative to tile start + int offset = 0; + + int directCopy = (zzero == 0.0) && (zscale == 1.0); + +# define SCALE_AND_DIST_INT_PRINT(TYPE, SIZE) { \ + TYPE *TILEptr = (TYPE *) &matrix->buffer[SIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (*TILEptr == zblank) { \ + *TILEptr = oblank; \ + } else { \ + *TILEptr = directCopy ? *RAWptr : *RAWptr * zscale + zzero; \ + fprintf (stderr, "dis: %08x : %08x\n", *TILEptr, *RAWptr); \ + } } } + + // we need to set up switches for all of the possible combinations: + // this macro is used at the inner switch to run the actual loop +# define SCALE_AND_DIST_INT(OTYPE, OSIZE) { \ + OTYPE *TILEptr = (OTYPE *) &matrix->buffer[OSIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (*TILEptr == zblank) { \ + *TILEptr = oblank; \ + } else { \ + if (directCopy) { \ + *TILEptr = *RAWptr; \ + } else { \ + *TILEptr = *RAWptr * zscale + zzero; \ + } } } } + + // we need to set up switches for all of the possible combinations: + // this macro is used at the inner switch to run the actual loop +# define SCALE_AND_DIST_FLOAT(OTYPE, OSIZE) { \ + OTYPE *TILEptr = (OTYPE *) &matrix->buffer[OSIZE*(offset + start)]; \ + for (j = 0; j < Ztile[0]; j++, TILEptr++, RAWptr++) { \ + if (!isfinite(*TILEptr)) { \ + *TILEptr = oblank; \ + } else { \ + if (directCopy) { \ + *TILEptr = *RAWptr; \ + } else { \ + *TILEptr = *RAWptr * zscale + zzero; \ + } } } } + + // this macro sets up the outer switch and calls above macro with all output bitpix options +# define SETUP_RAWSIZE(ITYPE, ISIZE) { \ + ITYPE *RAWptr = (ITYPE *) &raw[i*ISIZE*Ztile[0]]; \ + switch (matrix->bitpix) { \ + case 8: SCALE_AND_DIST_INT (char, 1); break; \ + case 16: SCALE_AND_DIST_INT (short, 2); break; \ + case 32: SCALE_AND_DIST_INT (int, 4); break; \ + case -32: SCALE_AND_DIST_FLOAT (float, 4); break; \ + case -64: SCALE_AND_DIST_FLOAT (double, 8); break; \ + default: abort(); \ + } } + + // this macro sets up the outer switch and calls above macro with all output bitpix options +# define SETUP_RAWSIZE_PRINT(TYPE, SIZE) { \ + TYPE *RAWptr = (TYPE *) &raw[i*SIZE*Ztile[0]]; \ + switch (matrix->bitpix) { \ + case 8: SCALE_AND_DIST_INT (char, 1); break; \ + case 16: SCALE_AND_DIST_INT (short, 2); break; \ + case 32: SCALE_AND_DIST_INT (int, 4); break; \ + case -32: SCALE_AND_DIST_FLOAT (float, 4); break; \ + case -64: SCALE_AND_DIST_FLOAT (double, 8); break; \ + default: abort(); \ + } } + + // loop over lines in the tile + for (i = 0; i < Nline; i++) { + switch (raw_bitpix) { + case 8: SETUP_RAWSIZE (char, 1); break; + case 16: SETUP_RAWSIZE (short, 2); break; + case 32: SETUP_RAWSIZE (int, 4); break; + case -32: SETUP_RAWSIZE (float, 4); break; + case -64: SETUP_RAWSIZE (double, 8); break; + default: abort(); + } + + // update the counters, carrying to the next dimension if needed + for (j = 1; j < matrix->Naxes; j++) { + counter[j] ++; + if (counter[j] == Ztile[j]) { + counter[j] = 0; + } else { + break; + } + } + if (j == matrix->Naxes) assert (i == Nline - 1); // we should be done here... + + // Naxes = 3 + // offset = counter[1]*matrix->Naxis[0] + counter[2]*matrix->Naxis[0]*matrix->Naxis[1] + + // offset = matrix->Naxis[0]*(counter[1] + matrix->Naxis[1]*(counter[2] + matrix->Naxis[2]*...)) + + // determine the offset of the next line relative to the start position + offset = counter[matrix->Naxes - 1]; + for (j = matrix->Naxes - 2; j >= 0; j--) { + offset = offset*matrix->Naxis[j] + counter[j]; + } + } + + free (counter); + free (Ztile); + return (TRUE); +} + +// bitpix is the input data size/type +int gfits_distribute_gzp2 (Matrix *matrix, char *raw, int Nraw, int raw_bitpix, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) { + + int i, j, k; + int *counter = NULL; + int *Ztile = NULL; + + ALLOCATE (counter, int, matrix->Naxes); + ALLOCATE (Ztile, int, matrix->Naxes); + + // counter for current row in tile to copy + // true sizes of this tile (in pixels) + for (i = 0; i < matrix->Naxes; i++) { + counter[i] = 0; + Ztile[i] = MIN ((matrix->Naxis[i] - otile[i]*ztile[i]), ztile[i]); + } + + // number of lines in the tile (in pixels) + int Nline = 1; + int Npix = matrix->Naxis[0]; + for (i = 1; i < matrix->Naxes; i++) { + Nline *= Ztile[i]; + Npix *= matrix->Naxis[i]; + } + + // double check reported size (needs the Isize) + assert (Nraw == Ztile[0]*Nline); + + // set the starting point of the tile: + // start = otile[0]*ztile[0] + otile[1]*ztile[1]*Naxis[0] + otile[2]*ztile[2]*Naxis[0]*Naxis[1] + ...; + // start = otile[0]*ztile[0] + Naxis[0]*(otile[1]*ztile[1] + Naxis[1]*(otile[2]*ztile[2] + ...)); + int start = otile[matrix->Naxes-1]*ztile[matrix->Naxes-1]; + for (i = matrix->Naxes - 2; i >= 0; i--) { + int coord = otile[i]*ztile[i]; + start = start*matrix->Naxis[i] + coord; + } + + int size = 0; + switch (raw_bitpix) { + case 8: size = 1; break; + case 16: size = 2; break; + case 32: size = 4; break; + case -32: size = 4; break; + case -64: size = 8; break; + default: abort(); + } + + // pixel offset in output array relative to tile start + int offset = 0; + + static int pass = 0; + for (k = 0; k < size; k++) { + for (i = 0; i < Nline; i++) { +# ifdef BYTE_SWAP + char *srcptr = &matrix->buffer[size*(offset + start) + (size - k - 1)]; +# else + char *srcptr = &matrix->buffer[size*(offset + start) + k]; +# endif + char *rawptr = &raw[i*Ztile[0] + k*Nraw]; + for (j = 0; j < Ztile[0]; j++, srcptr += size, rawptr ++) { + if (FALSE && (i == 0) && (j < 4) && (pass == 0)) { + fprintf (stderr, "0x%02hhx -> 0x%02hhx\n", (int)(srcptr - matrix->buffer), (int)(rawptr - raw)); + } + *srcptr = *rawptr; + // myAssert (srcptr - matrix->buffer < Npix*size, "memory overrun"); + // myAssert (rawptr - raw < Nraw*size, "memory overrun"); + } + + // update the counters, carrying to the next dimension if needed + for (j = 1; j < matrix->Naxes; j++) { + counter[j] ++; + if (counter[j] == Ztile[j]) { + counter[j] = 0; + } else { + break; + } + } + if (j == matrix->Naxes) assert (i == Nline - 1); // we should be done here... + + // Naxes = 3 + // offset = counter[1]*matrix->Naxis[0] + counter[2]*matrix->Naxis[0]*matrix->Naxis[1] + + // offset = matrix->Naxis[0]*(counter[1] + matrix->Naxis[1]*(counter[2] + matrix->Naxis[2]*...)) + + // determine the offset of the next line relative to the start position + offset = counter[matrix->Naxes - 1]; + for (j = matrix->Naxes - 2; j >= 0; j--) { + offset = offset*matrix->Naxis[j] + counter[j]; + } + } + } + pass ++; + + free (counter); + free (Ztile); + return (TRUE); +} + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_data.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_data.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/matrix/F_uncompress_data.c (revision 38447) @@ -26,31 +26,42 @@ int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); -int gfits_uncompress_data (char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize) { +# define ESCAPE(RET) { fprintf (stderr, "gzip error in %s @ %s:%d\n", __func__, __FILE__, __LINE__); return (RET); } + +int gfits_uncompress_data (char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int Nout_alloc, int out_pixsize) { int status; - static int Ninsum = 0; - static int Noutsum = 0; - if (!strcasecmp(cmptype, "GZIP_1")) { - unsigned long tNout = *Nout * out_pixsize; + // do not actually uncompress : this is used for testing + if (!strcasecmp(cmptype, "NONE") || !strcasecmp(cmptype, "NONE_1") || !strcasecmp(cmptype, "NONE_2")) { + memcpy (outdata, zdata, Nzdata); + myAssert (*Nout == Nzdata, "invalid size"); + *Nout = Nzdata / out_pixsize; + return (TRUE); + } + + if (!strcasecmp(cmptype, "GZIP_1") || !strcasecmp(cmptype, "GZIP_2")) { + // unsigned long tNout = *Nout * out_pixsize; + // input value of *Nout is number of BYTES + unsigned long tNout = Nout_alloc; // for GZIP data, I need to check for and remove the header on the first block: - gfits_gz_stripheader ((unsigned char *)zdata, &Nzdata); + // XXX maybe not anymore + // gfits_gz_stripheader ((unsigned char *)zdata, &Nzdata); // uncompress does not require us to know the expected number of pixel; it tells us the number + // XXX shouldn't we validate the result : we think we know the size status = gfits_uncompress ((Bytef *) outdata, &tNout, (Bytef *) zdata, Nzdata); - if (status != Z_OK) { - fprintf (stderr, "error in uncompress (GZIP)\n"); - return (FALSE); - } + if (status != Z_OK) ESCAPE(FALSE); + myAssert (*Nout == tNout, "uncompressed size mismatch"); *Nout = tNout / out_pixsize; + // output value of *Nout is number of PIXELS // the resulting uncompressed data is byteswapped - if (!gfits_byteswap_zdata (outdata, *Nout, out_pixsize)) return (FALSE); + // if (!gfits_byteswap_zdata (outdata, *Nout, out_pixsize)) return (FALSE); return (TRUE); } - if (!strcasecmp(cmptype, "RICE_1")) { + if (!strcasecmp(cmptype, "RICE_1") || !strcasecmp(cmptype, "RICE_ONE")) { int i, blocksize; // look for the BLOCKSIZE @@ -66,41 +77,36 @@ } - int Npix; - // Npix = *Nout * (out_pixsize / 4.0); - Npix = *Nout; - - Ninsum += Nzdata; - Noutsum += *Nout; - // fprintf (stderr, "%d comp bytes; %d uncomp 'pixels', totals: %d %d\n", Nzdata, Npix, Ninsum, Noutsum); + int status = FALSE; + int Npix = *Nout / out_pixsize; + int k; switch (out_pixsize) { case 4: // rice decompression from the CFITSIO source tree : we need to tell it the expected number of pixels - // is also REQUIRES 4byte output, which is fairly stupid. - if (fits_rdecomp ((unsigned char *) zdata, Nzdata, (unsigned int *) outdata, Npix, blocksize)) { - fprintf (stderr, "error in rice decompression\n"); - return (FALSE); + status = fits_rdecomp ((unsigned char *) zdata, Nzdata, (unsigned int *) outdata, Npix, blocksize); + break; + case 2: + if (0) { + fprintf (stderr, "Nout: %d, Nrawpix: %d\n", Nzdata, Npix); + fprintf (stderr, "cmp out: "); + for (k = 0; k < Nzdata; k++) { fprintf (stderr, "0x%02hhx ", zdata[k]); } + fprintf (stderr, "\n"); } - return (TRUE); - - case 2: - if (fits_rdecomp_short ((unsigned char *) zdata, Nzdata, (unsigned short *) outdata, Npix, blocksize)) { - fprintf (stderr, "error in rice decompression\n"); - return (FALSE); - } - return (TRUE); - + status = fits_rdecomp_short ((unsigned char *) zdata, Nzdata, (unsigned short *) outdata, Npix, blocksize); + break; case 1: - if (fits_rdecomp_byte ((unsigned char *) zdata, Nzdata, (unsigned char *) outdata, Npix, blocksize)) { - fprintf (stderr, "error in rice decompression\n"); - return (FALSE); - } - return (TRUE); - + status = fits_rdecomp_byte ((unsigned char *) zdata, Nzdata, (unsigned char *) outdata, Npix, blocksize); + break; default: fprintf (stderr, "invalid output pixel size %d\n", out_pixsize); - return (FALSE); + status = FALSE; + break; } - + if (status) { + *Nout = 0; + return FALSE; + } + *Nout = Npix; + return TRUE; } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_compress_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_compress_T.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_compress_T.c (revision 38447) @@ -0,0 +1,464 @@ +# include +# include +# include +# define VERBOSE_DUMP 0 + +// the user needs to specify the various compression options: +// ztilelen (may be 0) +// zcmptype + +// steps to convert an image to a compressed table: +// * determine the number of tiles (from ztile[] and image size) +// * construct an empty table with the right dimensions (1 column, Ntile rows) +// * loop over tiles +// * extract tile data to a buffer +// * compress the buffer data +// * insert in table heap +// * update header keywords + +// we do not try to support compression of an image with an associated table (not valid) + +# define ESCAPE { fprintf (stderr, "error in %s @ line %d\n", __func__, __LINE__); goto escape; } + +int gfits_collect_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows); +int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows); + +static float timeSum1 = 0.0; +static float timeSum2 = 0.0; +static float timeSum2a = 0.0; +static float timeSum2b = 0.0; +static float timeSum2c = 0.0; +static float timeSum2d = 0.0; +static float timeSum2e = 0.0; +static float timeSum3 = 0.0; + +void gfits_compress_timing () { + fprintf (stderr, "cmp times: %f %f %f\n", timeSum1, timeSum2, timeSum3); + fprintf (stderr, "cmp times: %f %f %f %f %f\n", timeSum2a, timeSum2b, timeSum2c, timeSum2d, timeSum2e); + + timeSum1 = 0.0; + timeSum2 = 0.0; + timeSum2a = 0.0; + timeSum2b = 0.0; + timeSum2c = 0.0; + timeSum2d = 0.0; + timeSum2e = 0.0; + timeSum3 = 0.0; +} + +int gfits_compress_table (FTable *srctable, FTable *tgttable, int ztilelen, char *zcmptype) { + + int i, j; + + char keyword[81]; + + char *raw = NULL; + char *zdata = NULL; + + Header *srcheader = srctable->header; + Header *tgtheader = tgttable->header; + + // XXX EAM: + struct timeval startTimer, stopTimer; + float dtime; + gettimeofday (&startTimer, (void *) NULL); + + int Ntile, ztilelast; + if (!ztilelen) ztilelen = srcheader->Naxis[1]; + + if (!srcheader->Naxis[1]) { + Ntile = 0; + ztilelast = 0; + } else { + if (srcheader->Naxis[1] % ztilelen) { + Ntile = srcheader->Naxis[1] / ztilelen + 1; + ztilelast = srcheader->Naxis[1] % ztilelen; + } else { + Ntile = srcheader->Naxis[1] / ztilelen; + ztilelast = ztilelen; + } + } + + // EXTNAME is not mandatory, but I need to keep it + char extname[80]; + if (!gfits_scan (srcheader, "EXTNAME", "%s", 1, extname)) { + strcpy (extname, "COMPRESSED_TABLE"); + } + + // creates an empty (Naxes = 2, Naxis[i] = 0) table header with XTENSION = BINTABLE, EXTNAME = COMPRESSED_TABLE + if (!gfits_create_table_header (tgtheader, "BINTABLE", extname)) ESCAPE; + + int Nfields; + if (!gfits_scan (srcheader, "TFIELDS", "%d", 1, &Nfields)) ESCAPE; + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum1 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + TableField *fields = NULL; + ALLOCATE_ZERO (fields, TableField, Nfields); + for (i = 0; i < Nfields; i++) { + snprintf (keyword, 80, "TTYPE%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].ttype)) ESCAPE; + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].ttype_cmt)) ESCAPE; + + // TUNIT is not mandatory + snprintf (keyword, 80, "TUNIT%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tunit)) { + fields[i].tunit[0] = 0; + } else { + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tunit_cmt)) ESCAPE; + } + + snprintf (keyword, 80, "TFORM%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tformat)) ESCAPE; + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tformat_cmt)) ESCAPE; + + // for now we set all fields to the requested type. since I do not yet know the column data types I cannot yet assign automatic cmptypes + strcpy (fields[i].zctype, zcmptype); + + // by using "P" format, we are limited to 32bit pointers (2GB heap) + if (!gfits_define_bintable_column (tgtheader, "1QB(0)", fields[i].ttype, fields[i].ttype_cmt, fields[i].tunit, 1.0, 0.0)) ESCAPE; + } + + // allocates the default data array (tgttable->buffer) + if (!gfits_create_table (tgtheader, tgttable)) ESCAPE; + + // copy original header to output header (XXX this needs to be finished) + if (!gfits_copy_keywords_compress (srcheader, tgtheader)) ESCAPE; + + // this allocates the pointer data array for the full set of tiles (NOT the heap) + char *tmpbuffer = NULL; + ALLOCATE_ZERO (tmpbuffer, char, 16*Ntile); // need space for Ntile entries, each of width 16 bytes (2 long) + for (i = 0; i < Nfields; i++) { + if (!gfits_set_bintable_column (tgtheader, tgttable, fields[i].ttype, tmpbuffer, Ntile)) ESCAPE; + } + free (tmpbuffer); + + // add ZIMAGE from output header + if (!gfits_modify_alt (tgtheader, "ZTABLE", "%t", 1, TRUE)) ESCAPE; + + // define compression-specific keywords, update header as needed. + if (!gfits_modify (tgtheader, "ZTILELEN", "%d", 1, ztilelen)) ESCAPE; + + if (!gfits_modify (tgtheader, "ZNAXIS1", OFF_T_FMT, 1, srcheader->Naxis[0])) ESCAPE; + if (!gfits_modify (tgtheader, "ZNAXIS2", OFF_T_FMT, 1, srcheader->Naxis[1])) ESCAPE; + + off_t pcount; + if (!gfits_scan (srcheader, "PCOUNT", OFF_T_FMT, 1, &pcount)) ESCAPE; + if (!gfits_modify (tgtheader, "ZPCOUNT", OFF_T_FMT, 1, pcount)) ESCAPE; + + off_t theap; + if (gfits_scan (srcheader, "THEAP", OFF_T_FMT, 1, &theap)) { + if (!gfits_modify (tgtheader, "ZTHEAP", OFF_T_FMT, 1, theap)) ESCAPE; + } + + int max_width = 0; + int offset = 0; // bytes from first column of first field to the current field (accumulate to set) + + for (i = 0; i < Nfields; i++) { + snprintf (keyword, 81, "ZFORM%d", i+1); + if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[i].tformat)) ESCAPE; + if (!gfits_varlength_column_define (tgttable, &fields[i].zdef, i+1)) ESCAPE; + if (!gfits_bintable_format (fields[i].tformat, fields[i].datatype, &fields[i].Nvalues, &fields[i].pixsize)) ESCAPE; // + fields[i].rowsize = fields[i].Nvalues*fields[i].pixsize; + max_width = MAX(max_width, fields[i].rowsize); + + if (!strcasecmp (fields[i].zctype, "AUTO")) { + if (!strcmp (fields[i].datatype, "short") || + !strcmp (fields[i].datatype, "float") || + !strcmp (fields[i].datatype, "double")|| + !strcmp (fields[i].datatype, "int64_t")) { + strcpy (fields[i].zctype, "GZIP_2"); + goto got_cmptype; + } + if (!strcmp (fields[i].datatype, "int")) { + strcpy (fields[i].zctype, "RICE_1"); + goto got_cmptype; + } + if (!strcmp (fields[i].datatype, "byte") || + !strcmp (fields[i].datatype, "char")) { + strcpy (fields[i].zctype, "GZIP_1"); + goto got_cmptype; + } + } + got_cmptype: + + + // OVERRIDE: RICE can only be used on integer fields + if (!strcasecmp (fields[i].zctype, "RICE_1") || !strcasecmp (fields[i].zctype, "RICE_ONE")) { + if (!strcmp (fields[i].datatype, "float") || !strcmp (fields[i].datatype, "double") || !strcmp (fields[i].datatype, "int64_t")) { + strcpy (fields[i].zctype, "GZIP_2"); + } + } + + // OVERRIDE: GZIP_2 invalid for B (use GZIP_1) + if (!strcasecmp (fields[i].zctype, "GZIP_2") && !strcmp (fields[i].datatype, "byte")) { + strcpy (fields[i].zctype, "GZIP_1"); + } + + // compression type per column (XXX for now we are just using zcmptype, as set above) + snprintf (keyword, 81, "ZCTYP%d", i+1); + if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[i].zctype)) ESCAPE; + + fields[i].offset = offset; + offset += fields[i].rowsize; + } + + // allocate the intermediate storage buffers + int Nzdata_alloc = max_width*ztilelen + 100; + ALLOCATE (raw, char, max_width*ztilelen); + ALLOCATE (zdata, char, Nzdata_alloc); + + gfits_dump_raw_table (srctable, "cmp"); + + // XXX TIMER 2 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + // compress the data : copy into a tile, compress the tile, then add to the output table + // each tile -> 1 row of the output table + for (i = 0; i < Ntile; i++) { + + for (j = 0; j < Nfields; j++) { + + gettimeofday (&startTimer, (void *) NULL); + + int Nrows = (i == Ntile - 1) ? ztilelast : ztilelen; + int row_start = i*ztilelen; + // ^- first row for this tile & field + // NOTE: assumes each tile has the same length, ex the last (true for now) + + // copy the raw pixels from their native matrix locations to the temporary output buffer + if (!strcasecmp(fields[j].zctype, "GZIP_2") || !strcasecmp(fields[j].zctype, "NONE_2")) { + if (!gfits_collect_table_gzp2 (srctable, &fields[j], raw, row_start, Nrows)) ESCAPE; + } else { + if (!gfits_collect_table_data (srctable, &fields[j], raw, row_start, Nrows)) ESCAPE; + } + + // XXX TIMER 2a + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2a += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "c1: "); + for (k = 0; k < Nrows*fields[j].Nvalues*fields[j].pixsize; k++) { + fprintf (stderr, "%02hhx", raw[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + // optname, optvalue = NULL, Noptions = 0 + + int Nraw = Nrows*fields[j].Nvalues; // number of pixels + if (!strcasecmp(fields[j].zctype, "GZIP_1")) { + if (!gfits_byteswap_zdata (raw, Nraw * fields[j].pixsize, fields[j].pixsize)) ESCAPE; + } + + // XXX TIMER 2b + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2b += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "c2: "); + for (k = 0; k < Nrows*fields[j].Nvalues*fields[j].pixsize; k++) { + fprintf (stderr, "%02hhx", raw[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression + if (!gfits_compress_data (zdata, &Nzdata, fields[j].zctype, NULL, NULL, 0, raw, Nraw, fields[j].pixsize, 0, 0)) ESCAPE; + + // XXX TIMER 2c + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2c += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "c3: "); + for (k = 0; k < Nzdata; k++) { + fprintf (stderr, "%02hhx", zdata[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + if (strcasecmp(fields[j].zctype, "NONE_2") && // NONE and NONE_1 not swapped? + strcasecmp(fields[j].zctype, "GZIP_1") && + strcasecmp(fields[j].zctype, "GZIP_2") && + strcasecmp(fields[j].zctype, "RICE_1") && + strcasecmp(fields[j].zctype, "RICE_ONE")) { + if (!gfits_byteswap_zdata (zdata, Nzdata, fields[j].pixsize)) ESCAPE; + } + + // XXX TIMER 2d + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2d += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "c4: "); + for (k = 0; k < Nzdata; k++) { + fprintf (stderr, "%02hhx", zdata[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + if (!gfits_varlength_column_add_data (tgttable, zdata, Nzdata, i, &fields[j].zdef)) ESCAPE; + // XXX TIMER 2e + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2e += dtime; + gettimeofday (&startTimer, (void *) NULL); + + } + } + + for (i = 0; i < Nfields; i++) { + if (!gfits_varlength_column_finish (tgttable, &fields[i].zdef)) ESCAPE; + } + + // XXX TIMER 3 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum3 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + gfits_dump_cmp_table (tgttable, "cmp"); + + free (raw); + free (zdata); + return (TRUE); + +escape: + FREE (raw); + FREE (zdata); + return FALSE; +} + +# define VERBOSE 0 + +// raw_pixsize is the bytes / pixel for the input matrix +// place the raw image bytes for the current tile into the tile buffer +int gfits_collect_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + + char *tblbuffer = &table->buffer[Nx*row_start + field->offset]; + + int rowsize = field->rowsize; + + if (VERBOSE) fprintf (stderr, "distribute: "); + for (i = 0; i < Nrows; i++, tblbuffer += Nx) { + memcpy (&raw[i*rowsize], tblbuffer, rowsize); +# if (VERBOSE) + int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]); +# endif + } + if (VERBOSE) fprintf (stderr, "\n"); + + return (TRUE); +} + +int gfits_collect_table_data_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i; + + // we are copying NN rows of the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + + if (VERBOSE) fprintf (stderr, "collect: "); + for (i = 0; i < Nrows; i++) { + int row = row_start + i; + memcpy (&raw[i*field->rowsize], &table->buffer[Nx*row + field->offset], field->rowsize); +# if (VERBOSE) + int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]); +# endif + } + if (VERBOSE) fprintf (stderr, "\n"); + + return (TRUE); +} + +int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i, j, k; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + int Nvalues = field->Nvalues; + int pixsize = field->pixsize; + int offset = field->offset; + + char *rawptr = raw; + + for (k = 0; k < field->pixsize; k++) { +# ifdef BYTE_SWAP + char *tblptr_start = &table->buffer[Nx*row_start + offset + (pixsize - k - 1)]; +# else + char *tblptr_start = &table->buffer[Nx*row_start + offset + k]; +# endif + for (i = 0; i < Nrows; i++, tblptr_start += Nx) { + char *tblptr = tblptr_start; + for (j = 0; j < Nvalues; j++, tblptr += pixsize, rawptr++) { + *rawptr = *tblptr; + } + } + } + return (TRUE); +} + +int gfits_collect_table_gzp2_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i, j, k; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + + for (k = 0; k < field->pixsize; k++) { + for (i = 0; i < Nrows; i++) { + int row = row_start + i; + char *rawptr = &raw[i*field->Nvalues + k*Nrows*field->Nvalues]; +# ifdef BYTE_SWAP + char *tblptr = &table->buffer[Nx*row + field->offset + (field->pixsize - k - 1)]; +# else + char *tblptr = &table->buffer[Nx*row + field->offset + k]; +# endif + for (j = 0; j < field->Nvalues; j++, tblptr += field->pixsize, rawptr++) { + *rawptr = *tblptr; + myAssert (rawptr - raw < Nrows*field->Nvalues*field->pixsize, "oops"); + myAssert (tblptr - table->buffer < table->header->Naxis[0]*table->header->Naxis[1], "oops"); + } + } + } + return (TRUE); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_copy_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_copy_T.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_copy_T.c (revision 38447) @@ -6,9 +6,25 @@ /* find buffer size */ - out[0].validsize = in[0].validsize; - out[0].datasize = in[0].datasize; + out[0].validsize = in[0].validsize; + out[0].datasize = in[0].datasize; + out[0].heap_start = in[0].heap_start; ALLOCATE (out[0].buffer, char, out[0].datasize); memcpy (out[0].buffer, in[0].buffer, out[0].datasize); + return (TRUE); +} + +/*********************** copy ftable pointers (does not copy memory) ***********************************/ +int gfits_copy_ftable_ptr (FTable *in, FTable *out) { + + if (!in) return FALSE; + if (!out) return FALSE; + + /* find buffer size */ + out[0].validsize = in[0].validsize; + out[0].datasize = in[0].datasize; + out[0].heap_start = in[0].heap_start; + out[0].header = in[0].header; + out[0].buffer = in[0].buffer; return (TRUE); } @@ -28,7 +44,8 @@ /* find buffer size */ - out[0].datasize = in[0].datasize; - out[0].pad = in[0].pad; - out[0].Nrow = in[0].Nrow; + out[0].datasize = in[0].datasize; + // out[0].heap_start = in[0].heap_start; + out[0].pad = in[0].pad; + out[0].Nrow = in[0].Nrow; off_t Nrows = out[0].Nrow; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_create_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_create_T.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_create_T.c (revision 38447) @@ -4,8 +4,9 @@ /*********************** fits init table *******************************/ int gfits_init_table (FTable *table) { - table[0].header = NULL; - table[0].buffer = NULL; - table[0].datasize = 0; - table[0].validsize = 0; + table[0].header = NULL; + table[0].buffer = NULL; + table[0].datasize = 0; + table[0].heap_start = 0; + table[0].validsize = 0; return (TRUE); @@ -42,6 +43,21 @@ } table[0].datasize = Nbytes; + table[0].heap_start = gfits_heap_start (header); return (TRUE); +} + +/*********************** fits create table *******************************/ +int gfits_set_table_rows (Header *header, FTable *table, off_t Nrows) { + + header[0].Naxis[1] = Nrows; + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Nrows); + + off_t Nbytes = gfits_data_size (header); + REALLOCATE (table[0].buffer, char, Nbytes); + bzero (table[0].buffer, Nbytes); + table[0].datasize = Nbytes; + table[0].heap_start = gfits_heap_start (header); + return (TRUE); } @@ -60,4 +76,5 @@ Ny = table[0].header[0].Naxis[0]; table[0].datasize = gfits_data_size (header); + // table[0].heap_start = gfits_heap_start (header); table[0].pad = table[0].datasize - Nx*Ny; Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_define_column.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_define_column.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_define_column.c (revision 38447) @@ -66,7 +66,11 @@ sprintf (field, "TTYPE%d", Nfields); gfits_modify (header, field, "%s", 1, label); - gfits_modify_alt (header, field, "%C", 1, comment); - sprintf (field, "TUNIT%d", Nfields); - gfits_modify (header, field, "%s", 1, unit); + if (comment) { + gfits_modify_alt (header, field, "%C", 1, comment); + } + if (unit) { + sprintf (field, "TUNIT%d", Nfields); + gfits_modify (header, field, "%s", 1, unit); + } sprintf (field, "TFORM%d", Nfields); gfits_modify (header, field, "%s", 1, format); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_get_column.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_get_column.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_get_column.c (revision 38447) @@ -17,4 +17,9 @@ void *gfits_get_bintable_column_data (Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol) { + void *data = gfits_get_bintable_column_data_raw (header, table, label, type, Nrow, Ncol, FALSE); + return data; +} + +void *gfits_get_bintable_column_data_raw (Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol, char nativeOrder) { off_t Nx, Ny; @@ -71,15 +76,19 @@ } - /* convert data in-situ with correct type, byte swap and Bzero/Bscale */ + // NOTE: we have already copied the data to 'array', so the blocks below + // only need to swap if this is a direct copy Pin = array; Pout = array; + int directCopy = (Bzero == 0.0) && (Bscale == 1.0); + + /* convert data in-situ with correct type, byte swap and Bzero/Bscale */ if (!strcmp (type, "char")) { for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { - *(char *)Pout = *(char *)Pin*Bscale + Bzero; + if (!directCopy) { *(char *)Pout = *(char *)Pin*Bscale + Bzero; } } } if (!strcmp (type, "byte")) { for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { - *(char *)Pout = *(char *)Pin*Bscale + Bzero; + if (!directCopy) { *(char *)Pout = *(char *)Pin*Bscale + Bzero; } } } @@ -87,7 +96,7 @@ for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { # ifdef BYTE_SWAP - SWAP_BYTE; -# endif - *(short *)Pout = *(short *)Pin*Bscale + Bzero; + if (!nativeOrder) { SWAP_BYTE; } +# endif + if (!directCopy) { *(short *)Pout = *(short *)Pin*Bscale + Bzero; } } } @@ -95,24 +104,15 @@ for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { # ifdef BYTE_SWAP - SWAP_WORD; -# endif - *(int *)Pout = *(int *)Pin*Bscale + Bzero; + if (!nativeOrder) { SWAP_WORD; } +# endif + if (!directCopy) { *(int *)Pout = *(int *)Pin*Bscale + Bzero; } } } if (!strcmp (type, "int64_t")) { - if ((Bzero == 0.0) && (Bscale == 1.0)) { - for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { -# ifdef BYTE_SWAP - SWAP_DBLE; -# endif - *(int64_t *)Pout = *(int64_t *)Pin; - } - } else { - for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { -# ifdef BYTE_SWAP - SWAP_DBLE; -# endif - *(int64_t *)Pout = *(int64_t *)Pin*Bscale + Bzero; - } + for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_DBLE; } +# endif + if (!directCopy) { *(int64_t *)Pout = *(int64_t *)Pin*Bscale + Bzero; } } } @@ -120,7 +120,7 @@ for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { # ifdef BYTE_SWAP - SWAP_WORD; -# endif - *(float *)Pout = *(float *)Pin*Bscale + Bzero; + if (!nativeOrder) { SWAP_WORD; } +# endif + if (!directCopy) { *(float *)Pout = *(float *)Pin*Bscale + Bzero; } } } @@ -128,9 +128,11 @@ for (i = 0; i < Nval*Ny; i++, Pin+=Nbytes, Pout+=Nbytes) { # ifdef BYTE_SWAP - SWAP_DBLE; -# endif - *(double *)Pout = *(double *)Pin*Bscale + Bzero; - } - } + if (!nativeOrder) { SWAP_DBLE; } +# endif + if (!directCopy) { *(double *)Pout = *(double *)Pin*Bscale + Bzero; } + } + } + + // check that we supplied a valid type *Ncol = Nval; @@ -174,4 +176,18 @@ if (!gfits_get_bintable_column_type_by_N (header, N, type, Nval)) return FALSE; return (TRUE); +} + +/***********************/ +int gfits_get_bintable_column_raw (Header *header, FTable *table, char *label, void **data, char nativeOrder) { + + char type[16]; + off_t Nrow; + int Ncol; + + char *array = gfits_get_bintable_column_data_raw (header, table, label, type, &Nrow, &Ncol, nativeOrder); + if (array == NULL) return (FALSE); + + *data = array; + return TRUE; } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_read_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_read_T.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_read_T.c (revision 38447) @@ -45,19 +45,17 @@ if (Nread < gfits_data_min_size (table[0].header)) { fprintf (stderr, "error: fits read error in %s, read "OFF_T_FMT", need "OFF_T_FMT"\n", __func__, Nread, gfits_data_min_size (table[0].header)); - if (padIfShort) { - memset (&table[0].buffer[Nread], 0, Nbytes - Nread); - fprintf (stderr, "warning: file missing data, padding with zeros: USE AT YOUR OWN RISK!\n"); - table[0].validsize = Nread; - table[0].datasize = Nbytes; - return (TRUE); - } else { - gfits_free_table (table); + if (!padIfShort) { + gfits_free_table (table); return (FALSE); } - } - fprintf (stderr, "warning: file missing pad\n"); + memset (&table[0].buffer[Nread], 0, Nbytes - Nread); + fprintf (stderr, "warning: file missing data, padding with zeros: USE AT YOUR OWN RISK!\n"); + } else { + fprintf (stderr, "warning: file missing pad\n"); + } } table[0].validsize = Nread; table[0].datasize = Nbytes; + table[0].heap_start = gfits_heap_start (table->header); return (TRUE); } @@ -93,13 +91,10 @@ } fprintf (stderr, "error: fits read error in %s, read "OFF_T_FMT", need "OFF_T_FMT"\n", __func__, Nread, Nbytes); - if (padIfShort) { - memset (&table[0].buffer[Nread], 0, Nbytes - Nread); - fprintf (stderr, "warning: file missing data, padding with zeros: USE AT YOUR OWN RISK!\n"); - table[0].validsize = Nread; - table[0].datasize = Nbytes; - } else { + if (!padIfShort) { free (table[0].buffer); return (FALSE); } + memset (&table[0].buffer[Nread], 0, Nbytes - Nread); + fprintf (stderr, "warning: file missing data, padding with zeros: USE AT YOUR OWN RISK!\n"); } @@ -107,5 +102,7 @@ table[0].header[0].Naxis[1] = Nrows; gfits_modify (table[0].header, "NAXIS2", OFF_T_FMT, 1, Nrows); - table[0].datasize = gfits_data_size (table[0].header); + table[0].validsize = Nread; + table[0].datasize = Nbytes; + table[0].heap_start = gfits_heap_start (table->header); return (TRUE); @@ -122,4 +119,5 @@ Ny = table[0].header[0].Naxis[1]; table[0].datasize = gfits_data_size (table[0].header); + // table[0].heap_start = gfits_heap_start (table[0].header); table[0].pad = table[0].datasize - Nx*Ny; @@ -206,4 +204,5 @@ for (i = 0; i < Nrow; i++) table[0].row[i] = row[i]; table[0].datasize = gfits_data_size (table[0].header); + // table[0].heap_start = gfits_heap_start (table[0].header); table[0].pad = table[0].datasize - Nx*Ny; return (TRUE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_set_column.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_set_column.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_set_column.c (revision 38447) @@ -1,4 +1,8 @@ # include # include + +# define SWAP_NONE + +# ifdef BYTE_SWAP # define SWAP_BYTE { \ char tmp; \ @@ -14,9 +18,14 @@ tmp = Pout[2]; Pout[2] = Pout[5]; Pout[5] = tmp; \ tmp = Pout[3]; Pout[3] = Pout[4]; Pout[4] = tmp; } +# else +# define SWAP_BYTE +# define SWAP_WORD +# define SWAP_DBLE +# endif /***********************/ int gfits_set_bintable_column (Header *header, FTable *table, char *label, void *data, off_t Nrow) { - off_t Nx, Ny, nbytes; + off_t Nx, Ny; int i, N, Nfields; int Nval, Nbytes, Nstart, Nv, Nb; @@ -55,13 +64,9 @@ gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); + + // if no rows have yet been assigned, we need to allocate the full data buffer if (Ny == 0) { Ny = Nrow; - header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); - - nbytes = gfits_data_size (header); - REALLOCATE (table[0].buffer, char, MAX (nbytes, 1)); - bzero (table[0].buffer, nbytes); - table[0].datasize = nbytes; + gfits_set_table_rows (header, table, Ny); } if (Ny != Nrow) return (FALSE); @@ -143,4 +148,22 @@ } + if (!strcmp (type, "var")) { + for (i = 0; i < Nval*Nrow; i++, Pin+=Nbytes, Pout+=Nbytes) { + *(int *)Pout = *(int *)Pin; +# ifdef BYTE_SWAP + SWAP_WORD; +# endif + } + } + + if (!strcmp (type, "var64")) { + for (i = 0; i < Nval*Nrow; i++, Pin+=Nbytes, Pout+=Nbytes) { + *(int *)Pout = *(int *)Pin; +# ifdef BYTE_SWAP + SWAP_DBLE; +# endif + } + } + /* check array space */ if (Nx*Ny < Nx*(Nrow - 1) + Nstart + Nval*Nbytes) { @@ -161,7 +184,8 @@ /***********************/ -int gfits_set_bintable_column_reformat (Header *header, FTable *table, char *label, char *intype, void *data, off_t Nrow) { - - off_t Nx, Ny, nbytes; +// convert the input data array (of the specified intype) to the desired table data type. swap unless nativeOrder is requested +int gfits_set_bintable_column_reformat (Header *header, FTable *table, char *label, char *intype, void *data, off_t Nrow, char nativeOrder) { + + off_t Nx, Ny; int i, N, Nfields; int Nval, NbytesOut, Nstart, Nv, Nb; @@ -202,11 +226,5 @@ if (Ny == 0) { Ny = Nrow; - header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); - - nbytes = gfits_data_size (header); - REALLOCATE (table[0].buffer, char, MAX (nbytes, 1)); - bzero (table[0].buffer, nbytes); - table[0].datasize = nbytes; + gfits_set_table_rows (header, table, Ny); } if (Ny != Nrow) return (FALSE); @@ -230,12 +248,141 @@ // if (!strcmp (intype, #ITYPE)) { + int directCopy = (Bzero == 0.0) && (Bscale == 1.0); + +# define SET_VALUES(OUTNAME, OUTTYPE, INNAME, INTYPE, SWAP_OP, NBYTES_IN) \ + if (!strcmp (outtype, OUTNAME) && !strcmp (intype, INNAME)) { \ + int NbytesIn = NBYTES_IN; \ + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { \ + if (directCopy) { \ + *(OUTTYPE *)Pout = *(INTYPE *)Pin; \ + } else { \ + *(OUTTYPE *)Pout = (*(INTYPE *)Pin - Bzero) / Bscale; \ + } \ + if (!nativeOrder) { SWAP_OP; }}} + + SET_VALUES("char", char, "char", char, SWAP_NONE, 1); + SET_VALUES("byte", char, "char", char, SWAP_NONE, 1); + SET_VALUES("short", short, "char", char, SWAP_BYTE, 1); + SET_VALUES("int", int, "char", char, SWAP_WORD, 1); + SET_VALUES("int64_t", int64_t, "char", char, SWAP_DBLE, 1); + SET_VALUES("float", float, "char", char, SWAP_WORD, 1); + SET_VALUES("double", double, "char", char, SWAP_DBLE, 1); + + SET_VALUES("char", char, "byte", char, SWAP_NONE, 1); + SET_VALUES("byte", char, "byte", char, SWAP_NONE, 1); + SET_VALUES("short", short, "byte", char, SWAP_BYTE, 1); + SET_VALUES("int", int, "byte", char, SWAP_WORD, 1); + SET_VALUES("int64_t", int64_t, "byte", char, SWAP_DBLE, 1); + SET_VALUES("float", float, "byte", char, SWAP_WORD, 1); + SET_VALUES("double", double, "byte", char, SWAP_DBLE, 1); + + SET_VALUES("char", char, "short", short, SWAP_NONE, 2); + SET_VALUES("byte", char, "short", short, SWAP_NONE, 2); + SET_VALUES("short", short, "short", short, SWAP_BYTE, 2); + SET_VALUES("int", int, "short", short, SWAP_WORD, 2); + SET_VALUES("int64_t", int64_t, "short", short, SWAP_DBLE, 2); + SET_VALUES("float", float, "short", short, SWAP_WORD, 2); + SET_VALUES("double", double, "short", short, SWAP_DBLE, 2); + + SET_VALUES("char", char, "int", int, SWAP_NONE, 4); + SET_VALUES("byte", char, "int", int, SWAP_NONE, 4); + SET_VALUES("short", short, "int", int, SWAP_BYTE, 4); + SET_VALUES("int", int, "int", int, SWAP_WORD, 4); + SET_VALUES("int64_t", int64_t, "int", int, SWAP_DBLE, 4); + SET_VALUES("float", float, "int", int, SWAP_WORD, 4); + SET_VALUES("double", double, "int", int, SWAP_DBLE, 4); + + SET_VALUES("char", char, "int64_t", int64_t, SWAP_NONE, 8); + SET_VALUES("byte", char, "int64_t", int64_t, SWAP_NONE, 8); + SET_VALUES("short", short, "int64_t", int64_t, SWAP_BYTE, 8); + SET_VALUES("int", int, "int64_t", int64_t, SWAP_WORD, 8); + SET_VALUES("int64_t", int64_t, "int64_t", int64_t, SWAP_DBLE, 8); + SET_VALUES("float", float, "int64_t", int64_t, SWAP_WORD, 8); + SET_VALUES("double", double, "int64_t", int64_t, SWAP_DBLE, 8); + + SET_VALUES("char", char, "float", float, SWAP_NONE, 4); + SET_VALUES("byte", char, "float", float, SWAP_NONE, 4); + SET_VALUES("short", short, "float", float, SWAP_BYTE, 4); + SET_VALUES("int", int, "float", float, SWAP_WORD, 4); + SET_VALUES("int64_t", int64_t, "float", float, SWAP_DBLE, 4); + SET_VALUES("float", float, "float", float, SWAP_WORD, 4); + SET_VALUES("double", double, "float", float, SWAP_DBLE, 4); + + SET_VALUES("char", char, "double", double, SWAP_NONE, 8); + SET_VALUES("byte", char, "double", double, SWAP_NONE, 8); + SET_VALUES("short", short, "double", double, SWAP_BYTE, 8); + SET_VALUES("int", int, "double", double, SWAP_WORD, 8); + SET_VALUES("int64_t", int64_t, "double", double, SWAP_DBLE, 8); + SET_VALUES("float", float, "double", double, SWAP_WORD, 8); + SET_VALUES("double", double, "double", double, SWAP_DBLE, 8); + +# if (0) /** input == char **/ if (!strcmp (outtype, "char") && !strcmp (intype, "char")) { int NbytesIn = 1; for (i = 0; i < Nval*Nrow; i++, Pin+=NbytesIn, Pout+=NbytesOut) { - *(char *)Pout = (*(char *)Pin - Bzero) / Bscale; + if (directCopy) { *(char *)Pout = *(char *)Pin; } else { + *(char *)Pout = (*(char *)Pin - Bzero) / Bscale; + } } } if (!strcmp (outtype, "byte") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin+=NbytesIn, Pout+=NbytesOut) { + if (directCopy) { *(char *)Pout = *(char *)Pin; } else { + *(char *)Pout = (*(char *)Pin - Bzero) / Bscale; + } + } + } + if (!strcmp (outtype, "short") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { + if (directCopy) { *(short *)Pout = *(char *)Pin; } else { + *(short *)Pout = (*(char *)Pin - Bzero) / Bscale; + } +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_BYTE; } +# endif + } + } + if (!strcmp (outtype, "int") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { + *(int *)Pout = (*(char *)Pin - Bzero) / Bscale; +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_WORD; } +# endif + } + } + if (!strcmp (outtype, "int64_t") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { + *(int64_t *)Pout = (*(char *)Pin - Bzero) / Bscale; +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_DBLE; } +# endif + } + } + if (!strcmp (outtype, "float") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { + *(float *)Pout = (*(char *)Pin - Bzero) / Bscale; +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_WORD; } +# endif + } + } + if (!strcmp (outtype, "double") && !strcmp (intype, "char")) { + int NbytesIn = 1; + for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { + *(double *)Pout = (*(char *)Pin - Bzero) / Bscale; +# ifdef BYTE_SWAP + if (!nativeOrder) { SWAP_DBLE; } +# endif + } + } + + /** input == byte **/ + if (!strcmp (outtype, "char") && !strcmp (intype, "byte")) { int NbytesIn = 1; for (i = 0; i < Nval*Nrow; i++, Pin+=NbytesIn, Pout+=NbytesOut) { @@ -243,52 +390,5 @@ } } - if (!strcmp (outtype, "short") && !strcmp (intype, "char")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { - *(short *)Pout = (*(char *)Pin - Bzero) / Bscale; -# ifdef BYTE_SWAP - SWAP_BYTE; -# endif - } - } - if (!strcmp (outtype, "int") && !strcmp (intype, "char")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { - *(int *)Pout = (*(char *)Pin - Bzero) / Bscale; -# ifdef BYTE_SWAP - SWAP_WORD; -# endif - } - } - if (!strcmp (outtype, "int64_t") && !strcmp (intype, "char")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { - *(int64_t *)Pout = (*(char *)Pin - Bzero) / Bscale; -# ifdef BYTE_SWAP - SWAP_DBLE; -# endif - } - } - if (!strcmp (outtype, "float") && !strcmp (intype, "char")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { - *(float *)Pout = (*(char *)Pin - Bzero) / Bscale; -# ifdef BYTE_SWAP - SWAP_WORD; -# endif - } - } - if (!strcmp (outtype, "double") && !strcmp (intype, "char")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin += NbytesIn, Pout += NbytesOut) { - *(double *)Pout = (*(char *)Pin - Bzero) / Bscale; -# ifdef BYTE_SWAP - SWAP_DBLE; -# endif - } - } - - /** input == byte **/ - if (!strcmp (outtype, "char") && !strcmp (intype, "byte")) { + if (!strcmp (outtype, "byte") && !strcmp (intype, "byte")) { int NbytesIn = 1; for (i = 0; i < Nval*Nrow; i++, Pin+=NbytesIn, Pout+=NbytesOut) { @@ -296,10 +396,4 @@ } } - if (!strcmp (outtype, "byte") && !strcmp (intype, "byte")) { - int NbytesIn = 1; - for (i = 0; i < Nval*Nrow; i++, Pin+=NbytesIn, Pout+=NbytesOut) { - *(char *)Pout = (*(char *)Pin - Bzero) / Bscale; - } - } if (!strcmp (outtype, "short") && !strcmp (intype, "byte")) { int NbytesIn = 1; @@ -307,5 +401,5 @@ *(short *)Pout = (*(char *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -316,5 +410,5 @@ *(int *)Pout = (*(char *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -325,5 +419,5 @@ *(int64_t *)Pout = (*(char *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -334,5 +428,5 @@ *(float *)Pout = (*(char *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -343,5 +437,5 @@ *(double *)Pout = (*(char *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -366,5 +460,5 @@ *(short *)Pout = (*(short *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -375,5 +469,5 @@ *(int *)Pout = (*(short *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -384,5 +478,5 @@ *(int64_t *)Pout = (*(short *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -393,5 +487,5 @@ *(float *)Pout = (*(short *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -402,5 +496,5 @@ *(double *)Pout = (*(short *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -425,5 +519,5 @@ *(short *)Pout = (*(int *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -434,5 +528,5 @@ *(int *)Pout = (*(int *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -443,5 +537,5 @@ *(int64_t *)Pout = (*(int *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -452,5 +546,5 @@ *(float *)Pout = (*(int *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -461,5 +555,5 @@ *(double *)Pout = (*(int *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -484,5 +578,5 @@ *(short *)Pout = (*(int64_t *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -493,5 +587,5 @@ *(int *)Pout = (*(int64_t *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -502,5 +596,5 @@ *(int64_t *)Pout = (*(int64_t *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -511,5 +605,5 @@ *(float *)Pout = (*(int64_t *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -520,5 +614,5 @@ *(double *)Pout = (*(int64_t *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -543,5 +637,5 @@ *(short *)Pout = (*(float *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -552,5 +646,5 @@ *(int *)Pout = (*(float *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -561,5 +655,5 @@ *(int64_t *)Pout = (*(float *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -570,5 +664,5 @@ *(float *)Pout = (*(float *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -579,5 +673,5 @@ *(double *)Pout = (*(float *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -602,5 +696,5 @@ *(short *)Pout = (*(double *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_BYTE; + if (!nativeOrder) { SWAP_BYTE; } # endif } @@ -611,5 +705,5 @@ *(int *)Pout = (*(double *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -620,5 +714,5 @@ *(int64_t *)Pout = (*(double *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; + if (!nativeOrder) { SWAP_DBLE; } # endif } @@ -629,5 +723,5 @@ *(float *)Pout = (*(double *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_WORD; + if (!nativeOrder) { SWAP_WORD; } # endif } @@ -638,8 +732,9 @@ *(double *)Pout = (*(double *)Pin - Bzero) / Bscale; # ifdef BYTE_SWAP - SWAP_DBLE; -# endif - } - } + if (!nativeOrder) { SWAP_DBLE; } +# endif + } + } +# endif /* check array space */ @@ -683,5 +778,5 @@ int gfits_set_table_column (Header *header, FTable *table, char *label, void *data, off_t Nrow) { - off_t Nx, Ny, nbytes; + off_t Nx, Ny; int i, N, Nfields, Nval, Nbytes, Nstart, Nv, Nb; char tlabel[80], field[80], format[80], cformat[80], type[16], tmp[16]; @@ -713,11 +808,5 @@ if (Ny == 0) { Ny = Nrow; - header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); - - nbytes = gfits_data_size (header); - REALLOCATE (table[0].buffer, char, MAX (nbytes, 1)); - bzero (table[0].buffer, nbytes); - table[0].datasize = nbytes; + gfits_set_table_rows (header, table, Ny); } if (Ny != Nrow) return (FALSE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_format.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_format.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_format.c (revision 38447) @@ -8,13 +8,12 @@ int gfits_bintable_format (char *format, char *type, int *Nval, int *Nbytes) { - char *Lchar, *Fchar; - int Nv; - - if (format == (char *) NULL) return (FALSE); - if (format[0] == 0) return (FALSE); - Lchar = &format[strlen(format) - 1]; - - Nv = strtol (format, &Fchar, 10); - if (Fchar != Lchar) return (FALSE); + char *Fchar; + + if (format == (char *) NULL) return (FALSE); // format must be defined + if (format[0] == 0) return (FALSE); // format must not be empty string + // char *Lchar = &format[strlen(format) - 1]; // last-char pointer + + int Nv = strtol (format, &Fchar, 10); // Fchar points at end of leading number + // if (Fchar != Lchar) return (FALSE); -- this test is invalid for eg 1PB(nn) if ((Nv == 0) && (Fchar == format)) Nv = 1; @@ -51,7 +50,4 @@ { *Nbytes = 8; strcpy (type, "double"); *Nval = Nv; } break; - case 'P': - { *Nbytes = 8; strcpy (type, "var"); *Nval = 2*Nv; } - break; case 'C': { *Nbytes = 8; strcpy (type, "float"); *Nval = 2*Nv; } @@ -59,4 +55,10 @@ case 'M': { *Nbytes = 16; strcpy (type, "double"); *Nval = 2*Nv; } + break; + case 'P': + { *Nbytes = 4; strcpy (type, "var"); *Nval = 2*Nv; } + break; + case 'Q': + { *Nbytes = 8; strcpy (type, "var64"); *Nval = 2*Nv; } break; default: @@ -80,9 +82,9 @@ C - complex float M - complex double - P - var length array descrpt (64 bit) + P - var length array descrpt (32 bit pointer) + Q - var length array descrpt (64 bit pointer) all can be preceeded by integer N which specified number of entries in array - */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_row.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_row.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_row.c (revision 38447) @@ -28,4 +28,5 @@ REALLOCATE (table[0].buffer, char, MAX (nbytes, 1)); table[0].datasize = nbytes; + table[0].heap_start = gfits_heap_start (header); memcpy (&table[0].buffer[Nstart], data, Nbytes*Nrow); @@ -67,4 +68,5 @@ table[0].datasize = gfits_data_size (table[0].header); + // table[0].heap_start = gfits_heap_start (header); table[0].pad = table[0].datasize - Nx*Ny; @@ -102,4 +104,5 @@ REALLOCATE (table[0].buffer, char, MAX (nbytes, 1)); table[0].datasize = nbytes; + table[0].heap_start = gfits_heap_start (header); return (TRUE); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_varlength.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_varlength.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_table_varlength.c (revision 38447) @@ -2,4 +2,5 @@ # include +// fills the column definition structure for a specific column based on a given table. int gfits_varlength_column_define (FTable *ftable, VarLengthColumn *def, int column) { @@ -14,4 +15,6 @@ snprintf (field, 80, "TFORM%d", column); if (!gfits_scan (ftable->header, field, "%s", 1, format)) return (FALSE); + + def->column = column; // find and remove the max field length element @@ -48,4 +51,5 @@ if (*p1 == 'I') { def->nbytes = 2; def->format = *p1; } if (*p1 == 'J') { def->nbytes = 4; def->format = *p1; } + if (*p1 == 'K') { def->nbytes = 8; def->format = *p1; } if (*p1 == 'E') { def->nbytes = 4; def->format = *p1; } if (*p1 == 'D') { def->nbytes = 8; def->format = *p1; } @@ -54,18 +58,58 @@ if (!def->nbytes) { return (FALSE); } - /* scan columns to find column offset */ - def->Nstart = 0; + /* scan columns to find column offset for metadata column*/ + def->offset = 0; for (i = 1; i < column; i++) { sprintf (field, "TFORM%d", i); gfits_scan (ftable->header, field, "%s", 1, format); gfits_bintable_format (format, tmpline, &Nv, &Nb); - def->Nstart += Nv*Nb; + def->offset += Nv*Nb; } // heap_start must be long long so file may be very large - if (!gfits_scan (ftable->header, "THEAP", OFF_T_FMT, 1, &def->heap_start)) { - def->heap_start = ftable->header->Naxis[0]*ftable->header->Naxis[1]; - } - + // confirm that ftable->heap_start is correctly set? + + return TRUE; +} + +# define SWAP_WORD \ + tmp = pchar[0]; pchar[0] = pchar[3]; pchar[3] = tmp; \ + tmp = pchar[1]; pchar[1] = pchar[2]; pchar[2] = tmp; + +# define SWAP_DBLE \ + tmp = pchar[0]; pchar[0] = pchar[7]; pchar[7] = tmp; \ + tmp = pchar[1]; pchar[1] = pchar[6]; pchar[6] = tmp; \ + tmp = pchar[2]; pchar[2] = pchar[5]; pchar[5] = tmp; \ + tmp = pchar[3]; pchar[3] = pchar[4]; pchar[4] = tmp; + +int gfits_byteswap_varlength_column (FTable *ftable, int column) { + +# ifdef BYTE_SWAP + VarLengthColumn zdef; + if (!gfits_varlength_column_define (ftable, &zdef, column)) return FALSE; + + char *bufstart = &ftable->buffer[zdef.offset]; + + int i; + char *pchar, tmp; + + off_t Nx = ftable->header->Naxis[0]; + + for (i = 0; i < ftable->header->Naxis[1]; i++) { + if (zdef.mode == 'P') { + pchar = &bufstart[i*Nx]; + SWAP_WORD; + pchar = &bufstart[i*Nx + 4]; + SWAP_WORD; + } + if (zdef.mode == 'Q') { + char *pchar, tmp; + pchar = &bufstart[i*Nx]; + SWAP_DBLE; + pchar = &bufstart[i*Nx + 8]; + SWAP_DBLE; + } + } +# endif return TRUE; } @@ -75,41 +119,106 @@ void *gfits_varlength_column_pointer (FTable *ftable, VarLengthColumn *column, off_t row, off_t *length) { - void *result; - off_t offset, Nx; - if ((column->mode != 'P') && (column->mode != 'Q')) abort(); // find the values for the specified row // the values in the main table for this row and varlength column: - Nx = ftable->header->Naxis[0]; - -# define SWAP_WORD \ - tmp = pchar[0]; pchar[0] = pchar[3]; pchar[3] = tmp; \ - tmp = pchar[1]; pchar[1] = pchar[2]; pchar[2] = tmp; - -# ifdef BYTE_SWAP - char *pchar, tmp; - pchar = &ftable->buffer[row*Nx + column->Nstart]; - SWAP_WORD; - pchar = &ftable->buffer[row*Nx + column->Nstart + 4]; - SWAP_WORD; -# endif + off_t Nx = ftable->header->Naxis[0]; + off_t offset = 0; if (column->mode == 'P') { int *ptr; - ptr = (int *) &ftable->buffer[row*Nx + column->Nstart]; + ptr = (int *) &ftable->buffer[row*Nx + column->offset]; *length = ptr[0]; offset = ptr[1]; - } else if (column->mode == 'Q') { + } + if (column->mode == 'Q') { off_t *ptr; - ptr = (off_t *) &ftable->buffer[row*Nx + column->Nstart]; + ptr = (off_t *) &ftable->buffer[row*Nx + column->offset]; *length = ptr[0]; offset = ptr[1]; - } else { - // this was actually trapped above - abort(); - } - - result = (void *) (ftable->buffer + column->heap_start + offset); + } + + // fprintf (stderr, "length: %d, offset: %d\n", (int) *length, (int) offset); + + void *result = (void *) (ftable->buffer + ftable->heap_start + offset); return result; } + +// to generate a compressed image or a compressed table, we need to be able to add data +// for varlength columns to the heap. To start, assume we have defined the cartesian +// portion of the table correctly, and are only extending the heap. + +int gfits_varlength_column_add_data (FTable *table, char *data, off_t Ndata, int row, VarLengthColumn *column) { + + // find the current starting point for new data (end of current buffer main data + current heap + off_t heap_offset = gfits_data_min_size (table->header); + + // extend the buffer size to add the new data + off_t Nbytes = gfits_data_pad_size (heap_offset + Ndata); + REALLOCATE (table->buffer, char, Nbytes); + + char *heap_ptr = table->buffer + heap_offset; + + // add the new data to the table buffer + memcpy (heap_ptr, data, Ndata); + + // *** now we add the data description to the table *** + + if ((column->mode != 'P') && (column->mode != 'Q')) abort(); + + // find the cell for the specified row & column + // the values in the main table for this row and varlength column: + int Nx = table->header->Naxis[0]; + + // set the cell values + if (column->mode == 'P') { + int *ptr; + ptr = (int *) &table->buffer[row*Nx + column->offset]; + ptr[0] = Ndata; + ptr[1] = heap_offset - table->heap_start; + } + if (column->mode == 'Q') { + off_t *ptr; + ptr = (off_t *) &table->buffer[row*Nx + column->offset]; + ptr[0] = Ndata; + ptr[1] = heap_offset - table->heap_start; + } + + // update maxlen + // fprintf (stderr, "row: %d, Ndata: %d, maxlen: %d\n", (int) row, (int) Ndata, (int) column->maxlen); + column->maxlen = MAX (column->maxlen, Ndata); + + off_t pcount; + gfits_scan (table->header, "PCOUNT", OFF_T_FMT, 1, &pcount); + + pcount += Ndata; + gfits_modify (table->header, "PCOUNT", OFF_T_FMT, 1, pcount); + table->header->pcount = pcount; + table->datasize = gfits_data_size (table->header); + myAssert (table->datasize == Nbytes, "inconsistent header and data"); + + return TRUE; +} + +// update the header TFORM value for this entry +int gfits_varlength_column_finish (FTable *ftable, VarLengthColumn *def) { + + char field[81]; + char format[81]; + + // grab the value of TFORMn for this column + snprintf (field, 80, "TFORM%d", def->column); + if (!gfits_scan (ftable->header, field, "%s", 1, format)) return FALSE; + + // find and update the max field length element + char *p1 = strchr (format, '('); + if (!p1) return (FALSE); // not a valid varlength column -- missing (e_max) + + p1 ++; + *p1 = 0; + + snprintf (p1, 80 - (p1 - format), "%d)", def->maxlen); + if (!gfits_modify (ftable->header, field, "%s", 1, format)) return FALSE; + + return TRUE; +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_uncompress_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_uncompress_T.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_uncompress_T.c (revision 38447) @@ -0,0 +1,424 @@ +# include +# include +# include + +# define ESCAPE { fprintf (stderr, "error in %s @ line %d\n", __func__, __LINE__); goto escape; } + +int gfits_distribute_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows); +int gfits_distribute_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows); + +# define VERBOSE_DUMP 0 + +int gfits_dump_raw_table (FTable *table, char *message) { +# if (VERBOSE_DUMP) + int i; + fprintf (stderr, "%s data:\n", message); + for (i = 0; i < table->header->Naxis[0]*table->header->Naxis[1]; i++) { + // for (i = 0; i < 16; i++) { + fprintf (stderr, "%02hhx", table->buffer[i]); + if (i % 2) fprintf (stderr, " "); + if (i % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); +# endif + return TRUE; +} + +int gfits_dump_cmp_table (FTable *table, char *message) { +# if (VERBOSE_DUMP) + int i; + fprintf (stderr, "%s pntr:\n", message); + for (i = 0; i < table->header->Naxis[0]*table->header->Naxis[1]; i++) { + fprintf (stderr, "%02hhx", table->buffer[i]); + if (i % 2) fprintf (stderr, " "); + if (i % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + + fprintf (stderr, "%s data:\n", message); + for (i = 0; i < table->header->pcount; i++) { + fprintf (stderr, "%02hhx", table->buffer[table->heap_start + i]); + if (i % 2) fprintf (stderr, " "); + if (i % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); +# endif + return TRUE; +} + +static float timeSum1 = 0.0; +static float timeSum2 = 0.0; +static float timeSum2a = 0.0; +static float timeSum2b = 0.0; +static float timeSum2c = 0.0; +static float timeSum2d = 0.0; +static float timeSum2e = 0.0; +static float timeSum2f = 0.0; +static float timeSum3 = 0.0; + +void gfits_uncompress_timing () { + fprintf (stderr, "unc times: %f %f %f\n", timeSum1, timeSum2, timeSum3); + fprintf (stderr, "unc times: %f %f %f %f %f %f\n", timeSum2a, timeSum2b, timeSum2c, timeSum2d, timeSum2e, timeSum2f); + + timeSum1 = 0.0; + timeSum2 = 0.0; + timeSum2a = 0.0; + timeSum2b = 0.0; + timeSum2c = 0.0; + timeSum2d = 0.0; + timeSum2e = 0.0; + timeSum2f = 0.0; + timeSum3 = 0.0; +} + +int gfits_uncompress_table (FTable *srctable, FTable *tgttable) { + + int i; + + char keyword[81]; + + char *raw = NULL; + char *zdata = NULL; + + Header *srcheader = srctable->header; + Header *tgtheader = tgttable->header; + + // XXX EAM: + struct timeval startTimer, stopTimer; + float dtime; + gettimeofday (&startTimer, (void *) NULL); + + int ztilelen; + if (!gfits_scan (srcheader, "ZTILELEN", "%d", 1, &ztilelen)) ESCAPE; + // XXX if ZTILELEN is missing, it should have the value of ZNAXIS2 + + // EXTNAME is not mandatory, keep it if it exists: + char extname[80]; + if (!gfits_scan (srcheader, "EXTNAME", "%s", 1, extname)) { + strcpy (extname, "UNCOMPRESSED_TABLE"); + } + + // creates an empty (Naxes = 2, Naxis[i] = 0) table header with XTENSION = BINTABLE, EXTNAME = COMPRESSED_TABLE + if (!gfits_create_table_header (tgtheader, "BINTABLE", extname)) ESCAPE; + + int Nfields; + if (!gfits_scan (srcheader, "TFIELDS", "%d", 1, &Nfields)) ESCAPE; + + // below we generate a fake table with tmpbuffers. allocated it to the max needed size + int max_width = 0; + int offset = 0; + + TableField *fields = NULL; + ALLOCATE (fields, TableField, Nfields); + for (i = 0; i < Nfields; i++) { + snprintf (keyword, 80, "TTYPE%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].ttype)) ESCAPE; + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].ttype_cmt)) ESCAPE; + + // TUNIT is not mandatory + snprintf (keyword, 80, "TUNIT%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tunit)) { + fields[i].tunit[0] = 0; + } else { + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tunit_cmt)) ESCAPE; + } + + snprintf (keyword, 80, "ZFORM%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tformat)) ESCAPE; + if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tformat_cmt)) ESCAPE; + + snprintf (keyword, 80, "ZCTYP%d", i+1); + if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].zctype)) { + strcpy (fields[i].zctype, "GZIP_2"); // XXX NOTE: not yet supported + } + + if (!gfits_bintable_format (fields[i].tformat, fields[i].datatype, &fields[i].Nvalues, &fields[i].pixsize)) ESCAPE; + fields[i].rowsize = fields[i].Nvalues*fields[i].pixsize; + max_width = MAX(max_width, fields[i].rowsize); + + fields[i].offset = offset; + offset += fields[i].rowsize; + + if (!gfits_varlength_column_define (srctable, &fields[i].zdef, i + 1)) ESCAPE; + + if (!gfits_define_bintable_column (tgtheader, fields[i].tformat, fields[i].ttype, fields[i].ttype_cmt, fields[i].tunit, 1.0, 0.0)) ESCAPE; + } + + // allocates the default data array (tgttable->buffer) + if (!gfits_create_table (tgtheader, tgttable)) ESCAPE; + + // copy original header to output header (XXX this needs to be finished) + if (!gfits_copy_keywords_compress (srcheader, tgtheader)) ESCAPE; + + off_t Nx, Ny; + if (!gfits_scan (srcheader, "ZNAXIS1", OFF_T_FMT, 1, &Nx)) ESCAPE; + if (!gfits_scan (srcheader, "ZNAXIS2", OFF_T_FMT, 1, &Ny)) ESCAPE; + myAssert (Nx == tgtheader->Naxis[0], "table definition error?"); + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum1 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + // this allocates the pointer data array for the full set of tiles (NOT the heap) + char *tmpbuffer = NULL; + ALLOCATE_ZERO (tmpbuffer, char, max_width*Ny); + for (i = 0; i < Nfields; i++) { + if (!gfits_set_bintable_column (tgtheader, tgttable, fields[i].ttype, tmpbuffer, Ny)) ESCAPE; + } + free (tmpbuffer); + + int Ntile = srcheader->Naxis[1]; + int ztilelast; + if (Ntile == 0) { + ztilelast = 0; + } else { + ztilelast = (tgtheader->Naxis[1] % ztilelen) ? tgtheader->Naxis[1] % ztilelen : ztilelen; + } + + // allocate the intermediate storage buffers + int Nraw_alloc = max_width*ztilelen + 1000; + ALLOCATE (raw, char, Nraw_alloc); + ALLOCATE (zdata, char, max_width*ztilelen); + + // uncompress the data : extract a tile's compressed data, uncompress the tile, insert into full table + // each tile -> 1 row of the output table + + gfits_dump_cmp_table (srctable, "unc"); + + // XXX TIMER 2 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + off_t row; + for (row = 0; row < srcheader->Naxis[1]; row++) { + + for (i = 0; i < Nfields; i++) { + + // gfits_uncompress_data can take values specific to the compression mode + // optname, optvalue = NULL, Noptions = 0 + + // XXX note that this function currently byteswaps. I should redo the code to make + // byteswap of the varlength data segment a separate function + + gettimeofday (&startTimer, (void *) NULL); + + off_t Nzdata; + char *zdata = gfits_varlength_column_pointer (srctable, &fields[i].zdef, row, &Nzdata); + if (!zdata) ESCAPE; + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "u4: "); + for (k = 0; k < Nzdata; k++) { + fprintf (stderr, "%02hhx", zdata[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + // XXX TIMER 2a + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2a += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (strcasecmp(fields[i].zctype, "NONE_2") && + strcasecmp(fields[i].zctype, "GZIP_1") && + strcasecmp(fields[i].zctype, "GZIP_2") && + strcasecmp(fields[i].zctype, "RICE_1") && + strcasecmp(fields[i].zctype, "RICE_ONE")) { + // Nzdata is number of bytes + if (!gfits_byteswap_zdata (zdata, Nzdata, fields[i].pixsize)) ESCAPE; + } + + // XXX TIMER 2b + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2b += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "u3: "); + for (k = 0; k < Nzdata; k++) { + fprintf (stderr, "%02hhx", zdata[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + // XXX TIMER 2c + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2c += dtime; + gettimeofday (&startTimer, (void *) NULL); + + int Nrows = (row == Ntile - 1) ? ztilelast : ztilelen; + int Nraw = Nrows*fields[i].Nvalues*fields[i].pixsize; // expected number of bytes + if (!gfits_uncompress_data (zdata, Nzdata, fields[i].zctype, NULL, NULL, 0, raw, &Nraw, Nraw_alloc, fields[i].pixsize)) ESCAPE; + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "u2: "); + for (k = 0; k < Nraw*fields[i].pixsize; k++) { + fprintf (stderr, "%02hhx", raw[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + // XXX TIMER 2d + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2d += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (!strcasecmp(fields[i].zctype, "GZIP_1")) { + myAssert ((fields[i].zdef.format != 'C') && (fields[i].zdef.format != 'M'), "swap is probably wrong for C or M columns"); + if (!gfits_byteswap_zdata (raw, Nraw * fields[i].pixsize, fields[i].pixsize)) ESCAPE; + } + + // XXX TIMER 2e + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2e += dtime; + gettimeofday (&startTimer, (void *) NULL); + + if (VERBOSE_DUMP) { + int k; + fprintf (stderr, "u1: "); + for (k = 0; k < Nraw*fields[i].pixsize; k++) { + fprintf (stderr, "%02hhx", raw[k]); + if (k % 2) fprintf (stderr, " "); + // if (k % 32 == 31) fprintf (stderr, "\n"); + } + fprintf (stderr, "\n"); + } + + // int valid = (row == 0) && (i == 0); + + off_t row_start = row*ztilelen; + // ^- first row for this tile & field + // NOTE: assumes each tile has the same length, ex the last (true for now) + + // copy the raw pixels from their native matrix locations to the temporary output buffer + if (!strcasecmp(fields[i].zctype, "GZIP_2") || !strcasecmp(fields[i].zctype, "NONE_2")) { + if (!gfits_distribute_table_gzp2 (tgttable, &fields[i], raw, row_start, Nrows)) ESCAPE; + } else { + if (!gfits_distribute_table_data (tgttable, &fields[i], raw, row_start, Nrows)) ESCAPE; + } + + // XXX TIMER 2f + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2f += dtime; + gettimeofday (&startTimer, (void *) NULL); + } + } + + // XXX TIMER 3 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum3 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + gfits_dump_raw_table (tgttable, "unc"); + + free (raw); + free (zdata); + return (TRUE); + +escape: + FREE (raw); + FREE (zdata); + return FALSE; +} + +# define VERBOSE 0 + +// raw_pixsize is the bytes / pixel for the input matrix +// place the raw image bytes for the current tile into the tile buffer +int gfits_distribute_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + + char *tblbuffer = &table->buffer[Nx*row_start + field->offset]; + + int rowsize = field->rowsize; + + if (VERBOSE) fprintf (stderr, "distribute: "); + for (i = 0; i < Nrows; i++, tblbuffer += Nx) { + memcpy (tblbuffer, &raw[i*rowsize], rowsize); +# if (VERBOSE) + int j; for (j = 0; j < field->rowsize; j++) fprintf (stderr, "0x%02hhx ", table->buffer[Nx*row + field->offset + j]); +# endif + } + if (VERBOSE) fprintf (stderr, "\n"); + + return (TRUE); +} + +int gfits_distribute_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i, j, k; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + int Nvalues = field->Nvalues; + int pixsize = field->pixsize; + int offset = field->offset; + + char *rawptr = raw; + + for (k = 0; k < field->pixsize; k++) { +# ifdef BYTE_SWAP + char *tblptr_start = &table->buffer[Nx*row_start + offset + (pixsize - k - 1)]; +# else + char *tblptr_start = &table->buffer[Nx*row_start + offset + k]; +# endif + for (i = 0; i < Nrows; i++, tblptr_start += Nx) { + char *tblptr = tblptr_start; + for (j = 0; j < Nvalues; j++, tblptr += pixsize, rawptr++) { + *tblptr = *rawptr; + } + } + } + return (TRUE); +} + +int gfits_distribute_table_gzp2_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) { + + off_t i, j, k; + + // we are copying NN rows into the column which starts at XX and has MM bytes per row + + off_t Nx = table->header->Naxis[0]; + + for (k = 0; k < field->pixsize; k++) { + for (i = 0; i < Nrows; i++) { + int row = row_start + i; + char *rawptr = &raw[i*field->Nvalues + k*Nrows*field->Nvalues]; +# ifdef BYTE_SWAP + char *tblptr = &table->buffer[Nx*row + field->offset + (field->pixsize - k - 1)]; +# else + char *tblptr = &table->buffer[Nx*row + field->offset + k]; +# endif + for (j = 0; j < field->Nvalues; j++, tblptr += field->pixsize, rawptr++) { + *tblptr = *rawptr; + } + } + } + return (TRUE); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_T.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_T.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_T.c (revision 38447) @@ -14,4 +14,5 @@ status = fseeko (f, 0LL, SEEK_END); /* write table to end of file! */ + if (status) { perror ("fseeko: "); return (FALSE); } status = gfits_fwrite_table (f, table); @@ -26,6 +27,5 @@ Nbytes = fwrite (table[0].buffer, sizeof(char), table[0].datasize, f); - - if (Nbytes != table[0].datasize) return (FALSE); + if (Nbytes != table[0].datasize) { perror ("fwrite: "); return (FALSE); } return (TRUE); } @@ -45,4 +45,5 @@ /* file pointer is at beginning of desired table data */ start = ftello (f); + if (start < 0) { perror ("ftello: "); return FALSE; } for (i = 0; i < Nrow; i++) { @@ -50,5 +51,5 @@ fseeko (f, offset, SEEK_SET); Nbytes = fwrite (table[0].buffer[i], sizeof (char), Nx, f); - if (Nbytes != Nx) { return (FALSE); } + if (Nbytes != Nx) { perror ("fwrite: "); return (FALSE); } } @@ -58,7 +59,9 @@ offset = start + Nx*Ny; - fseeko (f, offset, SEEK_SET); + int status = fseeko (f, offset, SEEK_SET); + if (status) { perror ("fseeko: "); return FALSE; } + Nbytes = fwrite (pad, sizeof (char), Npad, f); - if (Nbytes != Npad) { return (FALSE); } + if (Nbytes != Npad) { perror ("fwrite: "); return (FALSE); } free (pad); @@ -84,4 +87,5 @@ Nx = ftable[0].header[0].Naxis[0]; // final output table size on disk ftable[0].datasize = gfits_data_size (ftable[0].header); + ftable[0].heap_start = gfits_heap_start (ftable[0].header); Nskip = start * Nx; @@ -96,4 +100,5 @@ // cursor must be at start of the table (after table header) if (fseeko (f, Nskip, SEEK_CUR)) { + perror ("fseeko: "); fprintf (stderr, "can't seek table start\n"); return (FALSE); @@ -101,7 +106,5 @@ Nwrite = fwrite (ftable[0].buffer, sizeof (char), Nbytes, f); - if (Nwrite != Nbytes) { - return (FALSE); - } + if (Nwrite != Nbytes) { perror ("fwrite: "); return (FALSE); } if (Ntotal >= Ndisk) { @@ -112,5 +115,5 @@ free (pad); - if (Nbytes != Npad) return (FALSE); + if (Nbytes != Npad) { perror ("fwrite: "); return (FALSE); } } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_TH.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_TH.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/table/F_write_TH.c (revision 38447) @@ -14,4 +14,5 @@ status = fseeko (f, 0LL, SEEK_END); /* write header to end of file! */ + if (status) { perror ("fseeko: "); return (FALSE); } status = gfits_fwrite_Theader (f, header); @@ -26,6 +27,6 @@ Nbytes = fwrite (header[0].buffer, sizeof(char), header[0].datasize, f); + if (Nbytes != header[0].datasize) { perror ("fwrite: "); return (FALSE); } - if (Nbytes != header[0].datasize) return (FALSE); return (TRUE); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/compress.sh =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/compress.sh (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/compress.sh (revision 38447) @@ -0,0 +1,446 @@ +input tap.dvo + +macro tests + test_image_all + test_table_all_rnd +end + +macro test_image_all + + tapPLAN 88 + + foreach mode NONE NONE_1 NONE_2 GZIP_1 GZIP_2 RICE_1 RICE_ONE + # foreach mode GZIP_1 + # foreach mode GZIP_2 + # foreach mode RICE_1 + foreach bitpix 8 16 32 -32 -64 + # foreach bitpix 32 + if (($mode == RICE_1) && ($bitpix < 0)) continue ; # RICE_1 is for int only + if (($mode == RICE_ONE) && ($bitpix < 0)) continue ; # RICE_1 is for int only + tapDIAG 2 "===== bitpix = $bitpix, mode = $mode =====" + test_image_single $bitpix $mode + end + end + + tapDONE +end + +macro test_image_single + if ($0 != 3) + echo "USAGE: test_image_single (bitpix) (mode)" + break + end + + local bitpix cmpmode + $bitpix = $1 + $cmpmode = $2 + + delete -q x z b c d e + + mcreate z 8 8 + set x = xramp(z) + wd x test.raw.fits -bitpix $bitpix -bzero 0 -bscale 1 + wd x test.cmp.fits -bitpix $bitpix -bzero 0 -bscale 1 -compress-mode $cmpmode + rd b test.cmp.fits -x 0 + set d = x - b + stat -q d + tapOK {abs($MEAN) < 0.001} "read our cmp $cmpmode $bitpix (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read our cmp $cmpmode $bitpix (SIGMA = $SIGMA)" + + if ($cmpmode == NONE) return + if ($cmpmode == NONE_1) return + if ($cmpmode == NONE_2) return + + if ($cmpmode == GZIP_1) + exec fpack -q 0 -g -S test.raw.fits > test.fpk.fits + end + if ($cmpmode == GZIP_2) + exec fpack -q 0 -g2 -S test.raw.fits > test.fpk.fits + end + if ($cmpmode == RICE_1) + exec fpack -r -S test.raw.fits > test.fpk.fits + end + # echo "load fpack version" + rd c test.fpk.fits -x 0 + set d = x - c + stat -q d + tapOK {abs($MEAN) < 0.001} "read fpack $cmpmode $bitpix (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read fpack $cmpmode $bitpix (SIGMA = $SIGMA)" + + if ($cmpmode == RICE_1_0) + echo "*** funpack fails on RICE_1 ****" + return + end + + # echo "run funpack on our version" + exec funpack -S test.cmp.fits > test.fun.fits + rd e test.fun.fits + set d = x - e + stat -q d + tapOK {abs($MEAN) < 0.001} "funpack our cmp $cmpmode $bitpix (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "funpack our cmp $cmpmode $bitpix (SIGMA = $SIGMA)" +end + +macro test_table_single_mode + if ($0 != 2) + echo "USAGE: test_table_single_mode (mode)" + break + end + + local cmpmode + $cmpmode = $1 + + delete -q x y ID + + $NPT = 10 + create x 0 $NPT + set y = x^2 + create ID 1 {$NPT + 1} -int + + $fields = ID x y + $format = IEE + + write -fits test test.raw.tbl $fields -format $format + write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode + + foreach f $fields + set $f\_raw = $f + delete $f + end + + echo "===== raw =====" + data test.raw.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat dv + end + delete -q $fields + + echo "===== cmp =====" + data test.cmp.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat dv + end +end + +macro test_table_all_single_column + + tapPLAN 88 + + local format cmpmode + + # foreach cmpmode NONE GZIP_1 GZIP_2 RICE_1 RICE_ONE + foreach cmpmode GZIP_2 + foreach format B I J K E D + if (($cmpmode == RICE_1) && ($format == E)) continue ; # RICE_1 is for int only + if (($cmpmode == RICE_ONE) && ($format == E)) continue ; # RICE_1 is for int only + if (($cmpmode == RICE_1) && ($format == D)) continue ; # RICE_1 is for int only + if (($cmpmode == RICE_ONE) && ($format == D)) continue ; # RICE_1 is for int only + if (($cmpmode == RICE_1) && ($format == K)) continue ; # RICE_1 is for int only + if (($cmpmode == RICE_ONE) && ($format == K)) continue ; # RICE_1 is for int only + test_table_single_column $cmpmode $format + end + end + tapDONE +end + +macro test_table_single_column + if ($0 != 3) + echo "USAGE: test_table_single_column (mode) (format)" + break + end + + local cmpmode format + $cmpmode = $1 + $format = $2 + + tapDIAG 2 "===== testing $cmpmode $format =====" + + delete -q x dv + + $NPT = 10000 + + if ($format == B) + create x -128 128 + else + create x 0 $NPT + end + + $fields = x + + write -fits test test.raw.tbl $fields -format $format + write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode + + foreach f $fields + set $f\_raw = $f + delete $f + end + + tapDIAG 2 "===== $cmpmode $format raw =====" + data test.raw.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read our raw $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read our raw $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields + + tapDIAG 2 "===== $cmpmode $format cmp =====" + data test.cmp.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read our cmp $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read our cmp $cmpmode $format (SIGMA = $SIGMA)" + end + + if ($cmpmode == NONE) return; # funpack will not recognize NONE + if ($cmpmode == RICE_ONE) return; # funpack will not recognize RICE_ONE for tables + + tapDIAG 2 "===== $cmpmode $format fpack =====" + + # try to run fpack on ours + if (1) + tapEXEC /bin/cp -f test.raw.tbl test.fpk.tbl + tapEXEC fpack -table -F test.fpk.tbl + + data test.fpk.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read fpack $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read fpack $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields + end + + break -auto off + echo $cmpmode $format + exec ls test.cmp.tbl test.fpk.tbl | fields -x 0 PCOUNT ZCTYP1 + break -auto on + + tapDIAG 2 "===== $cmpmode $format funpack =====" + + # try to run fpack on ours + tapEXEC /bin/cp -f test.cmp.tbl test.fun.tbl + tapEXEC funpack -F test.fun.tbl + + data test.fun.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read funpack $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read funpack $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields +end + +macro test_table_all_multi_column + + local format cmpmode + + foreach cmpmode NONE GZIP_1 + foreach format B I J K E D + test_table_multi_column $cmpmode $format + end + end +end + +macro test_table_multi_column + if ($0 != 3) + echo "USAGE: test_table_multi_column (mode) (format)" + break + end + + local cmpmode + $cmpmode = $1 + + delete -q x y z + + $NPT = 10000 + create x 0 $NPT + set y = x^2 + set z = 0.01*x + + local format + + $fields = x y z + $format = $2\$2\$2 + + $fields = x y + $format = $2\$2 + + write -fits test test.raw.tbl $fields -format $format + write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode + + foreach f $fields + set $f\_raw = $f + delete $f + end + + echo "===== $cmpmode $format raw =====" + data test.raw.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat dv + end + delete -q $fields + + echo "===== $cmpmode $format cmp =====" + data test.cmp.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat dv + end +end + +macro test_table_all_rnd + + local format cmpmode + + foreach cmpmode NONE NONE_1 NONE_2 GZIP_1 GZIP_2 RICE_1 RICE_ONE + test_table_rnd $cmpmode + end +end + +macro test_table_rnd + if ($0 != 2) + echo "USAGE: test_table_rnd (mode)" + break + end + + local cmpmode + $cmpmode = $1 + + tapDIAG 2 "===== testing $cmpmode =====" + + delete -q Achar Ashort Aint Along + delete -q Ffloat Fdouble + + # lrnd generates rnd int between 0 and 0x7fffff (2^31 - 1 inclusive) + + $NPT = 1000000 + # $NPT = 30 + create ni 0 $NPT -int + create nf 0 $NPT + + # set Achar = lrnd(ni) & 0x7f + set Achar = int(255.999*(drnd(ni) - 0.5)) + set Ashort = lrnd(ni) & 0x7fff + set Aint = lrnd(ni) + + # I cannot represent the full dynamic range of 64 bit with mana vector ints, but + # I can with the (double) floating point vector. however, I get the wrong bits + set Along_dbl = 9.223371e18 * (2.0*drnd(nf) - 1.0); + + cast Along = Along_dbl as int64 + + set Ffloat_dbl = 3.402822e+38 * (2.0*drnd(nf) - 1.0) + cast Ffloat = Ffloat_dbl as single + set Fdouble = 1.797692e+308 * (2.0*drnd(nf) - 1.0) + + local format + + $fields = Achar Ashort Aint Along Ffloat Fdouble + $format = BIJKED + + #$fields = Along + #$format = K + + write -fits test test.raw.tbl $fields -format $format + write -fits test test.cmp.tbl $fields -format $format -compress-mode $cmpmode + + foreach f $fields + set $f\_raw = $f + delete $f + end + + tapDIAG 2 "===== $cmpmode $format raw =====" + data test.raw.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read our raw $f $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read our raw $f $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields + + tapDIAG 2 "===== $cmpmode $format cmp =====" + data test.cmp.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read our cmp $f $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read our cmp $f $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields + + tapDIAG 2 "===== $cmpmode $format fpack =====" + + # try to run fpack on ours + if (1) + tapEXEC /bin/cp -f test.raw.tbl test.fpk.tbl + tapEXEC fpack -table -F test.fpk.tbl + + data test.fpk.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read fpack $f $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read fpack $f $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields + end + + break -auto off + echo $cmpmode $format + exec ls test.cmp.tbl test.fpk.tbl | fields -x 0 PCOUNT ZCTYP1 ZCTYP2 ZCTYP3 ZCTYP4 ZCTYP5 ZCTYP6 + break -auto on + + if ($cmpmode == NONE) return; # funpack will not recognize NONE + if ($cmpmode == NONE_1) return; # funpack will not recognize NONE + if ($cmpmode == NONE_2) return; # funpack will not recognize NONE + if ($cmpmode == RICE_ONE) return; # funpack will not recognize RICE_ONE for tables + + tapDIAG 2 "===== $cmpmode $format funpack =====" + + # try to run fpack on ours + tapEXEC /bin/cp -f test.cmp.tbl test.fun.tbl + tapEXEC funpack -F test.fun.tbl + + data test.fun.tbl + read -fits test $fields + + foreach f $fields + set dv = $f - $f\_raw + vstat -q dv + tapOK {abs($MEAN) < 0.001} "read funpack $f $cmpmode $format (MEAN = $MEAN)" + tapOK {$SIGMA < 0.001} "read funpack $f $cmpmode $format (SIGMA = $SIGMA)" + end + delete -q $fields +end + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/imagecomp.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/imagecomp.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/imagecomp.c (revision 38447) @@ -0,0 +1,325 @@ +# include +# include +# include "tap_ohana.h" + +int test_compress (int bitpix, char *zcmptype); +int test_compress_fullrange (int bitpix, char *zcmptype); +int test_compress_fulltile (int bitpix, char *zcmptype); + +// char *cmptype[] = {"NONE", "NONE_2", "GZIP_1", "GZIP_2", "PLIO_1", "RICE_1", "RICE_ONE", "HCOMPRESS_1", NULL}; +char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", NULL}; +int bitpix[] = {8, 16, 32, -32, -64, 0}; + +//char *cmptype[] = {"GZIP_1", NULL}; +//int bitpix[] = {16, 0}; + +static int NX = 10; +static int NY = 10; + +int main (int argc, char **argv) { + + plan_tests (3*3*234); + + diag ("libfits imagecomp.c tests"); + + int i, j; + for (i = 0; cmptype[i]; i++) { + for (j = 0; bitpix[j]; j++) { + if (!strcasecmp (cmptype[i], "RICE_1") && (bitpix[j] < 0)) continue; + if (!strcasecmp (cmptype[i], "RICE_ONE") && (bitpix[j] < 0)) continue; + test_compress (bitpix[j], cmptype[i]); + test_compress_fulltile (bitpix[j], cmptype[i]); + test_compress_fullrange (bitpix[j], cmptype[i]); + } + } + + NX = 100; NY = 33; + for (i = 0; cmptype[i]; i++) { + for (j = 0; bitpix[j]; j++) { + if (!strcasecmp (cmptype[i], "RICE_1") && (bitpix[j] < 0)) continue; + if (!strcasecmp (cmptype[i], "RICE_ONE") && (bitpix[j] < 0)) continue; + test_compress (bitpix[j], cmptype[i]); + test_compress_fulltile (bitpix[j], cmptype[i]); + test_compress_fullrange (bitpix[j], cmptype[i]); + } + } + + NX = 357; NY = 245; + for (i = 0; cmptype[i]; i++) { + for (j = 0; bitpix[j]; j++) { + if (!strcasecmp (cmptype[i], "RICE_1") && (bitpix[j] < 0)) continue; + if (!strcasecmp (cmptype[i], "RICE_ONE") && (bitpix[j] < 0)) continue; + test_compress (bitpix[j], cmptype[i]); + test_compress_fulltile (bitpix[j], cmptype[i]); + test_compress_fullrange (bitpix[j], cmptype[i]); + } + } + return exit_status(); +} + +int test_compress (int bitpix, char *zcmptype) { // make a table, compress, uncompress, compare : use compression zcmptype + + Header rawheader; + Matrix rawmatrix; + Header outheader; + Matrix outmatrix; + + Header theader; + FTable ftable; + ftable.header = &theader; + + diag ("--- starting test_compress with bitpix %d, zcmptype %s ---", bitpix, zcmptype); + ok (gfits_init_header (&rawheader), "init'ed the raw header"); + ok (gfits_init_matrix (&rawmatrix), "init'ed the raw matrix"); + ok (gfits_init_header (&outheader), "init'ed the out header"); + ok (gfits_init_matrix (&outmatrix), "init'ed the out matrix"); + + /* assign the necessary internal values */ + rawheader.bitpix = bitpix; + rawheader.Naxes = 2; + rawheader.Naxis[0] = NX; + rawheader.Naxis[1] = NY; + + /* create the appropriate header and matrix */ + ok (gfits_create_header (&rawheader), "created header"); + ok (gfits_create_matrix (&rawheader, &rawmatrix), "created matrix"); + + char *rawdata_char = (char *) rawmatrix.buffer; + short *rawdata_short = (short *) rawmatrix.buffer; + int *rawdata_int = (int *) rawmatrix.buffer; + float *rawdata_float = (float *) rawmatrix.buffer; + double *rawdata_double = (double *) rawmatrix.buffer; + + int ix, iy; + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + int IY = iy + 1; + switch (bitpix) { + case 8: rawdata_char [ix + NX*iy] = ix + IY*IY; break; + case 16: rawdata_short [ix + NX*iy] = ix + IY*IY; break; + case 32: rawdata_int [ix + NX*iy] = ix + IY*IY; break; + case -32: rawdata_float [ix + NX*iy] = ix + IY*IY; break; + case -64: rawdata_double[ix + NX*iy] = ix + IY*IY; break; + default: myAbort ("bad bitpix value"); + } + } + } + + // int Ztile[2] = {NX, NY}; + // ok (gfits_compress_image (&rawheader, &rawmatrix, &ftable, (int *) &Ztile, zcmptype), "compressed image"); + ok (gfits_compress_image (&rawheader, &rawmatrix, &ftable, NULL, zcmptype), "compressed image"); + ok (gfits_uncompress_image (&outheader, &outmatrix, &ftable), "uncompressed image"); + + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int Nbad = 0; + + char *outdata_char = (char *) outmatrix.buffer; + short *outdata_short = (short *) outmatrix.buffer; + int *outdata_int = (int *) outmatrix.buffer; + float *outdata_float = (float *) outmatrix.buffer; + double *outdata_double = (double *) outmatrix.buffer; + + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + switch (bitpix) { + case 8: if (rawdata_char [ix + NX*iy] != outdata_char [ix + NX*iy]) Nbad ++; break; + case 16: if (rawdata_short [ix + NX*iy] != outdata_short [ix + NX*iy]) Nbad ++; break; + case 32: if (rawdata_int [ix + NX*iy] != outdata_int [ix + NX*iy]) Nbad ++; break; + case -32: if (rawdata_float [ix + NX*iy] != outdata_float [ix + NX*iy]) Nbad ++; break; + case -64: if (rawdata_double[ix + NX*iy] != outdata_double[ix + NX*iy]) Nbad ++; break; + default: myAbort ("bad bitpix value"); + } + } + } + + ok (!Nbad, "all image pixels match"); + + gfits_free_header (&rawheader); + gfits_free_matrix (&rawmatrix); + + gfits_free_header (&outheader); + gfits_free_matrix (&outmatrix); + return TRUE; +} + +int test_compress_fulltile (int bitpix, char *zcmptype) { // make a table, compress, uncompress, compare : use compression zcmptype + + Header rawheader; + Matrix rawmatrix; + Header outheader; + Matrix outmatrix; + + Header theader; + FTable ftable; + ftable.header = &theader; + + diag ("--- starting test_compress with bitpix %d, zcmptype %s ---", bitpix, zcmptype); + ok (gfits_init_header (&rawheader), "init'ed the raw header"); + ok (gfits_init_matrix (&rawmatrix), "init'ed the raw matrix"); + ok (gfits_init_header (&outheader), "init'ed the out header"); + ok (gfits_init_matrix (&outmatrix), "init'ed the out matrix"); + + /* assign the necessary internal values */ + rawheader.bitpix = bitpix; + rawheader.Naxes = 2; + rawheader.Naxis[0] = NX; + rawheader.Naxis[1] = NY; + + /* create the appropriate header and matrix */ + ok (gfits_create_header (&rawheader), "created header"); + ok (gfits_create_matrix (&rawheader, &rawmatrix), "created matrix"); + + char *rawdata_char = (char *) rawmatrix.buffer; + short *rawdata_short = (short *) rawmatrix.buffer; + int *rawdata_int = (int *) rawmatrix.buffer; + float *rawdata_float = (float *) rawmatrix.buffer; + double *rawdata_double = (double *) rawmatrix.buffer; + + int ix, iy; + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + int IY = iy + 1; + switch (bitpix) { + case 8: rawdata_char [ix + NX*iy] = ix + IY*IY; break; + case 16: rawdata_short [ix + NX*iy] = ix + IY*IY; break; + case 32: rawdata_int [ix + NX*iy] = ix + IY*IY; break; + case -32: rawdata_float [ix + NX*iy] = ix + IY*IY; break; + case -64: rawdata_double[ix + NX*iy] = ix + IY*IY; break; + default: myAbort ("bad bitpix value"); + } + } + } + + int Ztile[2] = {NX, NY}; + ok (gfits_compress_image (&rawheader, &rawmatrix, &ftable, (int *) &Ztile, zcmptype), "compressed image"); + // ok (gfits_compress_image (&rawheader, &rawmatrix, &ftable, NULL, zcmptype), "compressed image"); + ok (gfits_uncompress_image (&outheader, &outmatrix, &ftable), "uncompressed image"); + + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int Nbad = 0; + + char *outdata_char = (char *) outmatrix.buffer; + short *outdata_short = (short *) outmatrix.buffer; + int *outdata_int = (int *) outmatrix.buffer; + float *outdata_float = (float *) outmatrix.buffer; + double *outdata_double = (double *) outmatrix.buffer; + + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + switch (bitpix) { + case 8: if (rawdata_char [ix + NX*iy] != outdata_char [ix + NX*iy]) Nbad ++; break; + case 16: if (rawdata_short [ix + NX*iy] != outdata_short [ix + NX*iy]) Nbad ++; break; + case 32: if (rawdata_int [ix + NX*iy] != outdata_int [ix + NX*iy]) Nbad ++; break; + case -32: if (rawdata_float [ix + NX*iy] != outdata_float [ix + NX*iy]) Nbad ++; break; + case -64: if (rawdata_double[ix + NX*iy] != outdata_double[ix + NX*iy]) Nbad ++; break; + default: myAbort ("bad bitpix value"); + } + } + } + + ok (!Nbad, "all image pixels match"); + + gfits_free_header (&rawheader); + gfits_free_matrix (&rawmatrix); + + gfits_free_header (&outheader); + gfits_free_matrix (&outmatrix); + return TRUE; +} + +int test_compress_fullrange (int bitpix, char *zcmptype) { // make a table, compress, uncompress, compare : use compression zcmptype + + Header rawheader; + Matrix rawmatrix; + Header outheader; + Matrix outmatrix; + + Header theader; + FTable ftable; + ftable.header = &theader; + + diag ("--- starting test_compress with bitpix %d, zcmptype %s ---", bitpix, zcmptype); + ok (gfits_init_header (&rawheader), "init'ed the raw header"); + ok (gfits_init_matrix (&rawmatrix), "init'ed the raw matrix"); + ok (gfits_init_header (&outheader), "init'ed the out header"); + ok (gfits_init_matrix (&outmatrix), "init'ed the out matrix"); + + /* assign the necessary internal values */ + rawheader.bitpix = bitpix; + rawheader.Naxes = 2; + rawheader.Naxis[0] = NX; + rawheader.Naxis[1] = NY; + + /* create the appropriate header and matrix */ + ok (gfits_create_header (&rawheader), "created header"); + ok (gfits_create_matrix (&rawheader, &rawmatrix), "created matrix"); + + char *rawdata_char = (char *) rawmatrix.buffer; + short *rawdata_short = (short *) rawmatrix.buffer; + int *rawdata_int = (int *) rawmatrix.buffer; + float *rawdata_float = (float *) rawmatrix.buffer; + double *rawdata_double = (double *) rawmatrix.buffer; + + long A = time(NULL); + srand48(A); + // srand48(1); + + int ix, iy; + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + switch (bitpix) { + case 8: rawdata_char [ix + NX*iy] = 0xff & lrand48(); break; + case 16: rawdata_short [ix + NX*iy] = 0xffff & lrand48(); break; + case 32: rawdata_int [ix + NX*iy] = 0xffffffff & lrand48(); break; + case -32: rawdata_float [ix + NX*iy] = FLT_MAX * (2.0*drand48() - 1.0); break; + case -64: rawdata_double[ix + NX*iy] = DBL_MAX * (2.0*drand48() - 1.0); break; + default: myAbort ("bad bitpix value"); + } + } + } + + // int Ztile[2] = {NX, NY}; + ok (gfits_compress_image (&rawheader, &rawmatrix, &ftable, NULL, zcmptype), "compressed image"); + + ok (gfits_uncompress_image (&outheader, &outmatrix, &ftable), "uncompressed image"); + + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int Nbad = 0; + + char *outdata_char = (char *) outmatrix.buffer; + short *outdata_short = (short *) outmatrix.buffer; + int *outdata_int = (int *) outmatrix.buffer; + float *outdata_float = (float *) outmatrix.buffer; + double *outdata_double = (double *) outmatrix.buffer; + + for (ix = 0; ix < NX; ix++) { + for (iy = 0; iy < NY; iy++) { + switch (bitpix) { + case 8: if (rawdata_char [ix + NX*iy] != outdata_char [ix + NX*iy]) Nbad ++; break; + case 16: if (rawdata_short [ix + NX*iy] != outdata_short [ix + NX*iy]) Nbad ++; break; + case 32: if (rawdata_int [ix + NX*iy] != outdata_int [ix + NX*iy]) Nbad ++; break; + case -32: if (rawdata_float [ix + NX*iy] != outdata_float [ix + NX*iy]) Nbad ++; break; + case -64: if (rawdata_double[ix + NX*iy] != outdata_double[ix + NX*iy]) Nbad ++; break; + default: myAbort ("bad bitpix value"); + } + } + } + + ok (!Nbad, "all image pixels match"); + + gfits_free_header (&rawheader); + gfits_free_matrix (&rawmatrix); + + gfits_free_header (&outheader); + gfits_free_matrix (&outmatrix); + return TRUE; +} + + +// fprintf (stderr, "%2d %2d : 0x%08x vs 0x%08x vs 0x%08x : %d\n", ix, iy, rawdata_int [ix + NX*iy], outdata_int [ix + NX*iy], tmpbuff[ix + NX*iy], outdata_int [ix + NX*iy] - rawdata_int [ix + NX*iy]); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/ricetest.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/ricetest.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/ricetest.c (revision 38447) @@ -0,0 +1,98 @@ +# include +# include +# include "tap_ohana.h" + +# define NPIX 100 +# define NBYTE 4*NPIX + +int main (int argc, char **argv) { + + plan_tests (45); + + diag ("libfits ricetest.c tests"); + + // buffers to store max array + char rawdata[NBYTE]; + char cmpdata[NBYTE]; + char outdata[NBYTE]; + + // ok (1, "failure"); + + if (1) { + char *rawvalue = (char *) rawdata; + char *outvalue = (char *) outdata; + + int j; + for (j = 0; j < 5; j++) { + int i; + for (i = 0; i < NPIX; i++) { + rawvalue[i] = i; + } + + int Ncmp = fits_rcomp_byte (rawvalue, NPIX, cmpdata, NBYTE, 32); + ok (Ncmp > 0, "compressed byte data"); + + int status = fits_rdecomp_byte (cmpdata, Ncmp, outdata, NPIX, 32); + ok (!status, "decompressed byte data"); + + int Nbad = 0; + for (i = 0; i < NPIX; i++) { + if (rawvalue[i] != outvalue[i]) Nbad ++; + } + + ok (!Nbad, "values match"); + } + } + + if (1) { + short *rawvalue = (short *) rawdata; + short *outvalue = (short *) outdata; + + int j; + for (j = 0; j < 5; j++) { + int i; + for (i = 0; i < NPIX; i++) { + rawvalue[i] = i; + } + + int Ncmp = fits_rcomp_short (rawvalue, NPIX, cmpdata, NBYTE, 32); + ok (Ncmp > 0, "compressed short data"); + + int status = fits_rdecomp_short (cmpdata, Ncmp, outdata, NPIX, 32); + ok (!status, "decompressed short data"); + + int Nbad = 0; + for (i = 0; i < NPIX; i++) { + if (rawvalue[i] != outvalue[i]) Nbad ++; + } + + ok (!Nbad, "values match"); + } + } + + if (1) { + int *rawvalue = (int *) rawdata; + int *outvalue = (int *) outdata; + + int j; + for (j = 0; j < 5; j++) { + int i; + for (i = 0; i < NPIX; i++) { + rawvalue[i] = i; + } + + int Ncmp = fits_rcomp (rawvalue, NPIX, cmpdata, NBYTE, 32); + ok (Ncmp > 0, "compressed int data"); + + int status = fits_rdecomp (cmpdata, Ncmp, outdata, NPIX, 32); + ok (!status, "decompressed int data"); + + int Nbad = 0; + for (i = 0; i < NPIX; i++) { + if (rawvalue[i] != outvalue[i]) Nbad ++; + } + ok (!Nbad, "values match"); + } + } + return exit_status(); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tablecomp.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tablecomp.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tablecomp.c (revision 38447) @@ -0,0 +1,453 @@ +# include +# include +# include "tap_ohana.h" + +int test_compress (char *zcmptype); +int test_compress_empty (char *zcmptype); +int test_compress_fullrange (char *zcmptype); +int test_compress_single_full (char *zcmptype); + +char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", "AUTO", NULL}; + +int main (int argc, char **argv) { + + plan_tests (668); + + diag ("libfits tablecomp.c tests"); + + // test_compress_empty ("GZIP_1"); + // exit (0); + + test_compress (NULL); + test_compress_fullrange (NULL); + + int i; + for (i = 0; cmptype[i]; i++) { + test_compress (cmptype[i]); + test_compress_fullrange (cmptype[i]); + test_compress_empty (cmptype[i]); + ok (ohana_memcheck_func (TRUE), "no memory corruption"); + } + + return exit_status(); +} + +int test_compress (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte", "none", 1.0, 0.0), "defined byte column"); + ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long", "none", 1.0, 0.0), "defined long column"); + ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + int Nval = 1000; + char *VAL_B; ALLOCATE (VAL_B, char, Nval); + short *VAL_I; ALLOCATE (VAL_I, short, Nval); + int *VAL_J; ALLOCATE (VAL_J, int, Nval); + int64_t *VAL_K; ALLOCATE (VAL_K, int64_t, Nval); + float *VAL_E; ALLOCATE (VAL_E, float, Nval); + double *VAL_D; ALLOCATE (VAL_D, double, Nval); + + int i; + for (i = 0; i < Nval; i++) { + VAL_B[i] = (i % 255) - 128; + VAL_I[i] = i; + VAL_J[i] = 10000*i + 100*i; + VAL_K[i] = 10000*i + 330*i; + VAL_E[i] = 0.1*i; + VAL_D[i] = 1.001*i; + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, 10, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + char *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"), "read byte table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + short *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + float *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + double *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // count mismatched values + int NVAL_B_bad = 0; + int NVAL_I_bad = 0; + int NVAL_J_bad = 0; + int NVAL_K_bad = 0; + int NVAL_E_bad = 0; + int NVAL_D_bad = 0; + + // for (i = 0; i < Nval; i++) { + // fprintf (stderr, "%0llx : %0llx\n", (long long) VAL_K[i], (long long) VAL_K_t[i]); + // } + + for (i = 0; i < Nval; i++) { + if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++; + if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++; + if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++; + if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++; + if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++; + if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++; + } + + ok (!NVAL_B_bad, "byte values match (input vs output)"); + ok (!NVAL_I_bad, "short values match (input vs output)"); + ok (!NVAL_J_bad, "int values match (input vs output)"); + ok (!NVAL_K_bad, "int64_t values match (input vs output)"); + ok (!NVAL_E_bad, "float values match (input vs output)"); + ok (!NVAL_D_bad, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + +int test_compress_fullrange (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte", "none", 1.0, 0.0), "defined byte column"); + ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long", "none", 1.0, 0.0), "defined long column"); + ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + int Nval = 1000; + char *VAL_B; ALLOCATE (VAL_B, char, Nval); + short *VAL_I; ALLOCATE (VAL_I, short, Nval); + int *VAL_J; ALLOCATE (VAL_J, int, Nval); + int64_t *VAL_K; ALLOCATE (VAL_K, int64_t, Nval); + float *VAL_E; ALLOCATE (VAL_E, float, Nval); + double *VAL_D; ALLOCATE (VAL_D, double, Nval); + + long A = time(NULL); + srand48(A); + + int i; + for (i = 0; i < Nval; i++) { + VAL_B[i] = 0xff & lrand48(); + VAL_I[i] = 0xffff & lrand48(); + VAL_J[i] = 0xffffffff & lrand48(); + VAL_K[i] = (((int64_t)lrand48()) << 32) + (int64_t)lrand48(); + VAL_E[i] = FLT_MAX * (2.0*drand48() - 1.0); + VAL_D[i] = DBL_MAX * (2.0*drand48() - 1.0); + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, 10, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + char *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"), "read byte table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + short *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + float *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + double *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // count mismatched values + int NVAL_B_bad = 0; + int NVAL_I_bad = 0; + int NVAL_J_bad = 0; + int NVAL_K_bad = 0; + int NVAL_E_bad = 0; + int NVAL_D_bad = 0; + + for (i = 0; i < Nval; i++) { + if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++; + if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++; + if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++; + if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++; + if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++; + if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++; + } + + ok (!NVAL_B_bad, "byte values match (input vs output)"); + ok (!NVAL_I_bad, "short values match (input vs output)"); + ok (!NVAL_J_bad, "int values match (input vs output)"); + ok (!NVAL_K_bad, "int64_t values match (input vs output)"); + ok (!NVAL_E_bad, "float values match (input vs output)"); + ok (!NVAL_D_bad, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + +int test_compress_single_full (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "I", "SEQ", "seq", "none", 1.0, 0.0), "defined short column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + int Nval = 1000; + short *SEQ; ALLOCATE (SEQ, short, Nval); + + int i; + for (i = 0; i < Nval; i++) { + SEQ[i] = i; + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "SEQ", SEQ, Nval), "set short table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, 0, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + short *SEQ_t = gfits_get_bintable_column_data (outheader, outtable, "SEQ", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // count mismatched values + int Nseq = 0; + for (i = 0; i < Nval; i++) { + if (SEQ[i] != SEQ_t[i]) Nseq++; + } + + ok (!Nseq, "short values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + +int test_compress_empty (char *zcmptype) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte", "none", 1.0, 0.0), "defined byte column"); + ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long", "none", 1.0, 0.0), "defined long column"); + ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + int Nval = 0; + char *VAL_B; ALLOCATE (VAL_B, char, Nval); + short *VAL_I; ALLOCATE (VAL_I, short, Nval); + int *VAL_J; ALLOCATE (VAL_J, int, Nval); + int64_t *VAL_K; ALLOCATE (VAL_K, int64_t, Nval); + float *VAL_E; ALLOCATE (VAL_E, float, Nval); + double *VAL_D; ALLOCATE (VAL_D, double, Nval); + + int i; + for (i = 0; i < Nval; i++) { + VAL_B[i] = (i % 255) - 128; + VAL_I[i] = i; + VAL_J[i] = 10000*i + 100*i; + VAL_K[i] = 10000*i + 330*i; + VAL_E[i] = 0.1*i; + VAL_D[i] = 1.001*i; + } + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column"); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, 0, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + char *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"), "read byte table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + short *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + float *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + double *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // count mismatched values + int NVAL_B_bad = 0; + int NVAL_I_bad = 0; + int NVAL_J_bad = 0; + int NVAL_K_bad = 0; + int NVAL_E_bad = 0; + int NVAL_D_bad = 0; + + // for (i = 0; i < Nval; i++) { + // fprintf (stderr, "%0llx : %0llx\n", (long long) VAL_K[i], (long long) VAL_K_t[i]); + // } + + for (i = 0; i < Nval; i++) { + if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++; + if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++; + if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++; + if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++; + if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++; + if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++; + } + + ok (!NVAL_B_bad, "byte values match (input vs output)"); + ok (!NVAL_I_bad, "short values match (input vs output)"); + ok (!NVAL_J_bad, "int values match (input vs output)"); + ok (!NVAL_K_bad, "int64_t values match (input vs output)"); + ok (!NVAL_E_bad, "float values match (input vs output)"); + ok (!NVAL_D_bad, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + return TRUE; +} + + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tap.dvo =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tap.dvo (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tap.dvo (revision 38447) @@ -0,0 +1,116 @@ +# -*-sh-*- + +if (not($?TAP_BREAK)) set TAP_BREAK = 0 +if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 0 + +macro tapEXEC + if ($0 < 3) + echo "USAGE: tapEXEC (command)" + break + end + + local i cmd result + + $cmd = "" + for i 1 $0 + $cmd = $cmd $$i + end + + if ($TAP_VERBOSE >= 2) echo $cmd + + break -auto off + exec $cmd >& tap.log + $result = $STATUS + break -auto on + + if (not($result)) + exec cat tap.log + echo "not ok : $cmd" + $TAP_NFAIL ++ + $TAP_LAST = 0 + if ($TAP_BREAK) + break + end + else + if ($TAP_VERBOSE >= 1) echo "ok : $cmd" + $TAP_LAST = 1 + end + $TAP_NDONE ++ +end + +macro tapOK + if ($0 != 3) + echo "USAGE: tapOK (condition) (message)" + break + end + + if ($1) + if ($TAP_VERBOSE >= 1) + echo "ok : $2" + end + $TAP_LAST = 1 + else + echo "not ok : $2" + $TAP_NFAIL ++ + $TAP_LAST = 0 + if ($TAP_BREAK) + break + end + end + $TAP_NDONE ++ +end + +macro tapPLAN + if ($0 != 2) + echo "USAGE: tapPLAN (Ntests)" + break + end + + $TAP_NTEST = $1 + $TAP_NFAIL = 0 + $TAP_NSKIP = 0 + $TAP_NDONE = 0 + if (not($?TAP_BREAK)) set TAP_BREAK = 0 +end + +macro tapDIAG + if ($0 != 3) + echo "USAGE: tapDIAG (level) (message)" + break + end + + if ($TAP_VERBOSE >= $1) echo "$2" +end + +macro tapSKIP + if ($0 != 2) + echo "USAGE: tapSKIP (nskip)" + break + end + + $TAP_NSKIP += $1 + $TAP_NDONE += $1 +end + +macro tapDONE + if ($0 != 1) + echo "USAGE: tapDONE" + break + end + + if ($TAP_NDONE != $TAP_NTEST) + echo "planned tests ($TAP_NTEST) not equal to done tests ($TAP_NDONE)" + end + + if ($TAP_NFAIL) + echo "failed $TAP_NFAIL of $TAP_NDONE" + end + + if ($TAP_NSKIP) + echo "skipped $TAP_NSKIP of $TAP_NDONE" + end + + if ($TAP_NFAIL + $TAP_NSKIP == 0) + echo "passed $TAP_NDONE tests" + end +end Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tcomptiming.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tcomptiming.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/tcomptiming.c (revision 38447) @@ -0,0 +1,207 @@ +# include +# include +# include "tap_ohana.h" + +int test_compress_timing (char *zcmptype, int Ntile); +void gfits_uncompress_timing (); + +char *cmptype[] = {"NONE", "NONE_1", "NONE_2", "GZIP_1", "GZIP_2", "RICE_1", "RICE_ONE", "AUTO", NULL}; + +static int Nval = 10000000; + +int main (int argc, char **argv) { + + plan_tests (668); + // plan_skip_all ("skipping"); + + diag ("libfits tablecomp.c tests"); + + // test_compress_empty ("GZIP_1"); + // exit (0); + + float lNtileMax = log10(Nval) + 0.1; + float lNtile = 0.0; + for (lNtile = 1.0; lNtile < lNtileMax; lNtile += 1.0) { + int Ntile = pow (10.0, lNtile); + fprintf (stderr, "--- Ntile = %d ---\n", Ntile); + test_compress_timing ("NONE_2", Ntile); + gfits_compress_timing(); + gfits_uncompress_timing(); + } + ok (ohana_memcheck_func (TRUE), "no memory corruption"); + + return exit_status(); +} + +int test_compress_timing (char *zcmptype, int Ntile) { // test 2: make a table, compress, uncompress, compare : use compression "NONE" + + Header header; + FTable ftable; + + diag ("--- starting test_compress with zcmptype %s ---", zcmptype); + ok (gfits_init_header (&header), "inited the header"); + ok (gfits_init_table (&ftable), "inited the table"); + + ok (gfits_create_table_header (&header, "BINTABLE", "TESTDATA"), "created the table header"); + + ok (gfits_define_bintable_column (&header, "B", "VAL_B", "byte", "none", 1.0, 0.0), "defined byte column"); + ok (gfits_define_bintable_column (&header, "I", "VAL_I", "short", "none", 1.0, 0.0), "defined short column"); + ok (gfits_define_bintable_column (&header, "J", "VAL_J", "int", "none", 1.0, 0.0), "defined int column"); + ok (gfits_define_bintable_column (&header, "K", "VAL_K", "long", "none", 1.0, 0.0), "defined long column"); + ok (gfits_define_bintable_column (&header, "E", "VAL_E", "float", "degree", 1.0, 0.0), "defined float column"); + ok (gfits_define_bintable_column (&header, "D", "VAL_D", "double", "degree", 1.0, 0.0), "defined double column"); + + // generate the output array that carries the data + ok (gfits_create_table (&header, &ftable), "created the basic table"); + + // XXX EAM: + struct timeval startTimer, stopTimer; + float dtime; + gettimeofday (&startTimer, (void *) NULL); + + float timeSum1 = 0.0; + float timeSum2 = 0.0; + float timeSum3 = 0.0; + float timeSum4 = 0.0; + float timeSum5 = 0.0; + float timeSum6 = 0.0; + float timeSum7 = 0.0; + + char *VAL_B; ALLOCATE (VAL_B, char, Nval); + short *VAL_I; ALLOCATE (VAL_I, short, Nval); + int *VAL_J; ALLOCATE (VAL_J, int, Nval); + int64_t *VAL_K; ALLOCATE (VAL_K, int64_t, Nval); + float *VAL_E; ALLOCATE (VAL_E, float, Nval); + double *VAL_D; ALLOCATE (VAL_D, double, Nval); + + long A = time(NULL); + srand48(A); + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum1 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + int i; + for (i = 0; i < Nval; i++) { + VAL_B[i] = 0xff & lrand48(); + VAL_I[i] = 0xffff & lrand48(); + VAL_J[i] = 0xffffffff & lrand48(); + VAL_K[i] = (((int64_t)lrand48()) << 32) + (int64_t)lrand48(); + VAL_E[i] = FLT_MAX * (2.0*drand48() - 1.0); + VAL_D[i] = DBL_MAX * (2.0*drand48() - 1.0); + } + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum2 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + // add the columns to the output array + ok (gfits_set_bintable_column (&header, &ftable, "VAL_B", VAL_B, Nval), "set byte table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_I", VAL_I, Nval), "set short table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_J", VAL_J, Nval), "set int table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_K", VAL_K, Nval), "set long table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_E", VAL_E, Nval), "set float table column"); + ok (gfits_set_bintable_column (&header, &ftable, "VAL_D", VAL_D, Nval), "set double table column"); + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum3 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + Header *outheader = &header; + FTable *outtable = &ftable; + + if (zcmptype) { + Header cmpheader; + FTable cmptable; + cmptable.header = &cmpheader; + ok (gfits_compress_table (&ftable, &cmptable, Ntile, zcmptype), "compressed table"); + + Header rawheader; + FTable rawtable; + rawtable.header = &rawheader; + ok (gfits_uncompress_table (&cmptable, &rawtable), "uncompressed table"); + + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); + + outheader = &rawheader; + outtable = &rawtable; + } + + char type[16]; + int Ncol; + off_t Nrow; + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum4 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + char *VAL_B_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_B", type, &Nrow, &Ncol); ok (!strcmp (type, "byte"), "read byte table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + short *VAL_I_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_I", type, &Nrow, &Ncol); ok (!strcmp (type, "short"), "read short table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int *VAL_J_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_J", type, &Nrow, &Ncol); ok (!strcmp (type, "int"), "read int table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + int64_t *VAL_K_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_K", type, &Nrow, &Ncol); ok (!strcmp (type, "int64_t"), "read int64_t table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + float *VAL_E_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_E", type, &Nrow, &Ncol); ok (!strcmp (type, "float"), "read float table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + double *VAL_D_t = gfits_get_bintable_column_data (outheader, outtable, "VAL_D", type, &Nrow, &Ncol); ok (!strcmp (type, "double"), "read double table column"); ok (Nrow == Nval, "right number of rows"); ok (Ncol == 1, "right number of cols"); + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum5 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + // count mismatched values + int NVAL_B_bad = 0; + int NVAL_I_bad = 0; + int NVAL_J_bad = 0; + int NVAL_K_bad = 0; + int NVAL_E_bad = 0; + int NVAL_D_bad = 0; + + for (i = 0; i < Nval; i++) { + if (VAL_B[i] != VAL_B_t[i]) NVAL_B_bad++; + if (VAL_I[i] != VAL_I_t[i]) NVAL_I_bad++; + if (VAL_J[i] != VAL_J_t[i]) NVAL_J_bad++; + if (VAL_K[i] != VAL_K_t[i]) NVAL_K_bad++; + if (VAL_E[i] != VAL_E_t[i]) NVAL_E_bad++; + if (VAL_D[i] != VAL_D_t[i]) NVAL_D_bad++; + } + + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum6 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + ok (!NVAL_B_bad, "byte values match (input vs output)"); + ok (!NVAL_I_bad, "short values match (input vs output)"); + ok (!NVAL_J_bad, "int values match (input vs output)"); + ok (!NVAL_K_bad, "int64_t values match (input vs output)"); + ok (!NVAL_E_bad, "float values match (input vs output)"); + ok (!NVAL_D_bad, "double values match (input vs output)"); + + gfits_free_header (&header); + gfits_free_table (&ftable); + if (zcmptype) { + gfits_free_header (outheader); + gfits_free_table (outtable); + } + // XXX TIMER 1 + gettimeofday (&stopTimer, (void *) NULL); + dtime = DTIME (stopTimer, startTimer); + timeSum7 += dtime; + gettimeofday (&startTimer, (void *) NULL); + + fprintf (stderr, "out times: %f %f %f %f %f %f %f\n", timeSum1, timeSum2, timeSum3, timeSum4, timeSum5, timeSum6, timeSum7); + + return TRUE; +} + + Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/test.zlib.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/test.zlib.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/test.zlib.c (revision 38447) @@ -0,0 +1,73 @@ +# include +# include +# include +# include +# include + +# define NSRC 100 +# define NTGT 200 + +int main (int argc, char **argv) { + + int i, err; + char srcbuf[NSRC], tgtbuf[NTGT], outbuf[NTGT]; + + { + z_stream zdn; + zdn.next_in = srcbuf; + zdn.avail_in = NSRC; + zdn.next_out = tgtbuf; + zdn.avail_out = NTGT; + + zdn.zalloc = Z_NULL; + zdn.zfree = Z_NULL; + zdn.opaque = Z_NULL; + + for (i = 0; i < NSRC; i++) srcbuf[i] = i; + memset (tgtbuf, 0, NTGT); + memset (outbuf, 0, NTGT); + + // the '5' is the compression level: make this a user argument + err = deflateInit(&zdn, 5); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = deflate(&zdn, Z_FINISH); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + + assert (zdn.total_out <= NTGT); + + fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; i < zdn.total_out; i++) { + fprintf (stderr, "%d : 0x%02hhx\n", i, tgtbuf[i]); + } + } + + { + z_stream zup; + zup.next_in = tgtbuf; + zup.avail_in = NTGT; + zup.next_out = outbuf; + zup.avail_out = NTGT; + + zup.zalloc = Z_NULL; + zup.zfree = Z_NULL; + zup.opaque = Z_NULL; + + // the '5' is the compression level: make this a user argument + err = inflateInit(&zup); + if (err != Z_OK) { fprintf (stderr, "error 1: %d vs %d\n", err, Z_OK); exit (1); } + + err = inflate(&zup, Z_FINISH); + if (err != Z_STREAM_END) { fprintf (stderr, "error 2: %d vs %d\n", err, Z_STREAM_END); exit (2); } + + assert (zup.total_out <= NTGT); + + fprintf (stderr, "result: %d bytes\n", (int) zup.total_out); + for (i = 0; i < zup.total_out; i++) { + fprintf (stderr, "%d : 0x%02hhx : 0x%02hhx\n", i, outbuf[i], srcbuf[i]); + assert (outbuf[i] == srcbuf[i]); + } + } + + exit (0); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/zlib.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/zlib.c (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libfits/test/zlib.c (revision 38447) @@ -0,0 +1,212 @@ +# include +# include +# include +# include "tap_ohana.h" + +# define VERBOSE 0 +# define NSRC 10 +# define NTGT 20 +# define NOUT 125 + +# define SWAP_DBLE { \ + char tmp; \ + tmp = Pout[0]; Pout[0] = Pout[7]; Pout[7] = tmp; \ + tmp = Pout[1]; Pout[1] = Pout[6]; Pout[6] = tmp; \ + tmp = Pout[2]; Pout[2] = Pout[5]; Pout[5] = tmp; \ + tmp = Pout[3]; Pout[3] = Pout[4]; Pout[4] = tmp; } + +int main (int argc, char **argv) { + + plan_tests (14); + + diag ("libfits zlib.c tests"); + + diag ("zlib version: %s", ZLIB_VERSION); + diag ("zlib vernum: %x", ZLIB_VERNUM); + + int i, err; + double srcbuf[NSRC], tgtbuf[NTGT], outbuf[NOUT]; + + { + z_stream zdn; + zdn.next_in = (char *) srcbuf; + zdn.avail_in = NSRC * sizeof(double); + zdn.next_out = (char *) tgtbuf; + zdn.avail_out = NTGT * sizeof(double); + + zdn.zalloc = Z_NULL; + zdn.zfree = Z_NULL; + zdn.opaque = Z_NULL; + + for (i = 0; i < NSRC; i++) srcbuf[i] = 1.001*(i + 10); + memset (tgtbuf, 0, NTGT * sizeof(double)); + memset (outbuf, 0, NOUT * sizeof(double)); + + for (i = 0; i < NSRC; i++) { + char *Pout = (char *) &srcbuf[i]; + SWAP_DBLE; + } + + char *rawdata = (char *) srcbuf; + if (VERBOSE) fprintf (stderr, "inp: "); + for (i = 0; VERBOSE && (i < NSRC*8); i++) { + fprintf (stderr, "0x%02hhx ", rawdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + // the '5' is the compression level: make this a user argument + err = deflateInit(&zdn, 5); + if (VERBOSE) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); + + err = deflate(&zdn, Z_FINISH); + if (VERBOSE) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + // dump out the data which failed to uncompress (and the input data) + char *cmpdata = (char *) tgtbuf; + if (VERBOSE) fprintf (stderr, "out: "); + for (i = 0; VERBOSE && (i < zdn.total_out); i++) { + fprintf (stderr, "0x%02hhx ", cmpdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zdn.total_out <= NTGT*sizeof(double), "output compressed size is good"); + + char *tgtdata = (char *) tgtbuf; + if (VERBOSE) fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; VERBOSE && i < zdn.total_out; i++) { + fprintf (stderr, "%d : 0x%02hhx\n", i, tgtdata[i]); + } + } + + { + z_stream zup; + zup.next_in = (char *) tgtbuf; + zup.avail_in = NTGT * sizeof(double); + zup.next_out = (char *) outbuf; + zup.avail_out = NOUT * sizeof(double); + + zup.zalloc = Z_NULL; + zup.zfree = Z_NULL; + zup.opaque = Z_NULL; + + // the '5' is the compression level: make this a user argument + err = inflateInit(&zup); + if (VERBOSE) fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); + + err = inflate(&zup, Z_FINISH); + if (VERBOSE) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + char *outdata = (char *) outbuf; + if (VERBOSE) fprintf (stderr, "unc: "); + for (i = 0; VERBOSE && (i < zup.total_out); i++) { + fprintf (stderr, "0x%02hhx ", outdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zup.total_out <= NTGT*sizeof(double), "uncompressed size is good"); + + int Nbad = 0; + for (i = 0; 0 && i < zup.total_out / sizeof(double); i++) { + if (outbuf[i] != srcbuf[i]) Nbad++; + } + ok (Nbad == 0, "no mismatched bytes"); + } + + /**** TEST 2 : use gzip headers ****/ + { + z_stream zdn; + zdn.next_in = (char *) srcbuf; + zdn.avail_in = NSRC * sizeof(double); + zdn.next_out = (char *) tgtbuf; + zdn.avail_out = NTGT * sizeof(double); + + zdn.zalloc = Z_NULL; + zdn.zfree = Z_NULL; + zdn.opaque = Z_NULL; + + for (i = 0; i < NSRC; i++) srcbuf[i] = 1.001*(i + 10); + memset (tgtbuf, 0, NTGT * sizeof(double)); + memset (outbuf, 0, NOUT * sizeof(double)); + + for (i = 0; i < NSRC; i++) { + char *Pout = (char *) &srcbuf[i]; + SWAP_DBLE; + } + + char *rawdata = (char *) srcbuf; + if (VERBOSE) fprintf (stderr, "inp: "); + for (i = 0; VERBOSE && (i < NSRC*8); i++) { + fprintf (stderr, "0x%02hhx ", rawdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + // the '5' is the compression level: make this a user argument + // windowBits = 31 = (15 + 16) = (2^15 window bits) + (create a gzip stream) + err = deflateInit2(&zdn, 1, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + if (VERBOSE) fprintf (stderr, "inp buffers cmp 0: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_OK, "result is Z_OK: %d vs %d", err, Z_OK); + + err = deflate(&zdn, Z_FINISH); + if (VERBOSE) fprintf (stderr, "out buffers cmp 1: %d => %d => %d\n", zdn.avail_in, zdn.avail_out, (int) zdn.total_out); + ok (err == Z_STREAM_END, "result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + // dump out the data which failed to uncompress (and the input data) + char *cmpdata = (char *) tgtbuf; + if (VERBOSE) fprintf (stderr, "out: "); + for (i = 0; VERBOSE && (i < zdn.total_out); i++) { + fprintf (stderr, "0x%02hhx ", cmpdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zdn.total_out <= NTGT*sizeof(double), "output compressed size is good"); + + char *tgtdata = (char *) tgtbuf; + if (VERBOSE) fprintf (stderr, "result: %d bytes\n", (int) zdn.total_out); + for (i = 0; VERBOSE && (i < zdn.total_out); i++) { + fprintf (stderr, "%d : 0x%02hhx\n", i, tgtdata[i]); + } + } + + { + z_stream zup; + zup.next_in = (char *) tgtbuf; + zup.avail_in = NTGT * sizeof(double); + zup.next_out = (char *) outbuf; + zup.avail_out = NOUT * sizeof(double); + + zup.zalloc = Z_NULL; + zup.zfree = Z_NULL; + zup.opaque = Z_NULL; + + // a value of 32 tells inflate to use the window size used in the compression AND to look for both zlib and gzip headers + // windowBits = 47 = (15 + 32) = (2^15 window bits) + (test for either gzip or zlib streams) + err = inflateInit2(&zup, 47); + if (VERBOSE) fprintf (stderr, "out buffers unc 0: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_OK, "inflateInit2 result is Z_OK: %d vs %d", err, Z_OK); + + err = inflate(&zup, Z_FINISH); + if (VERBOSE) fprintf (stderr, "out buffers unc 1: %d => %d => %d\n", zup.avail_in, zup.avail_out, (int) zup.total_out); + ok (err == Z_STREAM_END, "inflate result is Z_STREAM_END: %d vs %d", err, Z_STREAM_END); + + char *outdata = (char *) outbuf; + if (VERBOSE) fprintf (stderr, "unc: "); + for (i = 0; VERBOSE && (i < zup.total_out); i++) { + fprintf (stderr, "0x%02hhx ", outdata[i]); + } + if (VERBOSE) fprintf (stderr, "\n"); + + ok (zup.total_out <= NTGT*sizeof(double), "uncompressed size is good"); + + int Nbad = 0; + for (i = 0; 0 && i < zup.total_out / sizeof(double); i++) { + if (outbuf[i] != srcbuf[i]) Nbad++; + } + ok (Nbad == 0, "no mismatched bytes"); + } + + return exit_status(); +} Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/Makefile =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/Makefile (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/Makefile (revision 38447) @@ -1,5 +1,5 @@ default: install help: - @echo "make options: install libohana clean dist test typetest" + @echo "make options: install libohana clean dist test test.clean" include ../../Makefile.System @@ -10,5 +10,6 @@ MAN = $(HOME)/doc INC = $(HOME)/include -TESTDIR = $(HOME)/test +TESTDIR = $(HOME)/test +TESTBIN = $(HOME)/test include ../../Makefile.Common @@ -17,7 +18,10 @@ FULL_CPPFLAGS = $(BASE_CPPFLAGS) FULL_LDFLAGS = $(BASE_LDFLAGS) -TFLAGS = $(FULL_CFLAGS) $(FULL_CPPFLAGS) $(FULL_LDFLAGS) -lohana -ltap_ohana -install: $(DESTLIB)/libohana.a $(DESTLIB)/libohana.$(DLLTYPE) typetest +TEST_CFLAGS = $(BASE_CFLAGS) +TEST_CPPFLAGS = $(BASE_CPPFLAGS) +TEST_LDFLAGS = $(BASE_LDFLAGS) -lohana -ltap_ohana + +install: $(DESTLIB)/libohana.a $(DESTLIB)/libohana.$(DLLTYPE) libohana: $(LIB)/libohana.$(ARCH).a $(LIB)/libohana.$(ARCH).$(DLLTYPE) @@ -52,24 +56,4 @@ # $(SRC)/gsl_utils.$(ARCH).o \ -TYPETEST = \ -$(TESTDIR)/typetest.$(ARCH) - -$(TYPETEST) : $(LIB)/libohana.$(ARCH).a - -typetest: $(TYPETEST) - for i in $(TYPETEST); do $$i || exit 1; done - -TEST = \ -$(TESTDIR)/memtest.$(ARCH) -# $(TESTDIR)/string.$(ARCH) - -testcode: install $(TEST) -test: - $(MAKE) testcode - for i in $(TEST); do $$i; done - -quicktest: - for i in $(TEST); do $$i; done - $(OBJS): $(INCS) @@ -80,7 +64,8 @@ $(DESTLIB)/libohana.$(DLLTYPE): $(LIB)/libohana.$(ARCH).$(DLLTYPE) -$(TESTDIR)/%.$(ARCH) : $(TESTDIR)/%.c - $(CC) $^ -o $@ $(TFLAGS) - -.PHONY: test - +TESTPROG = memtest typetest string +$(TESTPROG) : % : $(TESTBIN)/% $(OBJS) +test: $(TESTPROG) + for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS); done +test.verbose: $(TESTPROG) + for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 |& $(TESTHARNESS) -v; done Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana.h (revision 38447) @@ -286,4 +286,5 @@ char *strcreate PROTO((char *string)); char *strncreate PROTO((char *string, int n)); +int strextend PROTO((char **input, char *format,...)) OHANA_FORMAT(printf, 2, 3); int scan_line PROTO((FILE *f, char *line)); int scan_line_maxlen PROTO((FILE *f, char *line, int maxlen)); @@ -393,5 +394,5 @@ char *memstr PROTO((char *m1, char *m2, int n)); -int write_fmt PROTO((int fd, char *format, ...)); +int write_fmt PROTO((int fd, char *format, ...)) OHANA_FORMAT(printf, 2, 3); char **isolate_elements PROTO((int argc, char **argv, int *nstack)); @@ -403,5 +404,5 @@ // gprint gets a stub implementation in libohana. opihi implements the real one. -int gprint (gpDest dest, char *format, ...); +int gprint PROTO((gpDest dest, char *format, ...) OHANA_FORMAT(printf, 2, 3)); // rconnect is used to run remote programs with a pipe for communication Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana_allocate.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana_allocate.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/include/ohana_allocate.h (revision 38447) @@ -26,5 +26,5 @@ void ohana_free (const char *file, int line, const char *func, void *in); void ohana_memdump_func (int mode); -void ohana_memcheck_func (int mode); +int ohana_memcheck_func (int mode); void real_free (void *in); @@ -56,5 +56,5 @@ # else /* below: not OHANA_MEMORY */ -# define ohana_memcheck(X) /* NOP */ +# define ohana_memcheck(X) TRUE # define ohana_memdump(X) /* NOP */ void real_free (void *in); Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/ohana_allocate.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/ohana_allocate.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/ohana_allocate.c (revision 38447) @@ -207,9 +207,9 @@ } -void ohana_memcheck_func (int allmemory) { +int ohana_memcheck_func (int allmemory) { if (!lastBlock) { fprintf (stderr, "no memory allocated\n"); - return; + return TRUE; } @@ -253,5 +253,6 @@ fprintf (stderr, "%zd memory blocks allocated (%zd bytes total), %zd good, %zd bad\n", Ntotal, Nbytes, Ngood, Nbad); - return; + if (Nbad) return FALSE; + return TRUE; } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/string.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/string.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/src/string.c (revision 38447) @@ -65,4 +65,32 @@ } +// extend input as needed to add the formatted pieces +// if input is NULL, allocate a new line +// the result of the format must be < 1024 bytes +int strextend (char **input, char *format,...) { + + int Nchar; + char tmpextra[1024], tmpline, *output; + va_list argp; + + va_start (argp, format); + Nchar = vsnprintf (tmpextra, 1024, format, argp); + if (Nchar > 1024 - 1) return FALSE; + + if (*input) { + Nchar = snprintf (&tmpline, 0, "%s %s", *input, tmpextra); + ALLOCATE (output, char, Nchar + 1); + snprintf (output, Nchar + 1, "%s %s", *input, tmpextra); + free (*input); + } else { + Nchar = strlen(tmpextra) + 1; + ALLOCATE (output, char, Nchar); + strcpy (output, tmpextra); + } + *input = output; + + return TRUE; +} + // replace a single entry of 'match' in the string with 'with' // (quick-and-dirty regex for a common case...) Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/string.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/string.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/string.c (revision 38447) @@ -4,5 +4,5 @@ int main (void) { - plan_tests (16); + plan_tests (20); diag ("libohana string.c tests"); @@ -67,4 +67,18 @@ skip_end(); } + + /*** strextend ***/ + { + int status; + char *string = NULL; + + status = strextend(&string, "hello %s", "there"); + ok (status, "strextend for NULL input"); + ok (!strcmp(string, "hello there"), "strextend format"); + + status = strextend(&string, "more words %d", 2); + ok (status, "strextend for non-NULL input"); + ok (!strcmp(string, "hello there more words 2"), "strextend format"); + } return exit_status(); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/typetest.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/typetest.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libohana/test/typetest.c (revision 38447) @@ -1,13 +1,14 @@ # include "ohana.h" +# include "tap_ohana.h" enum -{ + { OHANA_LITTLE_ENDIAN = 0x03020100ul, OHANA_BIG_ENDIAN = 0x00010203ul, -}; + }; static const union { - unsigned char bytes[4]; - int value; + unsigned char bytes[4]; + int value; } ohana_host_order = { { 0, 1, 2, 3 } }; @@ -15,70 +16,56 @@ // we need to verify some type-related issues: -int main (int argc, char **argv) { +int main (void) { - int status; + plan_tests (4); - status = 0; + diag ("libohana typetest.c tests"); - // 1) have we defined BYTE_SWAP correctly? + int status = 0; + + // 1) have we defined BYTE_SWAP correctly? # ifdef BYTE_SWAP - if (OHANA_HOST_ORDER == OHANA_BIG_ENDIAN) { - fprintf (stderr, "ERROR: BYTE_SWAP is defined on a big endian machine\n"); - fprintf (stderr, "(see libohana/include/ohana.h)\n"); - status = 1; - } else { - fprintf (stderr, "BYTE_SWAP is small endian\n"); - } + ok (OHANA_HOST_ORDER == OHANA_LITTLE_ENDIAN, "BYTE_SWAP defined on small endian hardware (see libohana/include/ohana.h)"); # else - if (OHANA_HOST_ORDER == OHANA_BIG_ENDIAN) { - fprintf (stderr, "NOT_BYTE_SWAP is big endian\n"); - } else { - fprintf (stderr, "ERROR: BYTE_SWAP not defined on a small endian machine\n"); - fprintf (stderr, "(see libohana/include/ohana.h)\n"); - status = 1; - } + ok (OHANA_HOST_ORDER == OHANA_BIG_ENDIAN, "BYTE_SWAP NOT defined on big endian hardware (see libohana/include/ohana.h)"); # endif - // 2) have we defined NAN correctly? - { - float fvalue; - double dvalue; + // 2) have we defined NAN correctly? + { + float fvalue; + double dvalue; - fvalue = NAN; - dvalue = NAN; + fvalue = NAN; + dvalue = NAN; - if (isfinite(fvalue)) { - fprintf (stderr, "ERROR: NAN definition fails for float\n"); - fprintf (stderr, "fvalue: %e\n", fvalue); - status = 1; - } - if (isfinite(dvalue)) { - fprintf (stderr, "ERROR: NAN definition fails for double\n"); - fprintf (stderr, "dvalue: %e\n", dvalue); - status = 1; - } - - fprintf (stderr, "fvalue: %e (isfinite: %d)\n", fvalue, isfinite(fvalue)); - fprintf (stderr, "dvalue: %e (isfinite: %d)\n", dvalue, isfinite(dvalue)); + ok (isnan(fvalue), "float NAN correctly defined"); + ok (isnan(dvalue), "double NAN correctly defined"); # ifdef __STDC_VERSION__ - fprintf (stderr, "STDC_VERSION: %ld\n", __STDC_VERSION__); + diag ("STDC_VERSION: %ld\n", __STDC_VERSION__); # else - fprintf (stderr, "STDC_VERSION is not set\n"); + diag ("STDC_VERSION is not set"); # endif - } + } - // 3) have we defined OFF_T_FMT correctly? - off_t big_value; + // 3) have we defined OFF_T_FMT correctly? this test should actually raise a compile-time error + off_t big_value = 10; - big_value = 10; - fprintf (stderr, "this is a big int: "OFF_T_FMT" n'est pas?\n", big_value); + char line[80]; + status = snprintf (line, 80, "this is a big int: "OFF_T_FMT" n'est pas?", big_value); + // 1234567890123456789 0123456789012 + ok (status == 32, "correctly formatted an off_t int with %d chars", status); - // we could move the % out of the FMT and allow constructions like this: - // # define OFF_T_FMT "jd" - // fprintf (stderr, "this is a big int: %06"OFF_T_FMT_ALT" n'est pas?\n", big_value); - // still kind of ugly... + // we could move the % out of the FMT and allow constructions like this: + // # define OFF_T_FMT "jd" + // fprintf (stderr, "this is a big int: %06"OFF_T_FMT_ALT" n'est pas?\n", big_value); + // still kind of ugly... - exit (status); + fprintf (stderr, "max float: %e\n", FLT_MAX); + fprintf (stderr, "max double: %e\n", DBL_MAX); + fprintf (stderr, "min float: %e\n", FLT_MIN); + fprintf (stderr, "min double: %e\n", DBL_MIN); + + return exit_status(); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/libtap/tapcheck.pl =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/libtap/tapcheck.pl (revision 38447) +++ /branches/eam_branches/ipp-20150419/Ohana/src/libtap/tapcheck.pl (revision 38447) @@ -0,0 +1,41 @@ +#!/usr/bin/env perl + +# command line options: +# -v (pass through tap output) +# -vv (pass through comments as well) +$VERBOSE = 0; +foreach $arg (@ARGV) { + if ($arg eq "-v") { $VERBOSE = 1; } + # if ($arg eq "-vv") { $VERBOSE = 2; } +} + +# capture stdin and generate a summary +@list = ; + +$headline = ""; +$Npass = 0; +$Nfail = 0; +foreach $line (@list) { + chomp $line; + if ($line =~ m|^ok|) { $Npass ++; } + if ($line =~ m|^not ok|) { $Nfail ++; } + if ($line =~ m|^#|) { + $Ncomment ++; + if ($headline eq "") { + $headline = $line; + } else { + # skip all but the first comment (headline) + if ($VERBOSE) { next; } + } + } + if ($VERBOSE) { + print "$line\n"; + } +} + +if (!$VERBOSE) { print "$headline\n"; } +print "Npass: $Npass, Nfail: $Nfail\n"; + +if ($Nfail) { exit 1; } +exit 0; + Index: /branches/eam_branches/ipp-20150419/Ohana/src/markrock/src/markrock.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/markrock/src/markrock.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/markrock/src/markrock.c (revision 38447) @@ -38,5 +38,5 @@ catalog.filename = argv[1]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -45,5 +45,5 @@ exit (2); } - if (catalog.Naves_disk) { + if (catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/find_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/find_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/find_images.c (revision 38447) @@ -25,5 +25,5 @@ } - timage = gfits_table_get_Image (&db[0].ftable, &Ntimage, &db[0].swapped); + timage = gfits_table_get_Image (&db[0].ftable, &Ntimage, &db[0].scaledValue, &db[0].nativeOrder); if (!timage) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/markstar.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/markstar.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/markstar/src/markstar.c (revision 38447) @@ -37,9 +37,10 @@ set_db (&db); + gfits_db_init (&db); dvo_image_lock (&db, ImageCat, 3600.0, LCK_XCLD); catalog.filename = argv[1]; catalog.Nsecfilt = GetPhotcodeNsecfilt (); - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -48,5 +49,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/getptolemy.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/getptolemy.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/getptolemy.c (revision 38447) @@ -26,5 +26,5 @@ // set the parameters which guide catalog open/load/create catalog.filename = skylist[0].filename[i]; - catalog.catflags = LOAD_AVES | LOAD_MEAS; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE; catalog.Nsecfilt = 0; @@ -36,7 +36,7 @@ dvo_catalog_unlock (&catalog); - // Naves_disk == 0 implies an empty catalog file + // Naverage_disk == 0 implies an empty catalog file // for only_match, skip empty catalogs - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_free (&catalog); continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/rfits.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/rfits.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/rfits.c (revision 38447) @@ -31,5 +31,5 @@ off_t Nstars; - stars = gfits_table_get_SMPData (&table, &Nstars, NULL); + stars = gfits_table_get_SMPData (&table, &Nstars, NULL, NULL); if (!stars) { fprintf (stderr, "ERROR: failed to read stars\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/wfits.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/wfits.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/mosastro/src/wfits.c (revision 38447) @@ -19,5 +19,5 @@ table.header = &theader; - gfits_table_set_SMPData (&table, stars, Nstars); + gfits_table_set_SMPData (&table, stars, Nstars, TRUE); gfits_write_header (filename, header); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/include/photdbc.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/include/photdbc.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/include/photdbc.h (revision 38447) @@ -54,4 +54,5 @@ char *CATMODE; /* raw, mef, split, mysql */ char *CATFORMAT; /* internal, elixir, loneos, panstarrs */ +char *CATCOMPRESS; /* ?? */ char PhotCodeFile[DVO_MAX_PATH]; Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/ConfigInit.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/ConfigInit.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/ConfigInit.c (revision 38447) @@ -63,4 +63,7 @@ WarnConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + // NOTE: in this program, CATFORMAT, CATMODE, CATCOMPRESS may only be set + // as command-line options, eg: -set-format PS1_V5 + if (!success) { fprintf (stderr, "missing config parameter\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/Shutdown_dvodist.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 38447) @@ -11,4 +11,5 @@ // lock the image db table + gfits_db_init (&db); int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD); if (!status) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/args.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/args.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/args.c (revision 38447) @@ -71,4 +71,12 @@ remove_argument (N, &argc, argv); CATMODE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog format (PS1_V1, PS1_REF, etc) + CATCOMPRESS = NULL; + if ((N = get_argument (argc, argv, "-set-compress"))) { + remove_argument (N, &argc, argv); + CATCOMPRESS = strcreate (argv[N]); remove_argument (N, &argc, argv); } @@ -257,4 +265,12 @@ } + // override input catalog format (PS1_V1, PS1_REF, etc) + CATCOMPRESS = NULL; + if ((N = get_argument (argc, argv, "-set-compress"))) { + remove_argument (N, &argc, argv); + CATCOMPRESS = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + /* specify portion of the sky */ REGION.Rmin = 0; @@ -344,5 +360,6 @@ fprintf (stderr, " -skip-images\n"); fprintf (stderr, " -only-images\n"); - fprintf (stderr, " -set-format (catformat)\n"); + fprintf (stderr, " -set-compress (catcompress) : NONE, GZIP_1, GZIP_2, RICE_1, AUTO\n"); + fprintf (stderr, " -set-format (catmode)\n"); fprintf (stderr, " -set-mode (catmode)\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/copy_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/copy_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/copy_images.c (revision 38447) @@ -19,4 +19,5 @@ if (status == -1) Shutdown ("ERROR: CATDIR %s does not exist, no data in database", path); + gfits_db_init (&in); status = dvo_image_lock (&in, ImageCat, 60.0, LCK_SOFT); if (!status) { @@ -38,4 +39,5 @@ // lock the output catalog + gfits_db_init (&out); status = dvo_image_lock (&out, ImageOut, 60.0, LCK_XCLD); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", ImageOut); @@ -46,5 +48,5 @@ dvo_image_create (&out, ZERO_POINT); - image = gfits_table_get_Image (&in.ftable, &Nimage, &in.swapped); + image = gfits_table_get_Image (&in.ftable, &Nimage, &in.scaledValue, &in.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/find_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/find_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/find_images.c (revision 38447) @@ -24,5 +24,5 @@ tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; - timage = gfits_table_get_Image (&db[0].ftable, &Ntimage, &db[0].swapped); + timage = gfits_table_get_Image (&db[0].ftable, &Ntimage, &db[0].scaledValue, &db[0].nativeOrder); if (!timage) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/flag_measures.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/flag_measures.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/flag_measures.c (revision 38447) @@ -15,5 +15,5 @@ if (VERBOSE) fprintf (stderr, "flagging bad measurements\n"); - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/join_stars.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/join_stars.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/join_stars.c (revision 38447) @@ -224,5 +224,5 @@ catalog[0].Naverage = Naves; catalog[0].Nmeasure = Nmeas; - catalog[0].Nsecf_mem = Naves*catalog[0].Nsecfilt; + catalog[0].Nsecfilt_mem = Naves*catalog[0].Nsecfilt; return; Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/make_subcatalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/make_subcatalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/make_subcatalog.c (revision 38447) @@ -190,5 +190,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * Nsecfilt; // XXX for now, don't copy the missing entries (these should be re-computed) Index: /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/photdbc_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 38447) @@ -24,5 +24,5 @@ incatalog.filename = hostID ? hostfile : skylist[0].filename[i]; incatalog.Nsecfilt = GetPhotcodeNsecfilt (); - incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; + incatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; // an error exit status here is a significant error @@ -32,5 +32,5 @@ } // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -51,8 +51,9 @@ // define outcatalog open parameters - outcatalog.catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : incatalog.catformat; - outcatalog.catmode = CATMODE ? dvo_catalog_catmode (CATMODE) : incatalog.catmode; - outcatalog.Nsecfilt = incatalog.Nsecfilt; // inherit from the incatalog - outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + outcatalog.catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : incatalog.catformat; + outcatalog.catmode = CATMODE ? dvo_catalog_catmode (CATMODE) : incatalog.catmode; + outcatalog.catcompress = CATCOMPRESS ? dvo_catalog_catcompress (CATCOMPRESS) : incatalog.catcompress; // set the default catcompress from config data + outcatalog.Nsecfilt = incatalog.Nsecfilt; // inherit from the incatalog + outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // output catalogs always represent the same skyregions as the input catalogs @@ -121,6 +122,6 @@ // options / arguments that can affect relastro_client -update-objects: - char command[DVO_MAX_PATH]; - snprintf (command, DVO_MAX_PATH, "photdbc_client %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D NMEAS_MIN %d -D NMEAS_MIN_FILTERED %d -D AVE_SIGMA_LIM %f -D SIGMA_MAX %f", + char *command = NULL; + strextend (&command, "photdbc_client %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D NMEAS_MIN %d -D NMEAS_MIN_FILTERED %d -D AVE_SIGMA_LIM %f -D SIGMA_MAX %f", tmproot, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, REGION.Rmin, REGION.Rmax, REGION.Dmin, REGION.Dmax, @@ -128,13 +129,13 @@ ); - char tmpline[DVO_MAX_PATH]; - if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } - if (ExcludeByInstMag) { snprintf (tmpline, DVO_MAX_PATH, "%s -instmag %f %f", command, INST_MAG_MIN, INST_MAG_MAX); strcpy (command, tmpline); } - if (ExcludeByMinSigma) { snprintf (tmpline, DVO_MAX_PATH, "%s -min-sigma %f", command, SIGMA_MIN_KEEP); strcpy (command, tmpline); } - if (ExcludeByMaxMinMag) { snprintf (tmpline, DVO_MAX_PATH, "%s -maxminmag %f", command, MAX_MIN_MAG); strcpy (command, tmpline); } - if (PHOTCODE_DROP_LIST) { snprintf (tmpline, DVO_MAX_PATH, "%s -photcode-drop %s", command, PHOTCODE_DROP_LIST); strcpy (command, tmpline); } - if (PHOTCODE_KEEP_LIST) { snprintf (tmpline, DVO_MAX_PATH, "%s -photcode-keep %s", command, PHOTCODE_KEEP_LIST); strcpy (command, tmpline); } - if (CATFORMAT) { snprintf (tmpline, DVO_MAX_PATH, "%s -set-format %s", command, CATFORMAT); strcpy (command, tmpline); } - if (CATMODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -set-mode %s", command, CATMODE); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (ExcludeByInstMag) { strextend (&command, "-instmag %f %f", INST_MAG_MIN, INST_MAG_MAX); } + if (ExcludeByMinSigma) { strextend (&command, "-min-sigma %f", SIGMA_MIN_KEEP); } + if (ExcludeByMaxMinMag) { strextend (&command, "-maxminmag %f", MAX_MIN_MAG); } + if (PHOTCODE_DROP_LIST) { strextend (&command, "-photcode-drop %s", PHOTCODE_DROP_LIST); } + if (PHOTCODE_KEEP_LIST) { strextend (&command, "-photcode-keep %s", PHOTCODE_KEEP_LIST); } + if (CATCOMPRESS) { strextend (&command, "-set-compress %s", CATCOMPRESS); } + if (CATFORMAT) { strextend (&command, "-set-format %s", CATFORMAT); } + if (CATMODE) { strextend (&command, "-set-mode %s", CATMODE); } if (PARALLEL_OUTHOSTS) { @@ -142,5 +143,5 @@ free (table_output->hosts[i].pathname); table_output->hosts[i].pathname = tmppath; - snprintf (tmpline, DVO_MAX_PATH, "%s -hostdir-output %s", command, table_output->hosts[i].pathname); strcpy (command, tmpline); + strextend (&command, "-hostdir-output %s", table_output->hosts[i].pathname); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/include/relastro.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/include/relastro.h (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/include/relastro.h (revision 38447) @@ -539,5 +539,4 @@ int hpm_objects (SkyRegion *region, Catalog *catalog); -int strextend (char *input, char *format,...); int launch_region_hosts (RegionHostTable *regionHosts); @@ -591,6 +590,4 @@ float getColorBlue (off_t meas, int cat); float getColorRed (off_t meas, int cat); - -int strextend (char *input, char *format,...); int areImagesLoaded (); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/BrightCatalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/BrightCatalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/BrightCatalog.c (revision 38447) @@ -172,5 +172,5 @@ GET_COLUMN(Nmeasure, "NMEASURE", short); GET_COLUMN(Nmissing, "NMISSING", short); - GET_COLUMN(Nextend, "NEXTEND", short); + GET_COLUMN(Ngalphot, "NGALPHOT", short); GET_COLUMN(measureOffset, "OFF_MEASURE", int); GET_COLUMN(missingOffset, "OFF_MISSING", int); @@ -208,5 +208,5 @@ average[i].Nmeasure = Nmeasure[i] ; average[i].Nmissing = Nmissing[i] ; - average[i].Nextend = Nextend[i] ; + average[i].Ngalphot = Ngalphot[i] ; average[i].measureOffset = measureOffset[i] ; average[i].missingOffset = missingOffset[i] ; @@ -241,5 +241,5 @@ free (Nmeasure); free (Nmissing); - free (Nextend); + free (Ngalphot); free (measureOffset); free (missingOffset); @@ -502,5 +502,5 @@ gfits_define_bintable_column (&theader, "I", "NMEASURE", "number of psf measurements", "", 1.0, 0.0); gfits_define_bintable_column (&theader, "I", "NMISSING", "number of missings", "", 1.0, 0.0); - gfits_define_bintable_column (&theader, "I", "NEXTEND", "number of extended measurements", "", 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "NGALPHOT", "number of galaxy shape measurements", "", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "OFF_MEASURE", "offset to first psf measurement", "", 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "OFF_MISSING", "offset to first missing obs", "", 1.0, 0.0); @@ -537,5 +537,5 @@ short *Nmeasure ; ALLOCATE (Nmeasure , short , catalog->Naverage); short *Nmissing ; ALLOCATE (Nmissing , short , catalog->Naverage); - short *Nextend ; ALLOCATE (Nextend , short , catalog->Naverage); + short *Ngalphot ; ALLOCATE (Ngalphot , short , catalog->Naverage); int *measureOffset ; ALLOCATE (measureOffset , int , catalog->Naverage); int *missingOffset ; ALLOCATE (missingOffset , int , catalog->Naverage); @@ -571,5 +571,5 @@ Nmeasure[i] = average[i].Nmeasure ; Nmissing[i] = average[i].Nmissing ; - Nextend[i] = average[i].Nextend ; + Ngalphot[i] = average[i].Ngalphot ; measureOffset[i] = average[i].measureOffset ; missingOffset[i] = average[i].missingOffset ; @@ -604,5 +604,5 @@ gfits_set_bintable_column (&theader, &ftable, "NMEASURE", Nmeasure, catalog->Naverage); gfits_set_bintable_column (&theader, &ftable, "NMISSING", Nmissing, catalog->Naverage); - gfits_set_bintable_column (&theader, &ftable, "NEXTEND", Nextend, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "NGALPHOT", Ngalphot, catalog->Naverage); gfits_set_bintable_column (&theader, &ftable, "OFF_MEASURE", measureOffset, catalog->Naverage); gfits_set_bintable_column (&theader, &ftable, "OFF_MISSING", missingOffset, catalog->Naverage); @@ -635,5 +635,5 @@ free (Nmeasure); free (Nmissing); - free (Nextend); + free (Ngalphot); free (measureOffset); free (missingOffset); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/ImageTable.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/ImageTable.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/ImageTable.c (revision 38447) @@ -7,4 +7,5 @@ FITS_DB db; + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); @@ -27,5 +28,5 @@ // convert database table to internal structure (binary to Image) // 'image' points to the same memory as db->ftable->buffer - Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { Shutdown ("ERROR: failed to read images"); @@ -42,4 +43,5 @@ // setup image table format and lock + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 38447) @@ -34,5 +34,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -41,5 +41,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -102,5 +102,5 @@ int Ngroups; - HostTableGroup *groups = HostTableGroups (table, &Ngroups); + HostTableGroup *groups = HostTableGroupsUniqueMachines (table, &Ngroups); // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines) @@ -152,49 +152,49 @@ // MaxDensityUse, MaxDensityValue - char command[1024]; - snprintf (command, 1024, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f", + char *command = NULL; + strextend (&command, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f", group->hosts[i][0].hostID, CATDIR, group->hosts[i][0].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR); - if (FIT_MODE == FIT_PM_ONLY) strextend (command, "-pm"); - if (FIT_MODE == FIT_PAR_ONLY) strextend (command, "-par"); - if (FIT_MODE == FIT_PM_AND_PAR) strextend (command, "-pmpar"); - - if (VERBOSE) strextend (command, "-v"); - if (VERBOSE2) strextend (command, "-vv"); - if (RESET) strextend (command, "-reset"); - if (UPDATE) strextend (command, "-update"); - - if (RESET_BAD_IMAGES) strextend (command, "-reset-bad-images"); - - if (ImagSelect) strextend (command, "-instmag %f %f", ImagMin, ImagMax); - if (MaxDensityUse) strextend (command, "-max-density %f", MaxDensityValue); - - if (USE_BASIC_CHECK) strextend (command, "-basic-image-search"); - if (FlagOutlier) strextend (command, "-clip %d", CLIP_THRESH); - if (ExcludeBogus) strextend (command, "-exclude-bogus %f", ExcludeBogusRadius); - - if (USE_FIXED_PIXCOORDS) strextend (command, "-D USE_FIXED_PIXCOORDS 1"); - - if (PHOTCODE_KEEP_LIST) strextend (command, "+photcode %s", PHOTCODE_KEEP_LIST); - if (PHOTCODE_SKIP_LIST) strextend (command, "-photcode %s", PHOTCODE_SKIP_LIST); - if (PhotFlagSelect) strextend (command, "+photflags"); - if (PhotFlagBad) strextend (command, "+photflagbad %d", PhotFlagBad); - if (PhotFlagPoor) strextend (command, "+photflagpoor %d", PhotFlagPoor); + if (FIT_MODE == FIT_PM_ONLY) strextend (&command, "-pm"); + if (FIT_MODE == FIT_PAR_ONLY) strextend (&command, "-par"); + if (FIT_MODE == FIT_PM_AND_PAR) strextend (&command, "-pmpar"); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (UPDATE) strextend (&command, "-update"); + + if (RESET_BAD_IMAGES) strextend (&command, "-reset-bad-images"); + + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (FlagOutlier) strextend (&command, "-clip %d", CLIP_THRESH); + if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + + if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); + + if (PHOTCODE_KEEP_LIST) strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); + if (PHOTCODE_SKIP_LIST) strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); + if (PhotFlagSelect) strextend (&command, "+photflags"); + if (PhotFlagBad) strextend (&command, "+photflagbad %d", PhotFlagBad); + if (PhotFlagPoor) strextend (&command, "+photflagpoor %d", PhotFlagPoor); // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values if (DCR_BLUE_COLOR_POS && DCR_BLUE_COLOR_NEG) { - strextend (command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); + strextend (&command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); } if (DCR_RED_COLOR_POS && DCR_RED_COLOR_NEG) { - strextend (command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); - } - - if (MinBadQF > 0.0) strextend (command, "-min-bad-psfqf %f", MinBadQF); - if (MaxMeanOffset != 10.0) strextend (command, "-max-mean-offset %f", MaxMeanOffset); + strextend (&command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); + } + + if (MinBadQF > 0.0) strextend (&command, "-min-bad-psfqf %f", MinBadQF); + if (MaxMeanOffset != 10.0) strextend (&command, "-max-mean-offset %f", MaxMeanOffset); if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - strextend (command, "-time %s %s", tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); @@ -270,49 +270,49 @@ // MaxDensityUse, MaxDensityValue - char command[1024]; - snprintf (command, 1024, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f", + char *command = NULL; + strextend (&command, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR); - if (FIT_MODE == FIT_PM_ONLY) strextend (command, "-pm"); - if (FIT_MODE == FIT_PAR_ONLY) strextend (command, "-par"); - if (FIT_MODE == FIT_PM_AND_PAR) strextend (command, "-pmpar"); - - if (VERBOSE) strextend (command, "-v"); - if (VERBOSE2) strextend (command, "-vv"); - if (RESET) strextend (command, "-reset"); - if (UPDATE) strextend (command, "-update"); - - if (RESET_BAD_IMAGES) strextend (command, "-reset-bad-images"); - - if (ImagSelect) strextend (command, "-instmag %f %f", ImagMin, ImagMax); - if (MaxDensityUse) strextend (command, "-max-density %f", MaxDensityValue); - - if (USE_BASIC_CHECK) strextend (command, "-basic-image-search"); - if (FlagOutlier) strextend (command, "-clip %d", CLIP_THRESH); - if (ExcludeBogus) strextend (command, "-exclude-bogus %f", ExcludeBogusRadius); - - if (USE_FIXED_PIXCOORDS) strextend (command, "-D USE_FIXED_PIXCOORDS 1"); - - if (PHOTCODE_KEEP_LIST) strextend (command, "+photcode %s", PHOTCODE_KEEP_LIST); - if (PHOTCODE_SKIP_LIST) strextend (command, "-photcode %s", PHOTCODE_SKIP_LIST); - if (PhotFlagSelect) strextend (command, "+photflags"); - if (PhotFlagBad) strextend (command, "+photflagbad %d", PhotFlagBad); - if (PhotFlagPoor) strextend (command, "+photflagpoor %d", PhotFlagPoor); + if (FIT_MODE == FIT_PM_ONLY) strextend (&command, "-pm"); + if (FIT_MODE == FIT_PAR_ONLY) strextend (&command, "-par"); + if (FIT_MODE == FIT_PM_AND_PAR) strextend (&command, "-pmpar"); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (UPDATE) strextend (&command, "-update"); + + if (RESET_BAD_IMAGES) strextend (&command, "-reset-bad-images"); + + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (FlagOutlier) strextend (&command, "-clip %d", CLIP_THRESH); + if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + + if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); + + if (PHOTCODE_KEEP_LIST) strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); + if (PHOTCODE_SKIP_LIST) strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); + if (PhotFlagSelect) strextend (&command, "+photflags"); + if (PhotFlagBad) strextend (&command, "+photflagbad %d", PhotFlagBad); + if (PhotFlagPoor) strextend (&command, "+photflagpoor %d", PhotFlagPoor); // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values if (DCR_BLUE_COLOR_POS && DCR_BLUE_COLOR_NEG) { - strextend (command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); + strextend (&command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); } if (DCR_RED_COLOR_POS && DCR_RED_COLOR_NEG) { - strextend (command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); - } - - if (MinBadQF > 0.0) strextend (command, "-min-bad-psfqf %f", MinBadQF); - if (MaxMeanOffset != 10.0) strextend (command, "-max-mean-offset %f", MaxMeanOffset); + strextend (&command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); + } + + if (MinBadQF > 0.0) strextend (&command, "-min-bad-psfqf %f", MinBadQF); + if (MaxMeanOffset != 10.0) strextend (&command, "-max-mean-offset %f", MaxMeanOffset); if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - strextend (command, "-time %s %s", tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/args.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/args.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/args.c (revision 38447) @@ -959,14 +959,2 @@ } -int strextend (char *input, char *format,...) { - - char tmpextra[1024], tmpline[1024]; - va_list argp; - - va_start (argp, format); - vsnprintf (tmpextra, 1024, format, argp); - snprintf (tmpline, 1024, "%s %s", input, tmpextra); - strcpy (input, tmpline); - - return TRUE; -} Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/assign_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/assign_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/assign_images.c (revision 38447) @@ -11,5 +11,5 @@ // convert database table to internal structure (binary to Image) // 'image' points to the same memory as db->ftable->buffer - Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/bcatalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/bcatalog.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/bcatalog.c (revision 38447) @@ -521,5 +521,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = oldcatalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * oldcatalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * oldcatalog[0].Nsecfilt; return (TRUE); @@ -628,5 +628,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = oldcatalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * oldcatalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * oldcatalog[0].Nsecfilt; return (TRUE); @@ -722,5 +722,5 @@ subcatalog[0].Nmeasure = Nmeasure; subcatalog[0].Nsecfilt = catalog[0].Nsecfilt; - subcatalog[0].Nsecf_mem = Naverage * catalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * catalog[0].Nsecfilt; return (TRUE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_catalogs.c (revision 38447) @@ -31,5 +31,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -38,5 +38,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -147,6 +147,6 @@ } - char command[1024]; - snprintf (command, 1024, "relastro_client -high-speed %s %s %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "relastro_client -high-speed %s %s %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f", PHOTCODE_A_LIST, PHOTCODE_B_LIST, RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); @@ -154,41 +154,36 @@ // options / arguments that can affect relastro_client -high-speed - char tmpline[1024]; - if (FIT_MODE == FIT_PM_ONLY) { snprintf (tmpline, 1024, "%s -pm", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PAR_ONLY) { snprintf (tmpline, 1024, "%s -par", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PM_AND_PAR) { snprintf (tmpline, 1024, "%s -pmpar", command); strcpy (command, tmpline); } - - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (VERBOSE2) { snprintf (tmpline, 1024, "%s -vv", command); strcpy (command, tmpline); } - if (RESET) { snprintf (tmpline, 1024, "%s -reset", command); strcpy (command, tmpline); } - if (ImagSelect) { snprintf (tmpline, 1024, "%s -instmag %f %f", command, ImagMin, ImagMax); strcpy (command, tmpline); } - if (MaxDensityUse) { snprintf (tmpline, 1024, "%s -max-density %f", command, MaxDensityValue); strcpy (command, tmpline); } + if (FIT_MODE == FIT_PM_ONLY) { strextend (&command, "-pm"); } + if (FIT_MODE == FIT_PAR_ONLY) { strextend (&command, "-par"); } + if (FIT_MODE == FIT_PM_AND_PAR) { strextend (&command, "-pmpar"); } + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (ImagSelect) { strextend (&command, "-instmag %f %f", ImagMin, ImagMax); } + if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } - if (USE_BASIC_CHECK) { snprintf (tmpline, 1024, "%s -basic-image-search", command); strcpy (command, tmpline); } - - if (FlagOutlier) { snprintf (tmpline, 1024, "%s -clip %d", command, CLIP_THRESH); strcpy (command, tmpline); } - - if (USE_FIXED_PIXCOORDS) { snprintf (tmpline, 1024, "%s -D USE_FIXED_PIXCOORDS 1", command); strcpy (command, tmpline); } - - if (PHOTCODE_KEEP_LIST) { snprintf (tmpline, 1024, "%s +photcode %s", command, PHOTCODE_KEEP_LIST); strcpy (command, tmpline); } - if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, 1024, "%s -photcode %s", command, PHOTCODE_SKIP_LIST); strcpy (command, tmpline); } - if (PhotFlagSelect) { snprintf (tmpline, 1024, "%s +photflags", command); strcpy (command, tmpline); } - if (PhotFlagBad) { snprintf (tmpline, 1024, "%s +photflagbad %d", command, PhotFlagBad); strcpy (command, tmpline); } - if (PhotFlagPoor) { snprintf (tmpline, 1024, "%s +photflagpoor %d", command, PhotFlagPoor); strcpy (command, tmpline); } + if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } + if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } + if (USE_FIXED_PIXCOORDS) { strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); } + if (PHOTCODE_KEEP_LIST) { strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); } + if (PHOTCODE_SKIP_LIST) { strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); } + if (PhotFlagSelect) { strextend (&command, "+photflags"); } + if (PhotFlagBad) { strextend (&command, "+photflagbad %d", PhotFlagBad); } + if (PhotFlagPoor) { strextend (&command, "+photflagpoor %d", PhotFlagPoor); } // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values - if (MinBadQF > 0.0) { snprintf (tmpline, 1024, "%s -min-bad-psfqf %f", command, MinBadQF); strcpy (command, tmpline); } - if (MaxMeanOffset != 10.0) { snprintf (tmpline, 1024, "%s -max-mean-offset %f", command, MaxMeanOffset); strcpy (command, tmpline); } - - if (WHERE_A[0]) { snprintf (tmpline, 1024, "%s -D WHERE_A \"%s\"", command, WHERE_A); strcpy (command, tmpline); } - if (WHERE_B[0]) { snprintf (tmpline, 1024, "%s -D WHERE_B \"%s\"", command, WHERE_B); strcpy (command, tmpline); } + if (MinBadQF > 0.0) { strextend (&command, "-min-bad-psfqf %f", MinBadQF); } + if (MaxMeanOffset != 10.0) { strextend (&command, "-max-mean-offset %f", MaxMeanOffset); } + + if (WHERE_A[0]) { strextend (&command, "-D WHERE_A \"%s\"", WHERE_A); } + if (WHERE_B[0]) { strextend (&command, "-D WHERE_B \"%s\"", WHERE_B); } if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - snprintf (tmpline, 1024, "%s -time %s %s", command, tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); - strcpy (command, tmpline); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_objects.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_objects.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/high_speed_objects.c (revision 38447) @@ -32,5 +32,5 @@ catalogOut.filename = filename; // based on the input name, need to keep everything below the catdir portion catalogOut.Nsecfilt = Nsecfilt; - catalogOut.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; // load all data + catalogOut.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // load all data catalogOut.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data @@ -263,5 +263,5 @@ catalogOut.Nmeasure=Nmatchmeas; catalogOut.Nsecfilt=Nsecfilt; - catalogOut.Nsecf_mem=Nmatch*Nsecfilt; + catalogOut.Nsecfilt_mem=Nmatch*Nsecfilt; populate_tiny_values (&catalogOut, DVO_TV_MEASURE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_catalogs.c (revision 38447) @@ -29,5 +29,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -36,5 +36,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -145,6 +145,6 @@ } - char command[1024]; - snprintf (command, 1024, "relastro_client -hpm %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "relastro_client -hpm %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f", RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); @@ -152,38 +152,33 @@ // options / arguments that can affect relastro_client -high-speed - char tmpline[1024]; - if (FIT_MODE == FIT_PM_ONLY) { snprintf (tmpline, 1024, "%s -pm", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PAR_ONLY) { snprintf (tmpline, 1024, "%s -par", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PM_AND_PAR) { snprintf (tmpline, 1024, "%s -pmpar", command); strcpy (command, tmpline); } - - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (VERBOSE2) { snprintf (tmpline, 1024, "%s -vv", command); strcpy (command, tmpline); } - if (RESET) { snprintf (tmpline, 1024, "%s -reset", command); strcpy (command, tmpline); } - if (ImagSelect) { snprintf (tmpline, 1024, "%s -instmag %f %f", command, ImagMin, ImagMax); strcpy (command, tmpline); } - if (MaxDensityUse) { snprintf (tmpline, 1024, "%s -max-density %f", command, MaxDensityValue); strcpy (command, tmpline); } + if (FIT_MODE == FIT_PM_ONLY) { strextend (&command, "-pm"); } + if (FIT_MODE == FIT_PAR_ONLY) { strextend (&command, "-par"); } + if (FIT_MODE == FIT_PM_AND_PAR) { strextend (&command, "-pmpar"); } + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (ImagSelect) { strextend (&command, "-instmag %f %f", ImagMin, ImagMax); } + if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } - if (USE_BASIC_CHECK) { snprintf (tmpline, 1024, "%s -basic-image-search", command); strcpy (command, tmpline); } - - if (FlagOutlier) { snprintf (tmpline, 1024, "%s -clip %d", command, CLIP_THRESH); strcpy (command, tmpline); } - - if (USE_FIXED_PIXCOORDS) { snprintf (tmpline, 1024, "%s -D USE_FIXED_PIXCOORDS 1", command); strcpy (command, tmpline); } - - if (PHOTCODE_KEEP_LIST) { snprintf (tmpline, 1024, "%s +photcode %s", command, PHOTCODE_KEEP_LIST); strcpy (command, tmpline); } - if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, 1024, "%s -photcode %s", command, PHOTCODE_SKIP_LIST); strcpy (command, tmpline); } - if (PhotFlagSelect) { snprintf (tmpline, 1024, "%s +photflags", command); strcpy (command, tmpline); } - if (PhotFlagBad) { snprintf (tmpline, 1024, "%s +photflagbad %d", command, PhotFlagBad); strcpy (command, tmpline); } - if (PhotFlagPoor) { snprintf (tmpline, 1024, "%s +photflagpoor %d", command, PhotFlagPoor); strcpy (command, tmpline); } + if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } + if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } + if (USE_FIXED_PIXCOORDS) { strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); } + if (PHOTCODE_KEEP_LIST) { strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); } + if (PHOTCODE_SKIP_LIST) { strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); } + if (PhotFlagSelect) { strextend (&command, "+photflags"); } + if (PhotFlagBad) { strextend (&command, "+photflagbad %d", PhotFlagBad); } + if (PhotFlagPoor) { strextend (&command, "+photflagpoor %d", PhotFlagPoor); } // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values - if (MinBadQF > 0.0) { snprintf (tmpline, 1024, "%s -min-bad-psfqf %f", command, MinBadQF); strcpy (command, tmpline); } - if (MaxMeanOffset != 10.0) { snprintf (tmpline, 1024, "%s -max-mean-offset %f", command, MaxMeanOffset); strcpy (command, tmpline); } + if (MinBadQF > 0.0) { strextend (&command, "-min-bad-psfqf %f", MinBadQF); } + if (MaxMeanOffset != 10.0) { strextend (&command, "-max-mean-offset %f", MaxMeanOffset); } if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - snprintf (tmpline, 1024, "%s -time %s %s", command, tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); - strcpy (command, tmpline); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_objects.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_objects.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/hpm_objects.c (revision 38447) @@ -33,5 +33,5 @@ catalogOut.filename = filename; catalogOut.Nsecfilt = Nsecfilt; - catalogOut.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; // load all data + catalogOut.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; // load all data catalogOut.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data @@ -279,5 +279,5 @@ } catalogOut.Nsecfilt = Nsecfilt; - catalogOut.Nsecf_mem = Nsecfilt * catalogOut.Naverage; + catalogOut.Nsecfilt_mem = Nsecfilt * catalogOut.Naverage; populate_tiny_values (&catalogOut, DVO_TV_MEASURE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/launch_region_hosts.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/launch_region_hosts.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/launch_region_hosts.c (revision 38447) @@ -77,24 +77,24 @@ } - char command[1024]; - snprintf (command, 1024, "relastro -parallel-images %s", filename); - strextend (command, "-region-hosts %s", REGION_FILE); - strextend (command, "-region-hostID %d", host->hostID); - strextend (command, "-D CATDIR %s", CATDIR); - strextend (command, "-region %f %f %f %f", host->RminCat, host->RmaxCat, host->DminCat, host->DmaxCat); - strextend (command, "-statmode %s", STATMODE); - strextend (command, "-minerror %f", MIN_ERROR); - strextend (command, "-nloop %d", NLOOP); - strextend (command, "-threads %d", NTHREADS); + char *command = NULL; + strextend (&command, "relastro -parallel-images %s", filename); + strextend (&command, "-region-hosts %s", REGION_FILE); + strextend (&command, "-region-hostID %d", host->hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-region %f %f %f %f", host->RminCat, host->RmaxCat, host->DminCat, host->DmaxCat); + strextend (&command, "-statmode %s", STATMODE); + strextend (&command, "-minerror %f", MIN_ERROR); + strextend (&command, "-nloop %d", NLOOP); + strextend (&command, "-threads %d", NTHREADS); switch (FIT_TARGET) { case TARGET_SIMPLE: - strextend (command, "-update-simple"); + strextend (&command, "-update-simple"); break; case TARGET_CHIPS: - strextend (command, "-update-chips"); + strextend (&command, "-update-chips"); break; case TARGET_MOSAICS: - strextend (command, "-update-mosaics"); + strextend (&command, "-update-mosaics"); break; case TARGET_NONE: @@ -102,54 +102,54 @@ } - if (VERBOSE) strextend (command, "-v"); - if (VERBOSE2) strextend (command, "-vv"); - if (RESET) strextend (command, "-reset"); - if (UPDATE) strextend (command, "-update"); - if (PARALLEL) strextend (command, "-parallel"); - if (PARALLEL_MANUAL) strextend (command, "-parallel-manual"); - if (PARALLEL_SERIAL) strextend (command, "-parallel-serial"); - if (PHOTCODE_KEEP_LIST) strextend (command, "+photcode %s", PHOTCODE_KEEP_LIST); - if (PHOTCODE_SKIP_LIST) strextend (command, "-photcode %s", PHOTCODE_SKIP_LIST); - if (PHOTCODE_RESET_LIST) strextend (command, "-reset-to-photcode %s", PHOTCODE_RESET_LIST); + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (UPDATE) strextend (&command, "-update"); + if (PARALLEL) strextend (&command, "-parallel"); + if (PARALLEL_MANUAL) strextend (&command, "-parallel-manual"); + if (PARALLEL_SERIAL) strextend (&command, "-parallel-serial"); + if (PHOTCODE_KEEP_LIST) strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); + if (PHOTCODE_SKIP_LIST) strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); + if (PHOTCODE_RESET_LIST) strextend (&command, "-reset-to-photcode %s", PHOTCODE_RESET_LIST); - if (MaxDensityUse) strextend (command, "-max-density %f", MaxDensityValue); - if (ImagSelect) strextend (command, "-instmag %f %f", ImagMin, ImagMax); - if (ExcludeBogus) strextend (command, "-exclude-bogus %f", ExcludeBogusRadius); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); if (DCR_BLUE_COLOR_POS && DCR_BLUE_COLOR_NEG) { - strextend (command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); + strextend (&command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); } if (DCR_RED_COLOR_POS && DCR_RED_COLOR_NEG) { - strextend (command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); + strextend (&command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); } - if (PhotFlagSelect) strextend (command, "+photflags"); - if (PhotFlagBad) strextend (command, "+photflagbad %d", PhotFlagBad); - if (PhotFlagPoor) strextend (command, "+photflagpoor %d", PhotFlagPoor); + if (PhotFlagSelect) strextend (&command, "+photflags"); + if (PhotFlagBad) strextend (&command, "+photflagbad %d", PhotFlagBad); + if (PhotFlagPoor) strextend (&command, "+photflagpoor %d", PhotFlagPoor); - if (CHIPMAP) strextend (command, "-chipmap %d", CHIPMAP); - if (RESET_IMAGES) strextend (command, "-reset-images"); + if (CHIPMAP) strextend (&command, "-chipmap %d", CHIPMAP); + if (RESET_IMAGES) strextend (&command, "-reset-images"); - if (MinBadQF > 0.0) strextend (command, "-min-bad-psfqf %f", MinBadQF); - if (MaxMeanOffset != 10.0) strextend (command, "-max-mean-offset %f", MaxMeanOffset); + if (MinBadQF > 0.0) strextend (&command, "-min-bad-psfqf %f", MinBadQF); + if (MaxMeanOffset != 10.0) strextend (&command, "-max-mean-offset %f", MaxMeanOffset); - strextend (command, "-D RELASTRO_SRC_MEAS_TOOFEW %d", SRC_MEAS_TOOFEW); - strextend (command, "-D RELASTRO_SIGMA_LIM %f", SIGMA_LIM); - strextend (command, "-D RELASTRO_DPOS_MAX %f", DPOS_MAX); - strextend (command, "-D ADDSTAR_RADIUS %f", ADDSTAR_RADIUS); + strextend (&command, "-D RELASTRO_SRC_MEAS_TOOFEW %d", SRC_MEAS_TOOFEW); + strextend (&command, "-D RELASTRO_SIGMA_LIM %f", SIGMA_LIM); + strextend (&command, "-D RELASTRO_DPOS_MAX %f", DPOS_MAX); + strextend (&command, "-D ADDSTAR_RADIUS %f", ADDSTAR_RADIUS); - strextend (command, "-D USE_GALAXY_MODEL %d", USE_GALAXY_MODEL); + strextend (&command, "-D USE_GALAXY_MODEL %d", USE_GALAXY_MODEL); - strextend (command, "-D USE_ICRF_CORRECT %d", USE_ICRF_CORRECT); - strextend (command, "-D USE_ICRF_LOCAL %d", USE_ICRF_LOCAL); - strextend (command, "-D USE_ICRF_SHFIT %d", USE_ICRF_SHFIT); - strextend (command, "-D USE_ICRF_POLE %d", USE_ICRF_POLE); + strextend (&command, "-D USE_ICRF_CORRECT %d", USE_ICRF_CORRECT); + strextend (&command, "-D USE_ICRF_LOCAL %d", USE_ICRF_LOCAL); + strextend (&command, "-D USE_ICRF_SHFIT %d", USE_ICRF_SHFIT); + strextend (&command, "-D USE_ICRF_POLE %d", USE_ICRF_POLE); - if (USE_FIXED_PIXCOORDS) strextend (command, "-D USE_FIXED_PIXCOORDS 1"); + if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - strextend (command, "-time %s %s", tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_catalogs.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_catalogs.c (revision 38447) @@ -40,5 +40,5 @@ pcatalog->catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data pcatalog->catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - pcatalog->catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; // don't need to load all data at this point + pcatalog->catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; // don't need to load all data at this point pcatalog->Nsecfilt = GetPhotcodeNsecfilt (); @@ -48,5 +48,5 @@ exit (1); } - if (!pcatalog[0].Naves_disk) { + if (!pcatalog[0].Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", pcatalog[0].filename); dvo_catalog_unlock (pcatalog); @@ -160,36 +160,36 @@ // MaxDensityUse, MaxDensityValue - char command[1024]; - snprintf (command, 1024, "relastro_client -load-objects %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f -D RELASTRO_SIGMA_LIM %f", + char *command = NULL; + strextend (&command, "relastro_client -load-objects %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f -D RELASTRO_SIGMA_LIM %f", table->hosts[i].results, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR, SIGMA_LIM); - if (FIT_MODE == FIT_PM_ONLY) strextend (command, "-pm"); - if (FIT_MODE == FIT_PAR_ONLY) strextend (command, "-par"); - if (FIT_MODE == FIT_PM_AND_PAR) strextend (command, "-pmpar"); - - if (VERBOSE) strextend (command, "-v"); - if (VERBOSE2) strextend (command, "-vv"); - if (RESET) strextend (command, "-reset"); - if (ImagSelect) strextend (command, "-instmag %f %f", ImagMin, ImagMax); - if (MaxDensityUse) strextend (command, "-max-density %f", MaxDensityValue); - if (FlagOutlier) strextend (command, "-clip %d", CLIP_THRESH); - if (ExcludeBogus) strextend (command, "-exclude-bogus %f", ExcludeBogusRadius); - - if (USE_ICRF_CORRECT) strextend (command, "-D USE_ICRF_CORRECT %d", USE_ICRF_CORRECT); - if (USE_GALAXY_MODEL) strextend (command, "-D USE_GALAXY_MODEL %d", USE_GALAXY_MODEL); + if (FIT_MODE == FIT_PM_ONLY) strextend (&command, "-pm"); + if (FIT_MODE == FIT_PAR_ONLY) strextend (&command, "-par"); + if (FIT_MODE == FIT_PM_AND_PAR) strextend (&command, "-pmpar"); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + if (FlagOutlier) strextend (&command, "-clip %d", CLIP_THRESH); + if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + + if (USE_ICRF_CORRECT) strextend (&command, "-D USE_ICRF_CORRECT %d", USE_ICRF_CORRECT); + if (USE_GALAXY_MODEL) strextend (&command, "-D USE_GALAXY_MODEL %d", USE_GALAXY_MODEL); if (DCR_BLUE_COLOR_POS && DCR_BLUE_COLOR_NEG) { - strextend (command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); + strextend (&command, "-dcr-blue-color %s %s", DCR_BLUE_COLOR_POS, DCR_BLUE_COLOR_NEG); } if (DCR_RED_COLOR_POS && DCR_RED_COLOR_NEG) { - strextend (command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); - } - - if (USE_FIXED_PIXCOORDS) strextend (command, "-D USE_FIXED_PIXCOORDS 1"); - if (PHOTCODE_KEEP_LIST) strextend (command, "+photcode %s", PHOTCODE_KEEP_LIST); - if (PHOTCODE_SKIP_LIST) strextend (command, "-photcode %s", PHOTCODE_SKIP_LIST); - if (PhotFlagSelect) strextend (command, "+photflags"); - if (PhotFlagBad) strextend (command, "+photflagbad %d", PhotFlagBad); - if (PhotFlagPoor) strextend (command, "+photflagpoor %d", PhotFlagPoor); + strextend (&command, "-dcr-red-color %s %s", DCR_RED_COLOR_POS, DCR_RED_COLOR_NEG); + } + + if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); + if (PHOTCODE_KEEP_LIST) strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); + if (PHOTCODE_SKIP_LIST) strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); + if (PhotFlagSelect) strextend (&command, "+photflags"); + if (PhotFlagBad) strextend (&command, "+photflagbad %d", PhotFlagBad); + if (PhotFlagPoor) strextend (&command, "+photflagpoor %d", PhotFlagPoor); // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values @@ -197,5 +197,5 @@ char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - strextend (command, "-time %s %s", tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/load_images.c (revision 38447) @@ -16,5 +16,5 @@ // convert database table to internal structure - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_client.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_client.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_client.c (revision 38447) @@ -69,4 +69,5 @@ set_db (&db); + gfits_db_init (&db); /* lock and load the image db table */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_images.c (revision 38447) @@ -11,4 +11,5 @@ /* register database handle with shutdown procedure */ set_db (&db); + gfits_db_init (&db); /* lock and load the image db table */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_merge_source.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_merge_source.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_merge_source.c (revision 38447) @@ -45,5 +45,5 @@ catalog_src.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog_src.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog_src.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog_src.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog_src.Nsecfilt = GetPhotcodeNsecfilt (); @@ -52,5 +52,5 @@ exit (1); } - if (!catalog_src.Naves_disk) { + if (!catalog_src.Naverage_disk) { fprintf (stderr, "no data in %s, error in cat ID?\n", catalog_src.filename); exit (1); Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_objects.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_objects.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_objects.c (revision 38447) @@ -44,5 +44,5 @@ catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -52,5 +52,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -147,42 +147,36 @@ // SRC_MEAS_TOOFEW - char command[1024]; - snprintf (command, 1024, "relastro_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s", + char *command = NULL; + strextend (&command, "relastro_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE); - char tmpline[1024]; - if (FIT_MODE == FIT_PM_ONLY) { snprintf (tmpline, 1024, "%s -pm", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PAR_ONLY) { snprintf (tmpline, 1024, "%s -par", command); strcpy (command, tmpline); } - if (FIT_MODE == FIT_PM_AND_PAR) { snprintf (tmpline, 1024, "%s -pmpar", command); strcpy (command, tmpline); } - - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (VERBOSE2) { snprintf (tmpline, 1024, "%s -vv", command); strcpy (command, tmpline); } - if (RESET) { snprintf (tmpline, 1024, "%s -reset", command); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } - - if (ImagSelect) { snprintf (tmpline, 1024, "%s -instmag %f %f", command, ImagMin, ImagMax); strcpy (command, tmpline); } - if (MaxDensityUse) { snprintf (tmpline, 1024, "%s -max-density %f", command, MaxDensityValue); strcpy (command, tmpline); } - - if (FlagOutlier) { snprintf (tmpline, 1024, "%s -clip %d", command, CLIP_THRESH); strcpy (command, tmpline); } - - if (USE_FIXED_PIXCOORDS) { snprintf (tmpline, 1024, "%s -D USE_FIXED_PIXCOORDS 1", command); strcpy (command, tmpline); } - - if (PHOTCODE_KEEP_LIST) { snprintf (tmpline, 1024, "%s +photcode %s", command, PHOTCODE_KEEP_LIST); strcpy (command, tmpline); } - if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, 1024, "%s -photcode %s", command, PHOTCODE_SKIP_LIST); strcpy (command, tmpline); } - if (PhotFlagSelect) { snprintf (tmpline, 1024, "%s +photflags", command); strcpy (command, tmpline); } - if (PhotFlagBad) { snprintf (tmpline, 1024, "%s +photflagbad %d", command, PhotFlagBad); strcpy (command, tmpline); } - if (PhotFlagPoor) { snprintf (tmpline, 1024, "%s +photflagpoor %d", command, PhotFlagPoor); strcpy (command, tmpline); } + if (FIT_MODE == FIT_PM_ONLY) { strextend (&command, "-pm"); } + if (FIT_MODE == FIT_PAR_ONLY) { strextend (&command, "-par"); } + if (FIT_MODE == FIT_PM_AND_PAR) { strextend (&command, "-pmpar"); } + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (UPDATE) { strextend (&command, "-update"); } + if (ImagSelect) { strextend (&command, "-instmag %f %f", ImagMin, ImagMax); } + if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } + if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } + if (USE_FIXED_PIXCOORDS) { strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); } + if (PHOTCODE_KEEP_LIST) { strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); } + if (PHOTCODE_SKIP_LIST) { strextend (&command, "-photcode %s", PHOTCODE_SKIP_LIST); } + if (PhotFlagSelect) { strextend (&command, "+photflags"); } + if (PhotFlagBad) { strextend (&command, "+photflagbad %d", PhotFlagBad); } + if (PhotFlagPoor) { strextend (&command, "+photflagpoor %d", PhotFlagPoor); } // XXX note that the above pass in the flag as decimal -- also note that args.c cannot handle 0xHEX values - if (MinBadQF > 0.0) { snprintf (tmpline, 1024, "%s -min-bad-psfqf %f", command, MinBadQF); strcpy (command, tmpline); } - if (MaxMeanOffset != 10.0) { snprintf (tmpline, 1024, "%s -max-mean-offset %f", command, MaxMeanOffset); strcpy (command, tmpline); } + if (MinBadQF > 0.0) { strextend (&command, "-min-bad-psfqf %f", MinBadQF); } + if (MaxMeanOffset != 10.0) { strextend (&command, "-max-mean-offset %f", MaxMeanOffset); } if (TimeSelect) { char *tstart = ohana_sec_to_date (TSTART); char *tstop = ohana_sec_to_date (TSTOP); - snprintf (tmpline, 1024, "%s -time %s %s", command, tstart, tstop); + strextend (&command, "-time %s %s", tstart, tstop); free (tstart); free (tstop); - strcpy (command, tmpline); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_parallel_regions.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_parallel_regions.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/relastro/src/relastro_parallel_regions.c (revision 38447) @@ -17,4 +17,5 @@ // register database handle with shutdown procedure set_db (&db); + gfits_db_init (&db); db.mode = dvo_catalog_catmode (CATMODE); db.format = dvo_catalog_catformat (CATFORMAT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/tools/src/fields.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/tools/src/fields.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/tools/src/fields.c (revision 38447) @@ -66,4 +66,5 @@ GotFile = TRUE; GotField &= print_fields (filename, NULL, &header, argc, argv); + GotExtension = TRUE; continue; } @@ -72,4 +73,5 @@ GotFile = TRUE; GotField &= print_fields (filename, NULL, &header, argc, argv); + GotExtension = TRUE; continue; } Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_sgroups.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_sgroups.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_sgroups.c (revision 38447) @@ -12,5 +12,5 @@ imlink = *Imlink; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_tgroups.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_tgroups.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/find_image_tgroups.c (revision 38447) @@ -11,5 +11,5 @@ ImageLink *imlink; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fiximids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fiximids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fiximids.c (revision 38447) @@ -11,4 +11,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fixstkids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fixstkids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/fixstkids.c (revision 38447) @@ -11,4 +11,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images.c (revision 38447) @@ -21,5 +21,5 @@ /* use a vtable to keep the images to be calibrated */ - image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fiximids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fiximids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fiximids.c (revision 38447) @@ -20,5 +20,5 @@ /* use a vtable to keep the images to be calibrated */ - image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fixstkids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fixstkids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_fixstkids.c (revision 38447) @@ -19,5 +19,5 @@ /* use a vtable to keep the images to be calibrated */ - image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setphot.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setphot.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setphot.c (revision 38447) @@ -11,5 +11,5 @@ /* use a vtable to keep the images to be calibrated */ - image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setposangle.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setposangle.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/load_images_setposangle.c (revision 38447) @@ -19,5 +19,5 @@ /* use a vtable to keep the images to be calibrated */ - image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setfwhm.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setfwhm.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setfwhm.c (revision 38447) @@ -13,4 +13,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setphot.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setphot.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setphot.c (revision 38447) @@ -14,4 +14,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setposangle.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setposangle.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/setposangle.c (revision 38447) @@ -11,4 +11,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/subset_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/subset_images.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/subset_images.c (revision 38447) @@ -8,5 +8,5 @@ // convert from the binary I/O format to the internal structure (Image) - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/uniphot.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/uniphot.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/uniphot.c (revision 38447) @@ -19,4 +19,5 @@ set_db (&db); + gfits_db_init (&db); status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_ckids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_ckids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_ckids.c (revision 38447) @@ -46,5 +46,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -53,5 +53,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -111,13 +111,12 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "ckids_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "ckids_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } + if (UPDATE) { strextend (&command, "-update"); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fiximids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fiximids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fiximids.c (revision 38447) @@ -41,5 +41,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -48,5 +48,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -112,6 +112,6 @@ snprintf (table->hosts[i].results, 1024, "%s/fiximids.results.%s.dat", table->hosts[i].pathname, uniquer); - char command[1024]; - snprintf (command, 1024, "fiximids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "fiximids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); @@ -119,9 +119,8 @@ int saveImstats = VERBOSE_IMSTATS || SUMMARY_IMSTATS; - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (saveImstats) { snprintf (tmpline, 1024, "%s -imstats %s", command, table->hosts[i].results); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (saveImstats) { strextend (&command, "-imstats %s", table->hosts[i].results); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } + if (UPDATE) { strextend (&command, "-update"); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fixstkids.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fixstkids.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_fixstkids.c (revision 38447) @@ -41,5 +41,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -48,5 +48,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -127,6 +127,6 @@ snprintf (table->hosts[i].results, 1024, "%s/fixstkids.results.%s.dat", table->hosts[i].pathname, uniquer); - char command[1024]; - snprintf (command, 1024, "fixstkids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "fixstkids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); @@ -134,9 +134,8 @@ int saveImstats = VERBOSE_IMSTATS || SUMMARY_IMSTATS; - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (saveImstats) { snprintf (tmpline, 1024, "%s -imstats %s", command, table->hosts[i].results); strcpy (command, tmpline); } - if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (saveImstats) { strextend (&command, "-imstats %s", table->hosts[i].results); } + if (SINGLE_CPT) { strextend (&command, "-cpt %s", SINGLE_CPT); } + if (UPDATE) { strextend (&command, "-update"); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setastrom.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setastrom.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setastrom.c (revision 38447) @@ -43,5 +43,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -50,5 +50,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -99,20 +99,18 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "setastrom_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "setastrom_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } - if (KH_FILE) { snprintf (tmpline, 1024, "%s -KH %s", command, KH_FILE); strcpy (command, tmpline); } - if (DCR_FILE) { snprintf (tmpline, 1024, "%s -DCR %s", command, DCR_FILE); strcpy (command, tmpline); } - if (CAM_FILE) { snprintf (tmpline, 1024, "%s -CAM %s", command, CAM_FILE); strcpy (command, tmpline); } - if (UPDATE_CATFORMAT) { snprintf (tmpline, 1024, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); } - - if (KH_RESET) { snprintf (tmpline, 1024, "%s -KH-reset", command); strcpy (command, tmpline); } - if (DCR_RESET) { snprintf (tmpline, 1024, "%s -DCR-reset", command); strcpy (command, tmpline); } - if (CAM_RESET) { snprintf (tmpline, 1024, "%s -CAM-reset", command); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (UPDATE) { strextend (&command, "-update"); } + if (KH_FILE) { strextend (&command, "-KH %s", KH_FILE); } + if (DCR_FILE) { strextend (&command, "-DCR %s", DCR_FILE); } + if (CAM_FILE) { strextend (&command, "-CAM %s", CAM_FILE); } + if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } + if (KH_RESET) { strextend (&command, "-KH-reset"); } + if (DCR_RESET) { strextend (&command, "-DCR-reset"); } + if (CAM_RESET) { strextend (&command, "-CAM-reset"); } fprintf (stderr, "command: %s\n", command); Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setgalmodel.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setgalmodel.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setgalmodel.c (revision 38447) @@ -38,5 +38,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_STARPAR; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_STARPAR; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -45,5 +45,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -91,16 +91,15 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "setgalmodel_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "setgalmodel_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + if (VERBOSE) { strextend (&command, "-v"); } + if (TESTING) { strextend (&command, "-testing"); } + if (UPDATE) { strextend (&command, "-update"); } + if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } + fprintf (stderr, "command: %s\n", command); - - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (TESTING) { snprintf (tmpline, 1024, "%s -testing", command); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } - if (UPDATE_CATFORMAT) { snprintf (tmpline, 1024, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); } if (PARALLEL_MANUAL) continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setphot.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setphot.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setphot.c (revision 38447) @@ -54,5 +54,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -61,5 +61,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -110,16 +110,15 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "setphot_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "setphot_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + if (VERBOSE) { strextend (&command, "-v"); } + if (RESET) { strextend (&command, "-reset"); } + if (UPDATE) { strextend (&command, "-update"); } + if (UBERCAL) { strextend (&command, "-ubercal"); } + fprintf (stderr, "command: %s\n", command); - - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (RESET) { snprintf (tmpline, 1024, "%s -reset", command); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } - if (UBERCAL) { snprintf (tmpline, 1024, "%s -ubercal", command); strcpy (command, tmpline); } if (PARALLEL_MANUAL) continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setposangle.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setposangle.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_setposangle.c (revision 38447) @@ -55,5 +55,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -62,5 +62,5 @@ exit (1); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); dvo_catalog_unlock (&catalog); @@ -112,14 +112,13 @@ table->hosts[i].pathname = tmppath; - char command[1024]; - snprintf (command, 1024, "setposangle_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + char *command = NULL; + strextend (&command, "setposangle_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + if (VERBOSE) { strextend (&command, "-v"); } + if (UPDATE) { strextend (&command, "-update"); } + fprintf (stderr, "command: %s\n", command); - - char tmpline[1024]; - if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } - if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } if (PARALLEL_MANUAL) continue; Index: /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_uniphot.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_uniphot.c (revision 38446) +++ /branches/eam_branches/ipp-20150419/Ohana/src/uniphot/src/update_dvo_uniphot.c (revision 38447) @@ -14,5 +14,5 @@ Coords coords; - image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -61,5 +61,5 @@ for (i = 0; i < pglob.gl_pathc; i++) { catalog.filename = pglob.gl_pathv[i]; - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -69,5 +69,5 @@ exit (2); } - if (!catalog.Naves_disk) { + if (!catalog.Naverage_disk) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog);