Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/Makefile =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/Makefile (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/Makefile (revision 38446) @@ -166,5 +166,5 @@ $(BIN)/dvoutils.$(ARCH) : $(DVOUTILS) -INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify +INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify dvoutils # dependancy rules for binary code ######################### Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/include/dvomerge.h (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/include/dvomerge.h (revision 38446) @@ -42,5 +42,7 @@ int REPLACE_TYCHO; int FORCE_MERGE; +int MAX_CLIENTS; char *UPDATE_CATFORMAT; +char *UPDATE_CATCOMPRESS; int MATCHED_TABLES; Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadCatalog.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadCatalog.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadCatalog.c (revision 38446) @@ -8,5 +8,5 @@ // always load all of the data (if any exists) - catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_LENSING | LOAD_LENSOBJ | LOAD_STARPAR; + catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; catalog[0].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadImages.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadImages.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadImages.c (revision 38446) @@ -34,5 +34,5 @@ } - image = gfits_table_get_Image (&db->ftable, Nimage, &db->swapped); + image = gfits_table_get_Image (&db->ftable, Nimage, &db->scaledValue, &db->nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/args.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/args.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/args.c (revision 38446) @@ -44,4 +44,12 @@ } + /* limit the impact of a dvomerge -parallel */ + MAX_CLIENTS = 10; + if ((N = get_argument (*argc, argv, "-max-clients"))) { + remove_argument (N, argc, argv); + MAX_CLIENTS = atoi(argv[N]); + remove_argument (N, argc, argv); + } + /* use a different photcode file to define mean values */ if ((N = get_argument (*argc, argv, "-photcode-file"))) { @@ -62,4 +70,10 @@ remove_argument (N, argc, argv); UPDATE_CATFORMAT = strcreate (argv[N]); + remove_argument (N, argc, argv); + } + UPDATE_CATCOMPRESS = NULL; + if ((N = get_argument (*argc, argv, "-update-catcompress"))) { + remove_argument (N, argc, argv); + UPDATE_CATCOMPRESS = strcreate (argv[N]); remove_argument (N, argc, argv); } @@ -219,4 +233,10 @@ remove_argument (N, argc, argv); } + UPDATE_CATCOMPRESS = NULL; + if ((N = get_argument (*argc, argv, "-update-catcompress"))) { + remove_argument (N, argc, argv); + UPDATE_CATCOMPRESS = strcreate (argv[N]); + remove_argument (N, argc, argv); + } /* replace measurement, don't duplicate */ Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 38446) @@ -16,5 +16,5 @@ short *TgtCodes; - imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].swapped); + imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].scaledValue, &src[0].nativeOrder); if (!imagesSrc) { fprintf (stderr, "ERROR: failed to read images from src\n"); @@ -22,5 +22,5 @@ } - imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].swapped); + imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].scaledValue, &tgt[0].nativeOrder); if (!imagesTgt) { fprintf (stderr, "ERROR: failed to read images from tgt\n"); @@ -83,5 +83,5 @@ } - images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped); + images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].scaledValue, &in[0].nativeOrder); if (!images) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoconvert.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoconvert.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoconvert.c (revision 38446) @@ -82,5 +82,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -93,5 +93,5 @@ SkyListFree (inlist); - outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; dvo_catalog_save (&outcatalog, VERBOSE); dvo_catalog_unlock (&outcatalog); @@ -122,4 +122,5 @@ /*** load output/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", output); + gfits_db_init (&outDB); outDB.mode = dvo_catalog_catmode (CATMODE); outDB.format = dvo_catalog_catformat (CATFORMAT); @@ -135,4 +136,5 @@ /*** load input/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", input); + gfits_db_init (&inDB); // inDB.mode = dvo_catalog_catmode (CATMODE); // inDB.format = dvo_catalog_catformat (CATFORMAT); @@ -149,5 +151,5 @@ // convert the raw image table to Image type (byteswap if needed) - images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped); + images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.scaledValue, &inDB.nativeOrder); if (!images) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue.c (revision 38446) @@ -109,5 +109,5 @@ LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput); // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c (revision 38446) @@ -47,5 +47,5 @@ LoadCatalog (&incatalog, threadData->region, threadData->filename, "r", threadData->NsecfiltInput); // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeCreate.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 38446) @@ -145,5 +145,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -166,5 +166,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -204,4 +204,5 @@ /*** load output/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", output); + gfits_db_init (&outDB); outDB.mode = dvo_catalog_catmode (CATMODE); outDB.format = dvo_catalog_catformat (CATFORMAT); @@ -218,4 +219,5 @@ /*** load input1/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", input1); + gfits_db_init (&in1DB); in1DB.mode = dvo_catalog_catmode (CATMODE); in1DB.format = dvo_catalog_catformat (CATFORMAT); @@ -237,4 +239,5 @@ /*** load input2/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", input2); + gfits_db_init (&in2DB); in2DB.mode = dvo_catalog_catmode (CATMODE); in2DB.format = dvo_catalog_catformat (CATFORMAT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeFromList.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 38446) @@ -113,5 +113,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeImageIDs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeImageIDs.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeImageIDs.c (revision 38446) @@ -10,7 +10,8 @@ /*** load input1/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", input); + gfits_db_init (&inDB); inDB.mode = dvo_catalog_catmode (CATMODE); inDB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? + status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename); @@ -37,4 +38,5 @@ /*** load output/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", output); + gfits_db_init (&outDB); outDB.mode = dvo_catalog_catmode (CATMODE); outDB.format = dvo_catalog_catformat (CATFORMAT); @@ -93,7 +95,8 @@ /*** load input1/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", input); + gfits_db_init (&inDB); inDB.mode = dvo_catalog_catmode (CATMODE); inDB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename); @@ -115,4 +118,5 @@ /*** load output/Images.dat ***/ sprintf (ImageCat, "%s/Images.dat", output); + gfits_db_init (&outDB); outDB.mode = dvo_catalog_catmode (CATMODE); outDB.format = dvo_catalog_catformat (CATFORMAT); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38446) @@ -145,5 +145,5 @@ // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -179,10 +179,13 @@ // if no catalog already exists, use the input catalog to define the format or the specified format - if (outcatalog.Naves_disk == 0) { - if (UPDATE_CATFORMAT) { - outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); - } else { - outcatalog.catformat = incatalog.catformat; - } + if (UPDATE_CATFORMAT) { + outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } else { + outcatalog.catformat = incatalog.catformat; + } + if (UPDATE_CATCOMPRESS) { + outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS); + } else { + outcatalog.catcompress = incatalog.catcompress; } @@ -230,4 +233,6 @@ return TRUE; } + +int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput); // launch the dvomergeUpdate_client jobs to the parallel hosts @@ -257,7 +262,6 @@ } -# if (MACHINE_GROUPS) int Ngroups; - HostTableGroup *groups = HostTableGroups (table, &Ngroups); + HostTableGroup *groups = HostTableGroupsMaxNumber (table, &Ngroups, MAX_CLIENTS); // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines) @@ -267,26 +271,21 @@ dvomergeUpdate_parallel_group (&groups[i], absinput, absoutput, outsky); } -# else - dvomergeUpdate_parallel_table (table, absinput, absoutput); -# endif - return TRUE; } -// launch the dvomergeUpdate_client jobs to the parallel hosts -int dvomergeUpdate_parallel_table (HostTable *table, char *absinput, char *absoutput) { +int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput) { int i; - for (i = 0; i < table->Nhosts; i++) { + for (i = 0; i < group->Nhosts; i++) { // ensure that the paths are absolute path names - char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); - free (table->hosts[i].pathname); - table->hosts[i].pathname = tmppath; + char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH); + free (group->hosts[i][0].pathname); + group->hosts[i][0].pathname = tmppath; // options / arguments that can affect relastro_client -update-objects: char command[DVO_MAX_PATH]; snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", - absinput, absoutput, table->hosts[i].hostID, table->hosts[i].pathname, + absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS ); @@ -315,90 +314,5 @@ int status = system (command); if (status) { - fprintf (stderr, "ERROR running photdbc_client\n"); - exit (2); - } - } else { - // launch the job on the remote machine (no handshake) - int errorInfo = 0; - int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); - if (!pid) { - if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); - exit (1); - } - table->hosts[i].pid = pid; // save for future reference - } - } - - if (PARALLEL_MANUAL) { - fprintf (stderr, "run the photdbc_client commands above. when these are done, hit return\n"); - getchar(); - } - if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { - int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); - if (!status) { - fprintf (stderr, "error running one of the remote clients\n"); - return status; - } - } - - return TRUE; -} - -// launch the dvomergeUpdate_client jobs to the parallel hosts -int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput, SkyTable *sky) { - - int i, j; - for (i = 0; i < group->Nhosts; i++) { - - if (sky->Nregions < group->Nhosts) { - // do any of the regions want this host? - int wantThisHost = FALSE; - for (j = 0; j < sky->Nregions; j++) { - if (HostTableTestHost (&sky->regions[j], group->hosts[i][0].hostID)) { - wantThisHost = TRUE; - break; - } - } - if (!wantThisHost) { - // fprintf (stderr, "skip host %s\n", group->hosts[i][0].hostname); - continue; - } - } - - // ensure that the paths are absolute path names - char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH); - free (group->hosts[i][0].pathname); - group->hosts[i][0].pathname = tmppath; - - // options / arguments that can affect relastro_client -update-objects: - char command[DVO_MAX_PATH]; - snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", - absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, - UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS - ); - - char tmpline[DVO_MAX_PATH]; - if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } - if (VERIFY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify", command); strcpy (command, tmpline); } - if (VERIFY_CATALOG_ONLY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify-catalogs", command); strcpy (command, tmpline); } - if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace", command); strcpy (command, tmpline); } - if (PARALLEL_INPUT) { snprintf (tmpline, DVO_MAX_PATH, "%s -parallel-input", command); strcpy (command, tmpline); } - if (FORCE_MERGE) { snprintf (tmpline, DVO_MAX_PATH, "%s -force-merge", command); strcpy (command, tmpline); } - if (MATCHED_TABLES) { snprintf (tmpline, DVO_MAX_PATH, "%s -matched-tables", command); strcpy (command, tmpline); } - if (UPDATE_CATFORMAT) { snprintf (tmpline, DVO_MAX_PATH, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); } - - // add some config variables: - snprintf (tmpline, DVO_MAX_PATH, "%s -D CATMODE %s", command, CATMODE); strcpy (command, tmpline); - snprintf (tmpline, DVO_MAX_PATH, "%s -D CATFORMAT %s", command, CATFORMAT); strcpy (command, tmpline); - snprintf (tmpline, DVO_MAX_PATH, "%s -D SKY_DEPTH %d", command, SKY_DEPTH); strcpy (command, tmpline); - - fprintf (stderr, "command: %s\n", command); - - if (PARALLEL_MANUAL) continue; - - if (PARALLEL_SERIAL) { - int status = system (command); - if (status) { - fprintf (stderr, "ERROR running photdbc_client\n"); + fprintf (stderr, "ERROR running dvomerge_client\n"); exit (2); } @@ -416,5 +330,5 @@ if (PARALLEL_MANUAL) { - fprintf (stderr, "run the photdbc_client commands above. when these are done, hit return\n"); + fprintf (stderr, "run the dvomerge_client commands above. when these are done, hit return\n"); getchar(); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 38446) @@ -44,5 +44,5 @@ LoadCatalog (&incatalog, threadData->region, threadData->filename, "r", threadData->NsecfiltInput); // skip empty input catalogs - if (!incatalog.Naves_disk) { + if (!incatalog.Naverage_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairCPT.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairCPT.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairCPT.c (revision 38446) @@ -76,5 +76,5 @@ } - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); @@ -156,6 +156,8 @@ gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t", 1, FALSE); + myAbort ("test for compression"); + /* convert internal to external format */ - if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL)) { + if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL, TRUE)) { myAbort("trouble converting format"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c (revision 38446) @@ -46,5 +46,5 @@ FILE *cpmFile = NULL; - char catformat; + DVOCatFormat catformat; N = get_argument (argc, argv, "-delete-image-list"); @@ -246,5 +246,5 @@ gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); @@ -284,5 +284,5 @@ // convert internal to external format - if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) { + if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) { myAbort("trouble converting format"); } @@ -467,6 +467,8 @@ gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); + myAbort ("test for compression"); + /* convert internal to external format */ - if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) { + if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) { myAbort("trouble converting format"); } @@ -529,5 +531,5 @@ /* convert internal to external format */ - if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) { + if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) { myAbort("trouble converting format"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 38446) @@ -38,5 +38,5 @@ char *imageFilename = NULL; - char catformat; + DVOCatFormat catformat; N = get_argument (argc, argv, "-delete-images-by-extern-id"); @@ -279,8 +279,10 @@ gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); + myAbort ("fix compression"); + // this function can only handle PS1_V5 and later formats // NOTE: FtableToMeasure (and equivalent) free the data associated with cpmFtable. // We are left with the data in measure. - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); @@ -331,5 +333,5 @@ // convert internal to external format - if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) { + if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) { myAbort("trouble converting format"); } @@ -493,4 +495,5 @@ /* setup image table format and lock */ + gfits_db_init (&db); db.mode = dvo_catalog_catmode ("SPLIT"); db.format = dvo_catalog_catformat ("PS1_V5"); @@ -658,6 +661,8 @@ gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); + myAbort ("test for compression"); + /* convert internal to external format */ - if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) { + if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) { myAbort("trouble converting format"); } @@ -727,5 +732,5 @@ /* convert internal to external format */ - if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) { + if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) { myAbort("trouble converting format"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixCPT.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixCPT.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixCPT.c (revision 38446) @@ -86,5 +86,5 @@ Matrix matrix; - char catformat; + DVOCatFormat catformat; cpmFtable.header = &cpmHeaderTBL; @@ -120,5 +120,6 @@ } - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + myAbort ("fix cpts"); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); @@ -127,5 +128,5 @@ // convert internal to external format - if (!MeasureToFtable (&cpmFtable, NULL, measure, Nmeasure, catformat)) { + if (!MeasureToFtable (&cpmFtable, NULL, measure, Nmeasure, catformat, TRUE)) { myAbort("trouble converting format"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixImages.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 38446) @@ -255,4 +255,5 @@ /* setup image table format and lock */ + gfits_db_init (oldDB); db.mode = oldDB->mode; db.format = oldDB->format; @@ -280,5 +281,6 @@ memset (&db.ftable.buffer[Nx*Nout], ' ', nbytes - Nx*Nout); - db.swapped = TRUE; // internal representation + db.nativeOrder = TRUE; // internal representation + db.scaledValue = TRUE; // internal representation gfits_modify (&db.header, "NIMAGES", OFF_T_FMT, 1, Nout); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixTables.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixTables.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixTables.c (revision 38446) @@ -103,5 +103,5 @@ // convert internal to external format - if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) { + if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) { myAbort("trouble converting format"); } @@ -183,5 +183,5 @@ gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows); - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); @@ -326,6 +326,8 @@ gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); + myAbort ("test for compression"); + /* convert internal to external format */ - if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) { + if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) { myAbort("trouble converting format"); } @@ -388,5 +390,5 @@ /* convert internal to external format */ - if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) { + if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) { myAbort("trouble converting format"); } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c (revision 38446) @@ -128,5 +128,5 @@ gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c (revision 38446) @@ -32,5 +32,5 @@ FILE *cpmFile = NULL; - char catformat; + DVOCatFormat catformat; N = get_argument (argc, argv, "-images-vs-measures"); @@ -134,5 +134,5 @@ gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat); + measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); myAssert(measure, "failed to convert ftable to measure data"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 38446) @@ -39,5 +39,5 @@ // always load all of the data (if any exists) catalog.Nsecfilt = 0; - catalog.catflags = LOAD_SECF; // XXX this will fail for MEF version + catalog.catflags = DVO_LOAD_SECFILT; // XXX this will fail for MEF version catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data @@ -48,5 +48,5 @@ } - if (catalog.Naves_disk == 0) { + if (catalog.Naverage_disk == 0) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); @@ -64,7 +64,7 @@ insec = catalog.secfilt; - ALLOCATE (outsec, SecFilt, catalog.Naves_disk * Nsecfilt); + ALLOCATE (outsec, SecFilt, catalog.Naverage_disk * Nsecfilt); - for (k = 0; k < catalog.Naves_disk; k++) { + for (k = 0; k < catalog.Naverage_disk; k++) { for (j = 0; (j < catalog.Nsecfilt) && (j < Nsecfilt); j++) { outsec[k*Nsecfilt + j] = insec[k*NsecInput + j]; @@ -77,6 +77,6 @@ catalog.secfilt = outsec; catalog.Nsecfilt = Nsecfilt; - catalog.Nsecf_mem = Nsecfilt * catalog.Naves_disk; - catalog.Nsecf_disk = Nsecfilt * catalog.Naves_disk; + catalog.Nsecfilt_mem = Nsecfilt * catalog.Naverage_disk; + catalog.Nsecfilt_disk = Nsecfilt * catalog.Naverage_disk; dvo_catalog_save (&catalog, VERBOSE); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_load_image_index.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_load_image_index.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_load_image_index.c (revision 38446) @@ -87,5 +87,5 @@ off_t Nimage; - Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped); + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_uniq_images.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_uniq_images.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_uniq_images.c (revision 38446) @@ -132,4 +132,7 @@ } IDlist[n] ++; + if (VERBOSE && (IDlist[n] > 1)) { + fprintf (stderr, "duplicate ext_id %d in file %s\n", extern_id[i], imFile); + } } Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoverify_utils.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoverify_utils.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoverify_utils.c (revision 38446) @@ -178,5 +178,5 @@ catalog.catformat = DVO_FORMAT_UNDEF; // read-only,do not set the catformat catalog.catmode = DVO_MODE_UNDEF; // read-only, do not set the catmode - catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_LENSING | LOAD_LENSOBJ; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ; catalog.Nsecfilt = 0; @@ -187,6 +187,6 @@ } - // Naves_disk == 0 implies an empty catalog file, skip empty catalogs - if (catalog.Naves_disk == 0) { + // Naverage_disk == 0 implies an empty catalog file, skip empty catalogs + if (catalog.Naverage_disk == 0) { dvo_catalog_unlock (&catalog); dvo_catalog_free (&catalog); @@ -467,4 +467,5 @@ // load the iage database table + gfits_db_init (&inDB); status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? if (!status) { @@ -484,5 +485,5 @@ } - images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped); + images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.scaledValue, &inDB.nativeOrder); if (!images) { fprintf (stderr, "ERROR: failed to read images from src\n"); Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs.c (revision 38446) @@ -57,5 +57,5 @@ output[0].Naverage = Naverage; output[0].Nmeasure = Nmeasure; - output[0].Nsecf_mem = Naverage * Nsecfilt; + output[0].Nsecfilt_mem = Naverage * Nsecfilt; if (VERBOSE) { Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_new.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 38446) @@ -115,5 +115,5 @@ output[0].Nlensing = Nlensing; output[0].Nstarpar = Nstarpar; - output[0].Nsecf_mem = Naverage * NsecfiltOut; + output[0].Nsecfilt_mem = Naverage * NsecfiltOut; // If we are using dvomergeCreate to split an existing catalog, then the max objID in Index: /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 38445) +++ /branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 38446) @@ -112,5 +112,5 @@ /* set up pointers for linked list of measure */ - if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) { + if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeasure_disk)) { // this version is only valid if we have done a full catalog load, and if the catalog // is sorted while processed @@ -517,5 +517,5 @@ output[0].Nlensing = Nlens; output[0].Nstarpar = Nstar; - output[0].Nsecf_mem = Nave*NsecfiltOut; + output[0].Nsecfilt_mem = Nave*NsecfiltOut; if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas, Nlens: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nlens, Nmatch);