Index: trunk/Ohana/src/dvomerge/Makefile =================================================================== --- trunk/Ohana/src/dvomerge/Makefile (revision 38553) +++ trunk/Ohana/src/dvomerge/Makefile (revision 38986) @@ -40,6 +40,6 @@ $(SRC)/dvo_image_merge_dbs.$(ARCH).o \ $(SRC)/IDmapIO.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ $(SRC)/SetSignals.$(ARCH).o \ -$(SRC)/ConfigInit.$(ARCH).o \ $(SRC)/Shutdown.$(ARCH).o \ $(SRC)/help.$(ARCH).o \ @@ -51,4 +51,5 @@ $(SRC)/replace_match.$(ARCH).o \ $(SRC)/replace_tycho.$(ARCH).o \ +$(SRC)/repair_catalog_by_objID.$(ARCH).o \ $(SRC)/merge_catalogs_new.$(ARCH).o \ $(SRC)/merge_catalogs_old.$(ARCH).o @@ -75,4 +76,5 @@ $(SRC)/replace_match.$(ARCH).o \ $(SRC)/replace_tycho.$(ARCH).o \ +$(SRC)/repair_catalog_by_objID.$(ARCH).o \ $(SRC)/merge_catalogs_new.$(ARCH).o \ $(SRC)/merge_catalogs_old.$(ARCH).o @@ -122,4 +124,5 @@ $(SRC)/dvorepair.$(ARCH).o \ $(SRC)/dvorepairFixCPT.$(ARCH).o \ +$(SRC)/dvorepairFixWarpIDs.$(ARCH).o \ $(SRC)/dvorepairImagesVsMeasures.$(ARCH).o \ $(SRC)/dvorepairDeleteImageList.$(ARCH).o \ @@ -127,8 +130,12 @@ $(SRC)/dvorepairDeleteImagesByExternID.$(ARCH).o \ $(SRC)/dvorepairFixImages.$(ARCH).o \ +$(SRC)/dvorepair_by_objID.$(ARCH).o \ +$(SRC)/repair_catalog_by_objID.$(ARCH).o \ $(SRC)/psps_ids.$(ARCH).o \ $(SRC)/LoadImages.$(ARCH).o \ $(SRC)/ReadDeleteList.$(ARCH).o \ $(SRC)/ReadDeleteListExternID.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ $(SRC)/myIndex.$(ARCH).o \ $(SRC)/match_image.$(ARCH).o \ @@ -140,4 +147,6 @@ DVOVERIFY = \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ $(SRC)/dvoverify.$(ARCH).o \ $(SRC)/dvoverify_args.$(ARCH).o \ @@ -149,4 +158,6 @@ DVOVERIFY_CLIENT = \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ $(SRC)/dvoverify_client.$(ARCH).o \ $(SRC)/dvoverify_args.$(ARCH).o \ @@ -158,4 +169,6 @@ DVOUTILS = \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ $(SRC)/dvoutils.$(ARCH).o \ $(SRC)/dvoutils_args.$(ARCH).o \ Index: trunk/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- trunk/Ohana/src/dvomerge/include/dvomerge.h (revision 38553) +++ trunk/Ohana/src/dvomerge/include/dvomerge.h (revision 38986) @@ -47,4 +47,5 @@ int MATCHED_TABLES; +int REPAIR_BY_OBJID; char *SINGLE_CPT; @@ -202,5 +203,5 @@ int dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap)); -int replace_match PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in)); +int replace_match PROTO((Average *average_out, Measure *measure_out, off_t *next_meas, Average *average_in, Measure *measure_in)); int IDmapSave PROTO((char *filename, IDmapType *IDmap)); @@ -241,3 +242,8 @@ void replace_tycho_init (); -int replace_tycho (Average *averageInp, Measure *measureInp, Average *averageOut, Measure *measureOut); +int replace_tycho (Average *averageInp, Measure *measureInp, off_t *next_meas, Average *averageOut, Measure *measureOut); +int repair_catalog_by_objID (Catalog *catalog); + +int dvorepair_by_objID (int argc, char **argv); +int dvorepairFixWarpIDs (int argc, char **argv); + Index: trunk/Ohana/src/dvomerge/include/dvoutils.h =================================================================== --- trunk/Ohana/src/dvomerge/include/dvoutils.h (revision 38553) +++ trunk/Ohana/src/dvomerge/include/dvoutils.h (revision 38986) @@ -22,4 +22,10 @@ } DVOUTILS_OP_TYPE; +typedef struct { + int *externID; + short *photcode; + int Nimages; +} ImageData; + int VERBOSE; int DVOUTILS_OP; @@ -30,5 +36,12 @@ int dvoutils_args (int *argc, char **argv); int dvoutils_uniq_images (char *filename); -int *dvoutils_load_image_index (char *filename, int *nindex); + +ImageData *dvoutils_load_image_index (char *filename); Image *dvoutils_load_image_table (char *filename, int *nimage); + +int SetSignals PROTO((void)); +void SetProtect PROTO((int mode)); +void TrapSignal PROTO((int sig)); +int Shutdown PROTO((char *format, ...)) OHANA_FORMAT(printf, 1, 2); + Index: trunk/Ohana/src/dvomerge/include/dvoverify.h =================================================================== --- trunk/Ohana/src/dvomerge/include/dvoverify.h (revision 38553) +++ trunk/Ohana/src/dvomerge/include/dvoverify.h (revision 38986) @@ -37,4 +37,6 @@ int LIST_MISSING; +int IGNORE_SORTED_STATE; + SkyRegion UserPatch; @@ -62,2 +64,8 @@ void FreeImageIDs (void); + +int SetSignals PROTO((void)); +void SetProtect PROTO((int mode)); +void TrapSignal PROTO((int sig)); +int Shutdown PROTO((char *format, ...)) OHANA_FORMAT(printf, 1, 2); + Index: trunk/Ohana/src/dvomerge/src/LoadCatalog.c =================================================================== --- trunk/Ohana/src/dvomerge/src/LoadCatalog.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/LoadCatalog.c (revision 38986) @@ -9,5 +9,7 @@ // always load all of the data (if any exists) - catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + // XXXX TEMP HACK : skip GALPHOT + // XXXX catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR; catalog[0].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data Index: trunk/Ohana/src/dvomerge/src/ReadDeleteListExternID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/ReadDeleteListExternID.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/ReadDeleteListExternID.c (revision 38986) @@ -47,7 +47,7 @@ // confirm we have 9 fields broken by 8 spaces: space = c0; // pointer to track the space-separated words - indexPoint = c0; // pointer to the ID on this line + indexPoint = strchr(c0, ' '); // pointer to the ID on this line - for (j = 0; j < 8; j++) { + for (j = 0; j < 9; j++) { space = strchr(space, ' '); if (!space) { @@ -62,5 +62,5 @@ indexList[Nindex] = index; - // fprintf (stderr, "index: %d, line: %s\n", index, c0); + fprintf (stderr, "index: %d, line: %s\n", index, c0); Nindex ++; Index: trunk/Ohana/src/dvomerge/src/args.c =================================================================== --- trunk/Ohana/src/dvomerge/src/args.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/args.c (revision 38986) @@ -94,4 +94,10 @@ if ((N = get_argument (*argc, argv, "-replace-tycho"))) { REPLACE_TYCHO = TRUE; + remove_argument (N, argc, argv); + } + + REPAIR_BY_OBJID = FALSE; + if ((N = get_argument (*argc, argv, "-repair-by-objid"))) { + REPAIR_BY_OBJID = TRUE; remove_argument (N, argc, argv); } @@ -266,4 +272,10 @@ } + REPAIR_BY_OBJID = FALSE; + if ((N = get_argument (*argc, argv, "-repair-by-objid"))) { + REPAIR_BY_OBJID = TRUE; + remove_argument (N, argc, argv); + } + NTHREADS = 0; if ((N = get_argument (*argc, argv, "-threads"))) { Index: trunk/Ohana/src/dvomerge/src/build_links.c =================================================================== --- trunk/Ohana/src/dvomerge/src/build_links.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/build_links.c (revision 38986) @@ -145,8 +145,10 @@ n = average[i].measureOffset; average[i].measureOffset = N; + int myObjID = average[i].objID; for (k = 0; k < average[i].Nmeasure; k++, N++) { if (n == -1) abort(); tmpmeasure[N] = measure[n]; - if (measure[n].averef != i) abort(); + myAssert (measure[n].averef == i, "error in averef?"); + myAssert (measure[n].objID == myObjID, "error in objID?"); tmpmeasure[N].averef = i; n = next_meas[n]; Index: trunk/Ohana/src/dvomerge/src/dvoconvert.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoconvert.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoconvert.c (revision 38986) @@ -94,4 +94,10 @@ outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + + SetProtect (TRUE); + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); } + if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); } + SetProtect (FALSE); + dvo_catalog_save (&outcatalog, VERBOSE); dvo_catalog_unlock (&outcatalog); Index: trunk/Ohana/src/dvomerge/src/dvomergeContinue.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeContinue.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeContinue.c (revision 38986) @@ -133,11 +133,8 @@ // if we receive a signal which would cause us to exit, wait until the full catalog is written SetProtect (TRUE); - if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); - exit (1); - } + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); } + if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); } SetProtect (FALSE); - dvo_catalog_unlock (&outcatalog); dvo_catalog_free (&outcatalog); Index: trunk/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c (revision 38986) @@ -77,5 +77,10 @@ SetProtect (TRUE); if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); + fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); + threadData->state = TS_FAIL; + continue; + } + if (!dvo_catalog_unlock (&outcatalog)) { + fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename); threadData->state = TS_FAIL; continue; @@ -83,5 +88,4 @@ SetProtect (FALSE); - dvo_catalog_unlock (&outcatalog); dvo_catalog_free (&outcatalog); Index: trunk/Ohana/src/dvomerge/src/dvomergeCreate.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 38986) @@ -178,6 +178,9 @@ SkyListFree (inlist); - dvo_catalog_save (&outcatalog, VERBOSE); - dvo_catalog_unlock (&outcatalog); + SetProtect (TRUE); + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); } + if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); } + SetProtect (FALSE); + dvo_catalog_free (&outcatalog); } Index: trunk/Ohana/src/dvomerge/src/dvomergeFromList.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 38986) @@ -147,8 +147,6 @@ // if we receive a signal which would cause us to exit, wait until the full catalog is written SetProtect (TRUE); - if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outputfile); - exit (1); - } + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); exit (1); } + if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename); exit (1); } SetProtect (FALSE); @@ -158,5 +156,4 @@ OutputStatusFree (outstat, 1); - dvo_catalog_unlock (&outcatalog); dvo_catalog_free (&outcatalog); Index: trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 38986) @@ -177,14 +177,26 @@ LoadCatalog (&outcatalog, outlist[0].regions[j], outcatalog.filename, "w", NsecfiltOutput); - // if no catalog already exists, use the input catalog to define the format or the specified format if (UPDATE_CATFORMAT) { outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); } else { - outcatalog.catformat = incatalog.catformat; - } + // IF no catalog already exists, use the input catalog to define the format + if (outcatalog.Naverage_disk == 0) { + outcatalog.catformat = incatalog.catformat; + } + } + if (UPDATE_CATCOMPRESS) { outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS); } else { - outcatalog.catcompress = incatalog.catcompress; + // IF no catalog already exists, use the input catalog to define the compression + if (outcatalog.Naverage_disk == 0) { + outcatalog.catcompress = incatalog.catcompress; + } + } + + if (REPAIR_BY_OBJID) { + // For gpc1 / PV3, I broke some catalogs with dvomerge -replace: the last object can + // contaminate the first set of new measurements + repair_catalog_by_objID (&outcatalog); } @@ -195,5 +207,5 @@ } - if (!dvo_catalog_backup (&outcatalog, TRUE)) { + if (!dvo_catalog_backup (&outcatalog, "~", TRUE)) { fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", outlist[0].filename[j]); exit (1); @@ -202,16 +214,9 @@ // if we receive a signal which would cause us to exit, wait until the full catalog is written SetProtect (TRUE); - if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); - exit (1); - } + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); exit (1); } + if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename); exit (1); } SetProtect (FALSE); - if (!dvo_catalog_unlock (&outcatalog)) { - fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outlist[0].filename[j]); - exit (1); - } - - if (!dvo_catalog_unlink_backup (&outcatalog, TRUE)) { + if (!dvo_catalog_unlink_backup (&outcatalog, "~", TRUE)) { fprintf (stderr, "WARNING: failed to remove backup for catalog %s\n", outlist[0].filename[j]); } @@ -282,25 +287,25 @@ // options / arguments that can affect relastro_client -update-objects: - char command[DVO_MAX_PATH]; - snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", - absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, - UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS - ); - - char tmpline[DVO_MAX_PATH]; - if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } - if (VERIFY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify", command); strcpy (command, tmpline); } - if (VERIFY_CATALOG_ONLY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify-catalogs", command); strcpy (command, tmpline); } - if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace", command); strcpy (command, tmpline); } - if (REPLACE_TYCHO) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace-tycho", command); strcpy (command, tmpline); } - if (PARALLEL_INPUT) { snprintf (tmpline, DVO_MAX_PATH, "%s -parallel-input", command); strcpy (command, tmpline); } - if (FORCE_MERGE) { snprintf (tmpline, DVO_MAX_PATH, "%s -force-merge", command); strcpy (command, tmpline); } - if (MATCHED_TABLES) { snprintf (tmpline, DVO_MAX_PATH, "%s -matched-tables", command); strcpy (command, tmpline); } - if (UPDATE_CATFORMAT) { snprintf (tmpline, DVO_MAX_PATH, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); } + char *command = NULL; + strextend (&command, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", + absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS); + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERIFY) { strextend (&command, "-verify"); } + if (VERIFY_CATALOG_ONLY) { strextend (&command, "-verify-catalogs"); } + if (REPLACE_BY_PHOTCODE) { strextend (&command, "-replace"); } + if (REPLACE_TYCHO) { strextend (&command, "-replace-tycho"); } + if (PARALLEL_INPUT) { strextend (&command, "-parallel-input"); } + if (FORCE_MERGE) { strextend (&command, "-force-merge"); } + if (MATCHED_TABLES) { strextend (&command, "-matched-tables"); } + if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } + if (UPDATE_CATCOMPRESS) { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); } + if (REPAIR_BY_OBJID) { strextend (&command, "-repair-by-objid"); } // add some config variables: - snprintf (tmpline, DVO_MAX_PATH, "%s -D CATMODE %s", command, CATMODE); strcpy (command, tmpline); - snprintf (tmpline, DVO_MAX_PATH, "%s -D CATFORMAT %s", command, CATFORMAT); strcpy (command, tmpline); - snprintf (tmpline, DVO_MAX_PATH, "%s -D SKY_DEPTH %d", command, SKY_DEPTH); strcpy (command, tmpline); + strextend (&command, "-D CATMODE %s", CATMODE); + strextend (&command, "-D CATFORMAT %s", CATFORMAT); + strextend (&command, "-D SKY_DEPTH %d", SKY_DEPTH); fprintf (stderr, "command: %s\n", command); Index: trunk/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 38986) @@ -75,5 +75,10 @@ SetProtect (TRUE); if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); + fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); + threadData->state = TS_FAIL; + continue; + } + if (!dvo_catalog_unlock (&outcatalog)) { + fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); threadData->state = TS_FAIL; continue; @@ -81,5 +86,4 @@ SetProtect (FALSE); - dvo_catalog_unlock (&outcatalog); dvo_catalog_free (&outcatalog); Index: trunk/Ohana/src/dvomerge/src/dvomerge_client.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 38986) @@ -3,4 +3,5 @@ int main (int argc, char **argv) { + SetSignals (); dvomerge_client_help (argc, argv); ConfigInit (&argc, argv); @@ -83,6 +84,6 @@ // loop over the populatable output tables; check for data in input in the corresponding regions - SetPhotcodeTable(NULL); - + if (REPLACE_TYCHO) replace_tycho_init(); + dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap); Index: trunk/Ohana/src/dvomerge/src/dvorepair.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvorepair.c (revision 38986) @@ -7,7 +7,11 @@ // exit (2); + SetSignals (); dvorepair_help(argc, argv); + if (!strcmp(argv[1], "-fix-warp-ids")) dvorepairFixWarpIDs(argc, argv); + if (!strcmp(argv[1], "-fix-cpt")) dvorepairFixCPT(argc, argv); + if (!strcmp(argv[1], "-fix-cpt-by-objID")) dvorepair_by_objID(argc, argv); // if (!strcmp(argv[1], "-fix-tables")) dvorepairFixTables(argc, argv); if (!strcmp(argv[1], "-images-vs-measures")) dvorepairImagesVsMeasures(argc, argv); Index: trunk/Ohana/src/dvomerge/src/dvorepairCPT.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairCPT.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvorepairCPT.c (revision 38986) @@ -39,4 +39,6 @@ exit (2); } + + SetSignals (); imageFilename = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 38986) @@ -16,4 +16,7 @@ int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB); Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage); +int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete); +int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete); +int RepairAverage (Catalog *catalog); int dvorepairDeleteImagesByExternID (int argc, char **argv) { @@ -21,22 +24,11 @@ FITS_DB db; // database handle pointing to input image table - int i, j, N, NmeasureNew, Ndelete, Nvalid; - off_t Nimage, Nmeasure; + int i, N; + off_t Nimage; int nPass, raPass; Image *image; - Measure *measure; - Measure *measureNew; - - char *cpmFilenameSrc = NULL; - char *cptFilenameSrc = NULL; - char *cpsFilenameSrc = NULL; - char *cpmFilenameTgt = NULL; - char *cptFilenameTgt = NULL; - char *cpsFilenameTgt = NULL; - char *imageFilename = NULL; - - DVOCatFormat catformat; + char filename[DVO_MAX_PATH]; N = get_argument (argc, argv, "-delete-images-by-extern-id"); @@ -59,4 +51,10 @@ char *catdir = argv[1]; char *delList = argv[2]; + + sprintf (filename, "%s/Photcodes.dat", catdir); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error reading photcodes from %s\n", catdir); + exit (1); + } // load the image data @@ -196,12 +194,6 @@ } - ALLOCATE(cpmFilenameSrc, char, strlen(catdir) + 64); - ALLOCATE(cptFilenameSrc, char, strlen(catdir) + 64); - ALLOCATE(cpsFilenameSrc, char, strlen(catdir) + 64); - ALLOCATE(cpmFilenameTgt, char, strlen(catdir) + 64); - ALLOCATE(cptFilenameTgt, char, strlen(catdir) + 64); - ALLOCATE(cpsFilenameTgt, char, strlen(catdir) + 64); - - int NdeleteTotal = 0; + int NmeasureDelTotal = 0; + int NlensingDelTotal = 0; for (raPass = 0; raPass < nPass; raPass++) { @@ -231,154 +223,63 @@ // loop over the populated input regions - int Ncheck = 0; for (i = 0; i < inlist[0].Nregions; i++) { if (!inlist[0].regions[i][0].table) continue; - sprintf (cpmFilenameSrc, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name); - sprintf (cptFilenameSrc, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name); - sprintf (cpsFilenameSrc, "%s/%s.cps", catdir, inlist[0].regions[i][0].name); - sprintf (cpmFilenameTgt, "%s/%s.cpm.broken", catdir, inlist[0].regions[i][0].name); - sprintf (cptFilenameTgt, "%s/%s.cpt.broken", catdir, inlist[0].regions[i][0].name); - sprintf (cpsFilenameTgt, "%s/%s.cps.broken", catdir, inlist[0].regions[i][0].name); - - Header cpmHeaderPHU; - Header cpmHeaderTBL; - FTable cpmFtable; - - /*** read and examine the CPM file ***/ - { - if (VERBOSE) fprintf (stderr, "check %s\n", cpmFilenameSrc); - - cpmFtable.header = &cpmHeaderTBL; - - // open cpm file - FILE *cpmFile = fopen(cpmFilenameSrc, "r"); - if (!cpmFile) continue; - // myAssert(cpmFile, "failed to open cpm file"); - - // load the cpm header - if (!gfits_fread_header (cpmFile, &cpmHeaderPHU)) { - myAbort("failure to cpm header"); - } - - // move to TBL header - int Nbytes = cpmHeaderPHU.datasize + gfits_data_size (&cpmHeaderPHU); - fseeko (cpmFile, Nbytes, SEEK_SET); - - // read cpm TBL header - if (!gfits_fread_header (cpmFile, &cpmHeaderTBL)) { - myAbort("can't read header for cpm table"); - } - - // read Measure table data : format is irrelevant here */ - if (!gfits_fread_ftable_data (cpmFile, &cpmFtable, TRUE)) { - myAbort("can't read data for cpm table"); - } - - int NbytesPerRow; - gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); - - myAbort ("fix compression"); - - // this function can only handle PS1_V5 and later formats - // NOTE: FtableToMeasure (and equivalent) free the data associated with cpmFtable. - // We are left with the data in measure. - measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE); - myAssert(measure, "failed to convert ftable to measure data"); - - Nvalid = (int)(cpmFtable.validsize / NbytesPerRow); - Nvalid = MIN(Nmeasure, Nvalid); - - // close the input cpm file - fclose(cpmFile); - - // allocate an output array of measures (to replace, if needed) - ALLOCATE (measureNew, Measure, Nvalid); - - NmeasureNew = 0; - Ndelete = 0; - - // examine all measurements: find ones that need to be deleted - for (j = 0; j < Nvalid; j++) { - int imageID = measure[j].imageID; - myAssert(imageID, "measure is missing an image ID"); - // XXX this case is valid if we have REF detections (no associated image) - - int N = myIndexGetEntry(imageIDindex, imageID); - if (N < 0) { - // this detection comes from a non-existant image, delete - Ndelete ++; - continue; - } - - if (deleteImage[N]) { - Ndelete ++; - continue; - } - measureNew[NmeasureNew] = measure[j]; - NmeasureNew ++; - } + snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name); + + Catalog catalog; + + // set up the basic catalog info + dvo_catalog_init (&catalog, TRUE); + catalog.filename = filename; // XXX only allow non-parallel dbs for now + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT | DVO_LOAD_LENSING; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, inlist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); } - - NdeleteTotal += Ndelete; - - // if we actually want to delete any measurements, write out a new cpm file - if (Ndelete > 0) { - - fprintf (stderr, "deleting %d of %d (keep %d) detections from %s -> %s\n", (int) Ndelete, (int) Nvalid, (int) NmeasureNew, cpmFilenameSrc, cpmFilenameTgt); + if (!catalog.Naverage_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } - // the CPT and CPS tables need to be regenerated. This must happen first because, in the process, we also update measure->averef - // what about other tables? - RepairTableCPT_V1(cptFilenameSrc, cptFilenameTgt, cpsFilenameSrc, cpsFilenameTgt, measureNew, NmeasureNew, image, Nimage, imageIDindex, catformat); - - // convert internal to external format - if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) { - myAbort("trouble converting format"); - } - - // rename the old cpm file: - if (rename (cpmFilenameSrc, cpmFilenameTgt)) { - perror ("tried to rename file"); - exit (2); - } - - // create and write the output file - FILE *cpmFile = fopen(cpmFilenameSrc, "w"); - myAssert(cpmFile, "failed to open cpt file"); - - // write PHU header - if (!gfits_fwrite_header (cpmFile, &cpmHeaderPHU)) { - myAbort("can't write primary header"); - } - - Matrix matrix; - - // write the PHU matrix; this is probably a NOP, do I have to keep it in? - gfits_create_matrix (&cpmHeaderPHU, &matrix); - if (!gfits_fwrite_matrix (cpmFile, &matrix)) { - myAbort("can't write primary matrix"); - } - gfits_free_matrix (&matrix); - - // write the table data - if (!gfits_fwrite_ftable_range (cpmFile, &cpmFtable, 0, NmeasureNew, 0, NmeasureNew)) { - myAbort("can't write table data"); - } - fclose (cpmFile); - - gfits_free_table (&cpmFtable); - } else { - if (VERBOSE) fprintf (stderr, "nothing to delete in %s\n", cpmFilenameSrc); + /*** delete the measure and lensing entries matching the list of imageIDs ***/ + int NmeasureDel = 0; + DeleteMeasure (&catalog, imageIDindex, deleteImage, &NmeasureDel); + + int NlensingDel = 0; + DeleteLensing (&catalog, imageIDindex, deleteImage, &NlensingDel); + + fprintf (stderr, "deleting %d measure, %d lensing (keep %d, %d)\n", NmeasureDel, NlensingDel, (int) catalog.Nmeasure, (int) catalog.Nlensing); + + if (!NmeasureDel && !NlensingDel) { + if (VERBOSE) fprintf (stderr, "nothing to delete in %s\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; } - gfits_free_header (&cpmHeaderPHU); - gfits_free_header (&cpmHeaderTBL); - free (measure); - free (measureNew); - - Ncheck ++; - if (Ncheck % 1000 == 0) { - fprintf (stderr, "%s...", inlist[0].regions[i][0].name); + NmeasureDelTotal += NmeasureDel; + NlensingDelTotal += NlensingDel; + + // the CPT and CPS tables need to be regenerated. This must happen first because, in the process, we also update measure->averef + // what about other tables? + RepairAverage (&catalog); + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + if (!dvo_catalog_backup (&catalog, ".undel", TRUE)) { + fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", catalog.filename); + exit (1); } + + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + dvo_catalog_free (&catalog); } fprintf (stderr, "\n"); @@ -386,5 +287,5 @@ } - fprintf (stderr, "Deleted %d detections\n", NdeleteTotal); + fprintf (stderr, "Deleted %d measure, %d lensing detections\n", NmeasureDelTotal, NlensingDelTotal); off_t NnewImage; @@ -400,14 +301,9 @@ free (deleteIDs); - free(cpmFilenameSrc); - free(cptFilenameSrc); - free(cpsFilenameSrc); - free(cpmFilenameTgt); - free(cptFilenameTgt); - free(cpsFilenameTgt); - SkyTableFree(insky); gfits_db_free (&db); + + ohana_memcheck (TRUE); exit (0); @@ -661,5 +557,12 @@ gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); - myAbort ("test for compression"); + if (0) { + char compressMode[256]; + if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) { + if (strcmp (compressMode, "NONE")) { + myAbort ("fix compression"); + } + } + } /* convert internal to external format */ @@ -774,2 +677,228 @@ } +int RepairAverage (Catalog *catalog) { + + off_t i, j, Nave; + + // average is sorted so averef is valid + // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated) + // average.measureOffset is NOT valid + // average.lensingOffset is NOT valid + + Average *average = catalog->average; + Measure *measure = catalog->measure; + Lensing *lensing = catalog->lensing; + + // reset the values of average.Nmeasure, average.Nlensing + for (i = 0; i < catalog->Naverage; i++) { + average[i].Nmeasure = 0; + average[i].Nlensing = 0; + } + + // re-calculate average.Nmeasure + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = measure[i].averef; + myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match"); + average[Nave].Nmeasure ++; + } + + // re-calculate average.Nlensing + for (i = 0; i < catalog->Nlensing; i++) { + Nave = lensing[i].averef; + myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match"); + average[Nave].Nlensing ++; + } + + // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0 + ALLOCATE_PTR (averageNew, Average, catalog->Naverage); + ALLOCATE_PTR (averefNew, int, catalog->Naverage); + + Nave = 0; + for (i = 0; i < catalog->Naverage; i++) { + averefNew[i] = -1; + if (!average[i].Nmeasure && !average[i].Nlensing) continue; + + averageNew[Nave] = average[i]; + averefNew[i] = Nave; + Nave ++; + } + off_t NaverageNew = Nave; + + // update measure.averef values + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = averefNew[measure[i].averef]; + myAssert (Nave >= 0, "oops"); + measure[i].averef = Nave; + } + + // update lensing.averef values + for (i = 0; i < catalog->Nlensing; i++) { + Nave = averefNew[lensing[i].averef]; + myAssert (Nave >= 0, "oops"); + lensing[i].averef = Nave; + } + + // XXX need to update measureOffset and lensingOffset + + // measure[] should be blocked and sequential: + // measure[i+1].averef >= measure[i].averef + + // update average.measureOffset values + Nave = -1; + for (i = 0; i < catalog->Nmeasure; i++) { + if (measure[i].averef == Nave) continue; + Nave = measure[i].averef; + averageNew[Nave].measureOffset = i; + } + + // update average.lensingOffset values + Nave = -1; + for (i = 0; i < catalog->Nlensing; i++) { + if (lensing[i].averef == Nave) continue; + Nave = lensing[i].averef; + averageNew[Nave].lensingOffset = i; + } + + // check the result (measure -> average) + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = measure[i].averef; + myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match"); + myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match"); + } + // check the result (average -> measure) + for (i = 0; i < NaverageNew; i++) { + int m = averageNew[i].measureOffset; + for (j = 0; j < averageNew[i].Nmeasure; j++) { + myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match"); + myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match"); + myAssert(measure[j+m].averef == i, "averef broken"); + } + } + + // check the result (lensing -> average) + for (i = 0; i < catalog->Nlensing; i++) { + Nave = lensing[i].averef; + myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match"); + myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match"); + } + // check the result (average -> lensing) + for (i = 0; i < NaverageNew; i++) { + int m = averageNew[i].lensingOffset; + for (j = 0; j < averageNew[i].Nlensing; j++) { + myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match"); + myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match"); + myAssert(lensing[j+m].averef == i, "averef broken"); + } + } + + free (catalog->secfilt); + ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt); + for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) { + dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL); + } + + free (averefNew); + free (catalog->average); + catalog->average = averageNew; + catalog->Naverage = NaverageNew; + catalog->Naverage_disk = NaverageNew; + + catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt; + + return (TRUE); +} + +// delete measure entries +int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { + + int j; + + Measure *measure = catalog->measure; + + // allocate an output array of measures (to replace, if needed) + ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure); + + int NmeasureNew = 0; + int NmeasureDel = 0; + + // examine all measurements: find ones that need to be deleted + for (j = 0; j < catalog->Nmeasure; j++) { + int imageID = measure[j].imageID; + myAssert(imageID, "measure is missing an image ID"); + // XXX this case is valid if we have REF detections (no associated image) + + int N = myIndexGetEntry(imageIDindex, imageID); + if (N < 0) { + // this detection comes from a non-existant image; delete + NmeasureDel ++; + continue; + } + + // measure matches a bad image; delete + if (deleteImage[N]) { + NmeasureDel ++; + continue; + } + + // keep this measure + measureNew[NmeasureNew] = measure[j]; + NmeasureNew ++; + } + + free (catalog->measure); + catalog->measure = measureNew; + catalog->Nmeasure = NmeasureNew; + catalog->Nmeasure_disk = NmeasureNew; + + *nDelete = NmeasureDel; + + return TRUE; +} + +// delete lensing entries +int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { + + int j; + + Lensing *lensing = catalog->lensing; + + // allocate an output array of measures (to replace, if needed) + ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing); + + int NlensingNew = 0; + int NlensingDel = 0; + + // examine all lensingments: find ones that need to be deleted + for (j = 0; j < catalog->Nlensing; j++) { + int imageID = lensing[j].imageID; + myAssert(imageID, "lensing is missing an image ID"); + // XXX this case is valid if we have REF detections (no associated image) + + int N = myIndexGetEntry(imageIDindex, imageID); + if (N < 0) { + // this detection comes from a non-existant image; delete + NlensingDel ++; + continue; + } + + // lensing matches a bad image; delete + if (deleteImage[N]) { + NlensingDel ++; + continue; + } + + // keep this lensing + lensingNew[NlensingNew] = lensing[j]; + NlensingNew ++; + } + + free (catalog->lensing); + catalog->lensing = lensingNew; + catalog->Nlensing = NlensingNew; + catalog->Nlensing_disk = NlensingNew; + + *nDelete = NlensingDel; + + return TRUE; +} + Index: trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 38986) @@ -255,5 +255,5 @@ /* setup image table format and lock */ - gfits_db_init (oldDB); + gfits_db_init (&db); db.mode = oldDB->mode; db.format = oldDB->format; @@ -266,7 +266,8 @@ /* load or create the image table */ myAssert (db.dbstate == LCK_EMPTY, "do not overwrite exiting image table"); - myAssert (db.format == DVO_FORMAT_PS1_V2, "format mismatch"); + // myAssert (db.format == DVO_FORMAT_PS1_V2, "format mismatch"); dvo_image_create (&db, GetZeroPoint()); + gfits_copy_header (&oldDB->header, &db.header); // replace the existing buffer with the image array @@ -277,6 +278,8 @@ nbytes = gfits_data_size (db.ftable.header); db.ftable.datasize = nbytes; - db.ftable.buffer = (char *) imageOut; - REALLOCATE (db.ftable.buffer, char, MAX (nbytes, 1)); + + free (db.ftable.buffer); + ALLOCATE (db.ftable.buffer, char, MAX (nbytes, 1)); + memcpy (db.ftable.buffer, imageOut, nbytes); memset (&db.ftable.buffer[Nx*Nout], ' ', nbytes - Nx*Nout); @@ -295,4 +298,6 @@ dvo_image_unlock (&db); + gfits_db_free (&db); + return TRUE; } Index: trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c (revision 38986) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c (revision 38986) @@ -0,0 +1,233 @@ +# include "dvomerge.h" + +static int *warpIDs = NULL; +static int *projIDs = NULL; +static int *cellIDs = NULL; +static int *fileIDs = NULL; + +# define STRFAIL { fprintf (stderr, "failure on image %s\n", image[i].name); continue; } + +int load_warp_ids (char *idfile, int *nfiles); +int warpIDs_bisection (int *warpIDs, int threshold, int Nvalues); + +int dvorepairFixWarpIDs (int argc, char **argv) { + + FITS_DB db; // database handle pointing to input image table + + off_t Nimage; + + int N; + + Image *image; + + N = get_argument (argc, argv, "-fix-warp-ids"); + myAssert(N == 1, "programming error: -fix-images must be first from main"); + remove_argument (N, &argc, argv); + + if (argc != 3) { + fprintf (stderr, "USAGE: dvorepair -fix-warp-ids (catdir.list) (warp.ids)\n"); + fprintf (stderr, " catdir.list : list of databases of interest\n"); + fprintf (stderr, " warp.ids : list of warp_id, skycell.id, warp_skyfile_id map\n"); + exit (2); + } + + char *catdir_list = argv[1]; + char *idfile = argv[2]; + + // load the warp_id table + int Nfiles = 0; + load_warp_ids (idfile, &Nfiles); + fprintf (stderr, "loaded %d warp,proj,cell,file matches\n", Nfiles); + + char name[DVO_MAX_PATH]; + char catdir[DVO_MAX_PATH]; + char imageFilenameOld[DVO_MAX_PATH]; + char imageFilenameNew[DVO_MAX_PATH]; + + // read the list of catdirs and fix image tables for each + + FILE *f = fopen (catdir_list, "r"); + myAssert (f, "failed to open catdir.list %s\n", catdir_list); + + while (fscanf (f, "%s", catdir) != EOF) { + snprintf (imageFilenameOld, DVO_MAX_PATH, "%s/Images.dat", catdir); + snprintf (imageFilenameNew, DVO_MAX_PATH, "%s/Images.dat.fixed", catdir); + + if ((image = LoadImages (&db, imageFilenameOld, &Nimage)) == NULL) { + fprintf (stderr, "error loading images\n"); + exit (1); + } + + int i, j; + for (i = 0; i < Nimage; i++) { + if (image[i].externID) continue; + + strcpy (name, image[i].name); + + char *p0 = strchr (name , '.'); if (!p0) STRFAIL; + char *p1 = strchr (p0 + 1, '.'); if (!p1) STRFAIL; + char *p2 = strchr (p1 + 1, '.'); if (!p2) STRFAIL; + char *p3 = strchr (p2 + 1, '.'); if (!p3) STRFAIL; + char *p4 = strchr (p3 + 1, '.'); if (!p4) STRFAIL; + char *p5 = strchr (p4 + 1, '.'); if (!p5) STRFAIL; + char *p6 = strchr (p5 + 1, '.'); if (!p6) STRFAIL; + + *p6 = 0; + int myWarpID = atoi (p5 + 1); + + *p3 = 0; + *p4 = 0; + int myProjID = atoi (p2 + 1); + int myCellID = atoi (p3 + 1); + + int Nlo = warpIDs_bisection (warpIDs, myWarpID, Nfiles); + + int found = FALSE; + for (j = Nlo; !found && (j < Nfiles) && (warpIDs[j] <= myWarpID); j++) { + if (warpIDs[j] != myWarpID) continue; + if (projIDs[j] != myProjID) continue; + if (cellIDs[j] != myCellID) continue; + + // fprintf (stderr, "found it! (%d,%d,%d) = (%d,%d,%d) : %d\n", myWarpID, myProjID, myCellID, warpIDs[j], projIDs[j], cellIDs[j], fileIDs[j]); + + image[i].externID = fileIDs[j]; + image[i].sourceID = 34; + + found = TRUE; + } + if (!found) { + // fprintf (stderr, "did NOT find it! (%d,%d,%d)\n", myWarpID, myProjID, myCellID); + } + } + SaveImages(&db, imageFilenameNew, image, Nimage); + gfits_db_free (&db); + } + free (warpIDs); + free (projIDs); + free (cellIDs); + free (fileIDs); + + fclose (f); + ohana_memdump (TRUE); + + exit (0); +} + +int load_warp_ids (char *idfile, int *nfiles) { + + // load the warp ids + FILE *f = fopen (idfile, "r"); + myAssert (f, "failed to open warp file"); + + int NBUFFER = 30000000; + + char *buffer; + ALLOCATE (buffer, char, NBUFFER); + + int Nfiles = 0; + int NFILES = 1000; + ALLOCATE (warpIDs, int, NFILES); + ALLOCATE (projIDs, int, NFILES); + ALLOCATE (cellIDs, int, NFILES); + ALLOCATE (fileIDs, int, NFILES); + + int skipFirst = TRUE; + + int Nstart = 0; + int Ntotal = 0; + while (TRUE) { + int Nbytes = NBUFFER - 1 - Nstart; + bzero (&buffer[Nstart], Nbytes + 1); + + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + + Ntotal += Nread; + fprintf (stderr, "reading block: %d %d %d %d\n", Nstart, Nbytes, Nread, Ntotal); + + if (ferror (f)) { + perror ("error reading data file"); + break; + } + if (Nread == 0) break; // end of the file + + int Nlines = 0; + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + while (bufferStatus) { + char *c1 = strchr (c0, '\n'); // find the end of this current line + if (!c1) { + Nstart = strlen (c0); + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + *c1 = 0; // mark the end of the line + Nlines ++; + + if (skipFirst) { + skipFirst = FALSE; + c0 = c1 + 1; + continue; + } + + int warp_id, proj_id, cell_id, file_id; + int Nscan = sscanf (c0, "%d skycell.%d.%d %d", &warp_id, &proj_id, &cell_id, &file_id); + myAssert (Nscan == 4, "invalid line"); + + warpIDs[Nfiles] = warp_id; + projIDs[Nfiles] = proj_id; + cellIDs[Nfiles] = cell_id; + fileIDs[Nfiles] = file_id; + Nfiles ++; + + if (Nfiles == NFILES) { + NFILES += 1000; + REALLOCATE (warpIDs, int, NFILES); + REALLOCATE (projIDs, int, NFILES); + REALLOCATE (cellIDs, int, NFILES); + REALLOCATE (fileIDs, int, NFILES); + } + c0 = c1 + 1; + } + } + free (buffer); + fclose (f); + + isortfour (warpIDs, projIDs, cellIDs, fileIDs, Nfiles); + + *nfiles = Nfiles; + return TRUE; +} + +// return the index of the last value < threshold +int warpIDs_bisection (int *values, int threshold, int Nvalues) { + + int Nlo = 0; + int Nhi = Nvalues - 1; + + if (Nvalues < 1) return (-1); + if (threshold < values[Nlo]) return (-1); + + if (Nvalues < 2) return (0); + if (threshold > values[Nhi]) return (-1); + + int N; + while (Nhi - Nlo > 4) { + N = 0.5*(Nlo + Nhi); + if (values[N] < threshold) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, Nvalues - 1); + } + } + // values[Nlo] < threshold + // values[Nhi] >= threshold + + for (N = Nlo; N < Nhi; N++) { + if (values[N] >= threshold) { + return (N-1); + } + } + return (N); +} Index: trunk/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c (revision 38986) @@ -39,4 +39,5 @@ } + SetSignals (); catdir = argv[1]; Ntol = atoi(argv[2]); Index: trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c (revision 38986) +++ trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c (revision 38986) @@ -0,0 +1,42 @@ +# include "dvomerge.h" + +int dvorepair_by_objID (int argc, char **argv) { + + int N = get_argument (argc, argv, "-fix-cpt-by-objID"); + if (N != 1) { + fprintf (stderr, "-fix-cpt must be first from main\n"); + exit (2); + } + remove_argument (N, &argc, argv); + + if (argc != 2) { + fprintf (stderr, "USAGE: dvorepair -fix-cpt-by-objID (cptfile)\n"); + exit (2); + } + + char *cptFile = argv[1]; + + Catalog catalog; + + // set the parameters which guide catalog open/load/create + dvo_catalog_init (&catalog, TRUE); + catalog.filename = cptFile; + + // always load all of the data (if any exists) + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + + if (!dvo_catalog_open (&catalog, NULL, VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open catalog file %s\n", cptFile); + exit (2); + } + + repair_catalog_by_objID (&catalog); + + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save catalog %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock catalog %s\n", catalog.filename); exit (1); } + + return (TRUE); +} + Index: trunk/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c (revision 38986) @@ -82,6 +82,9 @@ catalog.Nsecfilt_disk = Nsecfilt * catalog.Naverage_disk; - dvo_catalog_save (&catalog, VERBOSE); - dvo_catalog_unlock (&catalog); + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + dvo_catalog_free (&catalog); } Index: trunk/Ohana/src/dvomerge/src/dvoutils.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoutils.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoutils.c (revision 38986) @@ -4,4 +4,5 @@ // check various options + SetSignals (); dvoutils_args (&argc, argv); Index: trunk/Ohana/src/dvomerge/src/dvoutils_load_image_index.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoutils_load_image_index.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoutils_load_image_index.c (revision 38986) @@ -2,8 +2,8 @@ // load the array of EXTERN_ID from the ImageIndex.fits tables -int *dvoutils_load_image_index (char *filename, int *nindex) { +ImageData *dvoutils_load_image_index (char *filename) { int Ncol; - off_t Nrow; + off_t Nrow, Nphot; Header header; Header theader; @@ -46,6 +46,14 @@ fclose (f); - int *externID = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol); + ImageData *imdata = NULL; + ALLOCATE (imdata, ImageData, 1); + + imdata->externID = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol); myAssert (!strcmp(type, "int"), "wrong column type"); + + imdata->photcode = gfits_get_bintable_column_data (&theader, &ftable, "PHOTCODE", type, &Nphot, &Ncol); + myAssert (Nphot == Nrow, "photcode & extern_id mis-match"); + + imdata->Nimages = Nrow; gfits_free_header (&header); @@ -54,7 +62,5 @@ gfits_free_table (&ftable); - *nindex = Nrow; - - return externID; + return imdata; } Index: trunk/Ohana/src/dvomerge/src/dvoutils_uniq_images.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoutils_uniq_images.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoutils_uniq_images.c (revision 38986) @@ -40,8 +40,6 @@ } - int Nimage; - // Image *image = dvoutils_load_image_table (imFile, &Nimage); - int *extern_id = dvoutils_load_image_index (imFile, &Nimage); - if (!extern_id) continue; + ImageData *imdata = dvoutils_load_image_index (imFile); + if (!imdata) continue; DBlist[Ndb] = strcreate (imFile); @@ -57,11 +55,22 @@ int i; - for (i = 0; i < Nimage; i++) { - if (!extern_id[i]) continue; - myMaxID = MAX(extern_id[i], myMaxID); - myMinID = MIN(extern_id[i], myMinID); + int Nbad = 0; + for (i = 0; i < imdata->Nimages; i++) { + if (!imdata->photcode[i] && imdata->externID[i]) myAbort ("invalid combo"); + + if (!imdata->photcode[i]) continue; + if (!imdata->externID[i]) { + Nbad ++; + continue; + } + myMaxID = MAX(imdata->externID[i], myMaxID); + myMinID = MIN(imdata->externID[i], myMinID); } - fprintf (stderr, "read %s, %d images, %d - %d vs %d - %d\n", imFile, (int) Nimage, myMinID, myMaxID, minID, maxID); + fprintf (stderr, "read %s, %d images, %d - %d vs %d - %d\n", imFile, (int) imdata->Nimages, myMinID, myMaxID, minID, maxID); + + if (Nbad) { + fprintf (stderr, "WARNING: %d images without extern_id set\n", (int) Nbad); + } // if this is new, we treat it a bit differently @@ -124,7 +133,7 @@ } - for (i = 0; i < Nimage; i++) { - if (!extern_id[i]) continue; - int n = extern_id[i] - minID; + for (i = 0; i < imdata->Nimages; i++) { + if (!imdata->externID[i]) continue; + int n = imdata->externID[i] - minID; if (IDlist[n] == 0) { DBfound[n] = Ndb; @@ -133,5 +142,5 @@ IDlist[n] ++; if (VERBOSE && (IDlist[n] > 1)) { - fprintf (stderr, "duplicate ext_id %d in file %s\n", extern_id[i], imFile); + fprintf (stderr, "duplicate ext_id %d in file %s\n", imdata->externID[i], imFile); } } @@ -140,7 +149,7 @@ fprintf (stderr, "------------ %s ----------\n", imFile); - for (i = 0; i < Nimage; i++) { - if (!extern_id[i]) continue; - fprintf (stderr, "%d => %d\n", extern_id[i], extern_id[i] - minID); + for (i = 0; i < imdata->Nimages; i++) { + if (!imdata->externID[i]) continue; + fprintf (stderr, "%d => %d\n", imdata->externID[i], imdata->externID[i] - minID); } @@ -165,5 +174,5 @@ int n = DBfound[i]; myAssert (n > -1, "impossible!"); - fprintf (stdout, "%d : %d -- in %s (%d) : %d \n", i + minID, IDlist[i], DBlist[n], DBfound[i], DBindex[i]); + fprintf (stdout, "primary %d : %d -- in %s (%d) : %d \n", i + minID, IDlist[i], DBlist[n], DBfound[i], DBindex[i]); } Index: trunk/Ohana/src/dvomerge/src/dvoverify.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoverify.c (revision 38986) @@ -14,4 +14,5 @@ // check various options + SetSignals (); dvoverify_args (&argc, argv); CATDIR = argv[1]; @@ -78,6 +79,6 @@ SkyListFree (skylist); - ohana_memcheck (TRUE); - ohana_memdump (TRUE); + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); exit (0); } Index: trunk/Ohana/src/dvomerge/src/dvoverify_args.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify_args.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoverify_args.c (revision 38986) @@ -24,4 +24,11 @@ if ((N = get_argument (*argc, argv, "-sorted"))) { CHECKSORTED = TRUE; + remove_argument (N, argc, argv); + } + + // in some cases, the header sorted state can claim F but is actually T + IGNORE_SORTED_STATE = FALSE; + if ((N = get_argument (*argc, argv, "-ignore-sorted-state"))) { + IGNORE_SORTED_STATE = TRUE; remove_argument (N, argc, argv); } @@ -50,6 +57,6 @@ free (filename); - ohana_memcheck (TRUE); - ohana_memdump (TRUE); + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); if (!isGood) exit (1); @@ -204,4 +211,11 @@ } + // in some cases, the header sorted state can claim F but is actually T + IGNORE_SORTED_STATE = FALSE; + if ((N = get_argument (*argc, argv, "-ignore-sorted-state"))) { + IGNORE_SORTED_STATE = TRUE; + remove_argument (N, argc, argv); + } + if (!HOST_ID || !HOSTDIR || (*argc != 2)) { fprintf (stderr, "USAGE: dvoverify_client (catdir) -results (file) -hostID ID -hostdir DIR [-region Rmin Rmax Dmin Dmax] [-v] [-s]\n\n"); Index: trunk/Ohana/src/dvomerge/src/dvoverify_catalogs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify_catalogs.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoverify_catalogs.c (revision 38986) @@ -117,15 +117,14 @@ // options / arguments that can affect relastro_client -update-objects: - char command[DVO_MAX_PATH]; - snprintf (command, DVO_MAX_PATH, "dvoverify_client %s -results %s -hostID %d -hostdir %s -region %f %f %f %f", - abscatdir, table->hosts[i].results, table->hosts[i].hostID, table->hosts[i].pathname, - UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax - ); + char *command = NULL; + strextend (&command, "dvoverify_client %s -results %s -hostID %d -hostdir %s -region %f %f %f %f", + abscatdir, table->hosts[i].results, table->hosts[i].hostID, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - char tmpline[DVO_MAX_PATH]; - if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } - if (CHECKSORTED) { snprintf (tmpline, DVO_MAX_PATH, "%s -s", command); strcpy (command, tmpline); } - if (!CHECK_IMAGE_ID) { snprintf (tmpline, DVO_MAX_PATH, "%s -skip-image-ids", command); strcpy (command, tmpline); } - if (LIST_MISSING) { snprintf (tmpline, DVO_MAX_PATH, "%s -list-missing", command); strcpy (command, tmpline); } + if (VERBOSE) { strextend (&command, "-v"); } + if (CHECKSORTED) { strextend (&command, "-s"); } + if (IGNORE_SORTED_STATE) { strextend (&command, "-ignore-sorted-state"); } + if (!CHECK_IMAGE_ID) { strextend (&command, "-skip-image-ids"); } + if (LIST_MISSING) { strextend (&command, "-list-missing"); } fprintf (stderr, "command: %s\n", command); @@ -149,4 +148,5 @@ table->hosts[i].pid = pid; // save for future reference } + free (command); } Index: trunk/Ohana/src/dvomerge/src/dvoverify_client.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify_client.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoverify_client.c (revision 38986) @@ -8,4 +8,5 @@ // check various options + SetSignals (); dvoverify_client_args (&argc, argv); CATDIR = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvoverify_utils.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify_utils.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/dvoverify_utils.c (revision 38986) @@ -188,5 +188,5 @@ dvo_catalog_init (&catalog, TRUE); catalog.filename = filename; - catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; catalog.Nsecfilt = 0; @@ -214,5 +214,5 @@ // if the table is NOT SORTED, do we have a subset of checks we can make? - if (!catalog.sorted) { + if (!catalog.sorted && !IGNORE_SORTED_STATE) { fprintf (stderr, "!"); dvo_catalog_unlock (&catalog); @@ -222,83 +222,37 @@ return TRUE; } - - int NmeasureTotal = 0; - int measureOffsetOK = TRUE; - int NlensingTotal = 0; - int lensingOffsetOK = TRUE; - int NlensobjTotal = 0; - int lensobjOffsetOK = TRUE; - for (i = 0; i < catalog.Naverage; i++) { - NmeasureTotal += catalog.average[i].Nmeasure; - if (VERBOSE && !(NmeasureTotal <= catalog.Nmeasure)) { - fprintf (stderr, "NmeasureTotal > catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure); - } - measureOffsetOK &= (catalog.average[i].measureOffset < catalog.Nmeasure); - if (VERBOSE && !(catalog.average[i].measureOffset < catalog.Nmeasure)) { - fprintf (stderr, "measureOffset >= catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].measureOffset, (int) catalog.Nmeasure); - } - measureOffsetOK &= (catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure); - if (VERBOSE && !(catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure)) { - fprintf (stderr, "measureOffset + Nmeasure > catalog.Nmeasure : %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure); - } - - // sum of Nlensing for each object < Nlensing for catalog - NlensingTotal += catalog.average[i].Nlensing; - if (VERBOSE && !(NlensingTotal <= catalog.Nlensing)) { - fprintf (stderr, "NlensingTotal > catalog.Nlensing: %d %d %d\n", i, catalog.average[i].Nlensing, (int) catalog.Nlensing); - } - // lensingOffset needs to be in range for each object - lensingOffsetOK &= (!catalog.Nlensing || (catalog.average[i].lensingOffset < catalog.Nlensing)); - if (VERBOSE && catalog.Nlensing && !(catalog.average[i].lensingOffset < catalog.Nlensing)) { - fprintf (stderr, "lensingOffset >= catalog.Nlensing: %d %d %d\n", i, catalog.average[i].lensingOffset, (int) catalog.Nlensing); - } - lensingOffsetOK &= (catalog.average[i].lensingOffset + catalog.average[i].Nlensing <= catalog.Nlensing); - if (VERBOSE && !(catalog.average[i].lensingOffset + catalog.average[i].Nlensing <= catalog.Nlensing)) { - fprintf (stderr, "lensingOffset + Nlensing > catalog.Nlensing : %d %d %d\n", i, catalog.average[i].Nlensing, (int) catalog.Nlensing); - } - - NlensobjTotal += catalog.average[i].Nlensobj; - if (VERBOSE && !(NlensobjTotal <= catalog.Nlensobj)) { - fprintf (stderr, "NlensobjTotal > catalog.Nlensobj: %d %d %d\n", i, catalog.average[i].Nlensobj, (int) catalog.Nlensobj); - } - lensobjOffsetOK &= (catalog.average[i].lensobjOffset < catalog.Nlensobj); - if (VERBOSE && !(catalog.average[i].lensobjOffset < catalog.Nlensobj)) { - fprintf (stderr, "lensobjOffset >= catalog.Nlensobj: %d %d %d\n", i, catalog.average[i].lensobjOffset, (int) catalog.Nlensobj); - } - lensobjOffsetOK &= (catalog.average[i].lensobjOffset + catalog.average[i].Nlensobj <= catalog.Nlensobj); - if (VERBOSE && !(catalog.average[i].lensobjOffset + catalog.average[i].Nlensobj <= catalog.Nlensobj)) { - fprintf (stderr, "lensobjOffset + Nlensobj > catalog.Nlensobj : %d %d %d\n", i, catalog.average[i].Nlensobj, (int) catalog.Nlensobj); - } - } - - if (!measureOffsetOK) { - fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog.filename); - status = FALSE; - } - - if (NmeasureTotal != catalog.Nmeasure) { - fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog.filename); - status = FALSE; - } - - if (!lensingOffsetOK) { - fprintf (stderr, "ERROR: catalog %s has an invalid lensingOffset\n", catalog.filename); - status = FALSE; - } - - if (NlensingTotal != catalog.Nlensing) { - fprintf (stderr, "ERROR: catalog %s has an invalid Nlensing\n", catalog.filename); - status = FALSE; - } - - if (!lensobjOffsetOK) { - fprintf (stderr, "ERROR: catalog %s has an invalid lensobjOffset\n", catalog.filename); - status = FALSE; - } - - if (NlensobjTotal != catalog.Nlensobj) { - fprintf (stderr, "ERROR: catalog %s has an invalid Nlensobj\n", catalog.filename); - status = FALSE; - } + +# define CHECK_TABLE(NAME) { \ + int Ntotal = 0; int offsetOK = TRUE; int isOK = FALSE; \ + for (i = 0; i < catalog.Naverage; i++) { \ + Ntotal += catalog.average[i].N##NAME; \ + if (VERBOSE && !(Ntotal <= catalog.N##NAME)) { \ + fprintf (stderr, "Ntotal (%s) > catalog.N##NAME: %d %d : %d > %d\n", #NAME, i, Ntotal, catalog.average[i].N##NAME, (int) catalog.N##NAME); \ + } \ + isOK = (catalog.average[i].N##NAME == 0) || (catalog.average[i].NAME##Offset < catalog.N##NAME); \ + offsetOK &= isOK; \ + if (VERBOSE && !isOK) { \ + fprintf (stderr, "%sOffset >= catalog.N%s: %d : %d >= %d\n", #NAME, #NAME, i, catalog.average[i].NAME##Offset, (int) catalog.N##NAME); \ + } \ + isOK = (catalog.average[i].N##NAME == 0) || (catalog.average[i].NAME##Offset + catalog.average[i].N##NAME <= catalog.N##NAME); \ + offsetOK &= isOK; \ + if (VERBOSE && !isOK) { \ + fprintf (stderr, "%sOffset + N%s > catalog.N%s : %d : %d + %d > %d\n", #NAME, #NAME, #NAME, i, catalog.average[i].N##NAME, catalog.average[i].NAME##Offset, (int) catalog.N##NAME); \ + } } \ + if (!offsetOK) { \ + fprintf (stderr, "ERROR: catalog %s has an invalid %sOffset\n", catalog.filename, #NAME); \ + status = FALSE; \ + } \ + if (Ntotal != catalog.N##NAME) { \ + fprintf (stderr, "ERROR: catalog %s has an invalid N%s\n", catalog.filename, #NAME); \ + status = FALSE; \ + } \ + } + + CHECK_TABLE(measure); + CHECK_TABLE(lensing); + CHECK_TABLE(lensobj); + CHECK_TABLE(starpar); + CHECK_TABLE(galphot); // if we have a problem with Nmeasure and/or measureOffset values, we @@ -312,27 +266,47 @@ // check measure <-> average links { - int objIDsOK = TRUE; - int catIDsOK = TRUE; - int averefOK = TRUE; + int NobjIDsBAD = 0; + int NcatIDsBAD = 0; + int NaverefBAD = 0; for (i = 0; i < catalog.Naverage; i++) { m = catalog.average[i].measureOffset; for (j = 0; j < catalog.average[i].Nmeasure; j++) { - objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID); - catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID); - averefOK &= (catalog.measure[m+j].averef == i); + if (catalog.average[i].objID != catalog.measure[m+j].objID) { + NobjIDsBAD ++; + + // check if objID matches R,D + Measure *measure = &catalog.measure[m+j]; + int iTest = measure->objID; + if (catalog.average[iTest].objID == iTest) { + double dR = 3600.0*(catalog.average[iTest].R - measure->R) * cos (RAD_DEG*measure->D); + double dD = 3600.0*(catalog.average[iTest].D - measure->D); + double dRad = hypot (dR, dD); + char *date = ohana_sec_to_date (measure->t); + fprintf (stderr, "matches ave %d = %d, %s : %6.3f %5d : %f, %f = %f\n", iTest, measure->objID, date, measure->M, measure->photcode, dR, dD, dRad); + free (date); + } else { + fprintf (stderr, "cannot find averef\n"); + } + } + if (catalog.average[i].catID != catalog.measure[m+j].catID) { + NcatIDsBAD ++; + } + if (catalog.measure[m+j].averef != i) { + NaverefBAD ++; + } } } - if (!objIDsOK) { - fprintf (stderr, "ERROR: catalog %s has invalid obj IDs\n", catalog.filename); - status = FALSE; - } - if (!catIDsOK) { - fprintf (stderr, "ERROR: catalog %s has invalid cat IDs\n", catalog.filename); - status = FALSE; - } - if (!averefOK) { - fprintf (stderr, "ERROR: catalog %s has invalid averef values\n", catalog.filename); + if (NobjIDsBAD) { + fprintf (stderr, "ERROR: catalog %s has %d invalid obj IDs\n", catalog.filename, NobjIDsBAD); + status = FALSE; + } + if (NcatIDsBAD) { + fprintf (stderr, "ERROR: catalog %s has %d invalid cat IDs\n", catalog.filename, NcatIDsBAD); + status = FALSE; + } + if (NaverefBAD) { + fprintf (stderr, "ERROR: catalog %s has %d invalid averef values\n", catalog.filename, NaverefBAD); status = FALSE; } @@ -387,4 +361,62 @@ if (!catIDsOK) { fprintf (stderr, "ERROR: catalog %s has invalid lensobj cat IDs\n", catalog.filename); + status = FALSE; + } + } + + // check starpar <-> average links + { + int objIDsOK = TRUE; + int catIDsOK = TRUE; + int averefOK = TRUE; + + for (i = 0; i < catalog.Naverage; i++) { + m = catalog.average[i].starparOffset; + for (j = 0; j < catalog.average[i].Nstarpar; j++) { + objIDsOK &= (catalog.average[i].objID == catalog.starpar[m+j].objID); + catIDsOK &= (catalog.average[i].catID == catalog.starpar[m+j].catID); + averefOK &= (catalog.starpar[m+j].averef == i); + } + } + + if (!objIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid starpar obj IDs\n", catalog.filename); + status = FALSE; + } + if (!catIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid starpar cat IDs\n", catalog.filename); + status = FALSE; + } + if (!averefOK) { + fprintf (stderr, "ERROR: catalog %s has invalid starpar averef values\n", catalog.filename); + status = FALSE; + } + } + + // check galphot <-> average links + { + int objIDsOK = TRUE; + int catIDsOK = TRUE; + int averefOK = TRUE; + + for (i = 0; i < catalog.Naverage; i++) { + m = catalog.average[i].galphotOffset; + for (j = 0; j < catalog.average[i].Ngalphot; j++) { + objIDsOK &= (catalog.average[i].objID == catalog.galphot[m+j].objID); + catIDsOK &= (catalog.average[i].catID == catalog.galphot[m+j].catID); + averefOK &= (catalog.galphot[m+j].averef == i); + } + } + + if (!objIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid galphot obj IDs\n", catalog.filename); + status = FALSE; + } + if (!catIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid galphot cat IDs\n", catalog.filename); + status = FALSE; + } + if (!averefOK) { + fprintf (stderr, "ERROR: catalog %s has invalid galphot averef values\n", catalog.filename); status = FALSE; } Index: trunk/Ohana/src/dvomerge/src/help.c =================================================================== --- trunk/Ohana/src/dvomerge/src/help.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/help.c (revision 38986) @@ -172,4 +172,5 @@ fprintf (stderr, "USAGE\n"); + fprintf (stderr, " dvorepair -fix-warp-ids (catdir) (idfile) - regenerate images.dat warp_skyfile_ids (EXTERN_ID) values\n"); fprintf (stderr, " dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n"); fprintf (stderr, " dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n"); Index: trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 38986) @@ -198,7 +198,6 @@ int Nreplace = 0; if (REPLACE_TYCHO) { - int Mout = output[0].average[n].measureOffset; int Minp = input[0].average[N].measureOffset; - Nreplace = replace_tycho (&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[Minp]); + Nreplace = replace_tycho (&output[0].average[n], output[0].measure, next_meas, &input[0].average[N], &input[0].measure[Minp]); if (Nreplace == 6) { input[0].found_t[N] = Nmeas; @@ -216,7 +215,6 @@ // index to first measure for this object // XXX this does not support lensing measurements - int Mout = output[0].average[n].measureOffset; - if (replace_match(&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[offset])) { - input[0].found_t[N] = Mout; + if (replace_match (&output[0].average[n], output[0].measure, next_meas, &input[0].average[N], &input[0].measure[offset])) { + input[0].found_t[N] = output[0].average[n].measureOffset; continue; } Index: trunk/Ohana/src/dvomerge/src/repair_catalog_by_objID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/repair_catalog_by_objID.c (revision 38986) +++ trunk/Ohana/src/dvomerge/src/repair_catalog_by_objID.c (revision 38986) @@ -0,0 +1,118 @@ +# include "dvomerge.h" + +int repair_catalog_by_objID (Catalog *catalog) { + + // first find the entries which are bad (there may be none) + int Nbad = 0; + + Average *average = catalog->average; + Measure *measure = catalog->measure; + + off_t i, j; + for (i = 0; i < catalog->Naverage; i++) { + off_t m = average[i].measureOffset; + for (j = 0; j < average[i].Nmeasure; j++) { + if (average[i].objID == measure[m+j].objID) continue; + + // this is a bad entry. now we need to decide if it can be repaired + + Measure *myMeasure = &measure[m+j]; + int iTest = myMeasure->objID; + myAssert (average[iTest].objID == iTest, "cannot find target source"); + + double dR = 3600.0*(average[iTest].R - myMeasure->R) * cos (RAD_DEG*myMeasure->D); + double dD = 3600.0*(average[iTest].D - myMeasure->D); + double dRad = hypot (dR, dD); + + // XXX dRad of 200.0 would allow Barnard's star (10"/yr) to be handled + myAssert (dRad < 200.0, "bad detection does not seem to match target source"); + + char *date = ohana_sec_to_date (myMeasure->t); + fprintf (stderr, "bad detection found: matches averef %d, objID %d, %s : %6.3f %5d : %f, %f = %f\n", iTest, myMeasure->objID, date, myMeasure->M, myMeasure->photcode, dR, dD, dRad); + free (date); + Nbad ++; + } + } + if (!Nbad) return TRUE; + + // we have at least one bad detection + // make a matched list of the measurement sequence and the objID values + + off_t Nmeasure = catalog->Nmeasure; + ALLOCATE_PTR (seq, off_t, Nmeasure); + ALLOCATE_PTR (objID, off_t, Nmeasure); + + for (i = 0; i < Nmeasure; i++) { + seq[i] = i; + objID[i] = measure[i].objID; + } + + // sort seq by objID + llsortpair (objID, seq, Nmeasure); + + // create a new array of measures in objID order + Measure *measureNew = NULL; + ALLOCATE (measureNew, Measure, Nmeasure); + + for (i = 0; i < Nmeasure; i++) { + j = seq[i]; + measureNew[i] = measure[j]; + } + + // reset averef and Nmeasure values for the average entries + off_t N = 0; + off_t myObjID = objID[0]; + off_t averef = 0; + average[averef].measureOffset = 0; + for (i = 0; i < Nmeasure; i++) { + if (objID[i] != myObjID) { + // we have hit the next entry in the list + average[averef].Nmeasure = N; + N = 0; + myObjID = objID[i]; + averef ++; + average[averef].measureOffset = i; + } + if (measureNew[i].averef != averef) { + fprintf (stderr, "fixing averef for measure "OFF_T_FMT" (%d -> "OFF_T_FMT")\n", i, measureNew[i].averef, averef); + measureNew[i].averef = averef; + } + N++; + } + average[averef].Nmeasure = N; + + free (catalog->measure); + catalog->measure = measureNew; + + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + for (i = 0; i < catalog->Naverage; i++) { + NmeasureTotal += catalog[0].average[i].Nmeasure; + if (!(NmeasureTotal <= catalog[0].Nmeasure)) { + fprintf (stderr, "too many measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure); + if (!(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure); + if (!(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) { + fprintf (stderr, "offset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + } + + int status = TRUE; + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename); + status = FALSE; + } + + if (NmeasureTotal != catalog[0].Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename); + status = FALSE; + } + + if (!status) exit (2); + + return TRUE; +} Index: trunk/Ohana/src/dvomerge/src/replace_match.c =================================================================== --- trunk/Ohana/src/dvomerge/src/replace_match.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/replace_match.c (revision 38986) @@ -1,6 +1,10 @@ # include "dvomerge.h" +// XXX this function does not work because it it loops over the average->Nmeasure entries +// as if they were sorted, but objects which have had entries added to them have elements +// out of sequence. I probably need to step through next_meas to find the sequence. + // average and measure pointers to the objects of interest and their first / current measurement -int replace_match (Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in) { +int replace_match (Average *average_out, Measure *measure_out, off_t *next_meas, Average *average_in, Measure *measure_in) { int Nout; @@ -8,12 +12,18 @@ unsigned int catID; + off_t m = average_out->measureOffset; + // find the matching photcode in the object's list of measurements for (Nout = 0; Nout < average_out[0].Nmeasure; Nout ++) { - if (measure_out[Nout].photcode != measure_in[0].photcode) continue; + myAssert (m > -1, "oops"); + if (measure_out[m].photcode != measure_in[0].photcode) { + m = next_meas[m]; + continue; + } // set the new measurements - averef = measure_out[Nout].averef; - catID = measure_out[Nout].catID; - measure_out[Nout] = measure_in[0]; + averef = measure_out[m].averef; + catID = measure_out[m].catID; + measure_out[m] = measure_in[0]; // old code: find R,D using average_in[0], the get offset relative to average_out[0]. no longer @@ -24,20 +34,20 @@ // measure_out[Nout].dD = 3600.0*(average_out[0].D - Din); - measure_out[Nout].dbFlags = 0; // XXX why reset these? - measure_out[Nout].averef = averef; - measure_out[Nout].objID = average_out[0].objID; - measure_out[Nout].catID = catID; + measure_out[m].dbFlags = 0; // XXX why reset these? + measure_out[m].averef = averef; + measure_out[m].objID = average_out[0].objID; + measure_out[m].catID = catID; - float dRoff = dvoOffsetR(&measure_out[Nout], average_out); + float dRoff = dvoOffsetR(&measure_out[m], average_out); // rationalize dR if (dRoff > +180.0*3600.0) { // average on high end of boundary, move star up - measure_out[Nout].R += 360.0; + measure_out[m].R += 360.0; dRoff -= 360.0*3600.0; } if (dRoff < -180.0*3600.0) { // average on low end of boundary, move star down - measure_out[Nout].R -= 360.0; + measure_out[m].R -= 360.0; dRoff += 360.0*3600.0; } @@ -49,5 +59,5 @@ if (fabs(dRoff*cosD) > 10*RADIUS) { fprintf (stderr, "surprisingly distant detection: %10.6f,%10.6f vs %10.6f,%10.6f\n", - average_out[0].R, average_out[0].D, measure_out[Nout].R, measure_out[Nout].D); + average_out[0].R, average_out[0].D, measure_out[m].R, measure_out[m].D); } } Index: trunk/Ohana/src/dvomerge/src/replace_tycho.c =================================================================== --- trunk/Ohana/src/dvomerge/src/replace_tycho.c (revision 38553) +++ trunk/Ohana/src/dvomerge/src/replace_tycho.c (revision 38986) @@ -10,5 +10,5 @@ } -int replace_tycho (Average *averageOut, Measure *measureOut, Average *averageInp, Measure *measureInp) { +int replace_tycho (Average *averageOut, Measure *measureOut, off_t *next_meas, Average *averageInp, Measure *measureInp) { int i; @@ -17,14 +17,19 @@ int index[6]; + off_t m = averageOut->measureOffset; + // find all tycho measurements for (i = 0; (i < averageOut->Nmeasure) && (Nindex < 6); i++) { + myAssert (m > -1, "oops"); + int valid = FALSE; - valid = valid || (measureOut[i].photcode == TYCHO_B); - valid = valid || (measureOut[i].photcode == TYCHO_V); - if (!valid) continue; - - index[Nindex] = i; - Nindex ++; + valid = valid || (measureOut[m].photcode == TYCHO_B); + valid = valid || (measureOut[m].photcode == TYCHO_V); + if (valid) { + index[Nindex] = m; + Nindex ++; + } + m = next_meas[m]; }