Index: trunk/Ohana/src/dvomerge/Makefile =================================================================== --- trunk/Ohana/src/dvomerge/Makefile (revision 33657) +++ trunk/Ohana/src/dvomerge/Makefile (revision 33963) @@ -1,5 +1,5 @@ default: all help: - @echo "make options: dvomerge (default)" + @echo "make options: dvomerge dvomerge_client dvoconvert (default)" include ../../Makefile.System @@ -17,8 +17,9 @@ FULL_LDFLAGS = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS) -dvomerge : $(BIN)/dvomerge.$(ARCH) -dvomerge : $(BIN)/dvoconvert.$(ARCH) +dvomerge : $(BIN)/dvomerge.$(ARCH) +dvomerge_client : $(BIN)/dvomerge_client.$(ARCH) +dvoconvert : $(BIN)/dvoconvert.$(ARCH) -all: dvomerge dvoconvert dvosecfilt +all: dvomerge dvomerge_client dvoconvert dvosecfilt # $(SRC)/dvomergeContinue.$(ARCH).o @@ -28,8 +29,10 @@ $(SRC)/dvomergeUpdate.$(ARCH).o \ $(SRC)/dvomergeCreate.$(ARCH).o \ +$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \ $(SRC)/dvomergeUpdate_threaded.$(ARCH).o \ $(SRC)/dvomergeFromList.$(ARCH).o \ $(SRC)/dvomergeImageIDs.$(ARCH).o \ $(SRC)/dvo_image_merge_dbs.$(ARCH).o \ +$(SRC)/IDmapIO.$(ARCH).o \ $(SRC)/SetSignals.$(ARCH).o \ $(SRC)/ConfigInit.$(ARCH).o \ @@ -41,4 +44,5 @@ $(SRC)/LoadCatalog.$(ARCH).o \ $(SRC)/build_links.$(ARCH).o \ +$(SRC)/replace_match.$(ARCH).o \ $(SRC)/merge_catalogs_new.$(ARCH).o \ $(SRC)/merge_catalogs_old.$(ARCH).o @@ -47,4 +51,27 @@ $(BIN)/dvomerge.$(ARCH) : $(DVOMERGE) + +DVOMERGE_CLIENT = \ +$(SRC)/dvomerge_client.$(ARCH).o \ +$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \ +$(SRC)/dvomergeImageIDs.$(ARCH).o \ +$(SRC)/dvo_image_merge_dbs.$(ARCH).o \ +$(SRC)/IDmapIO.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/help.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o \ +$(SRC)/SkyTablePopulatedRange.$(ARCH).o \ +$(SRC)/LoadCatalog.$(ARCH).o \ +$(SRC)/build_links.$(ARCH).o \ +$(SRC)/replace_match.$(ARCH).o \ +$(SRC)/merge_catalogs_new.$(ARCH).o \ +$(SRC)/merge_catalogs_old.$(ARCH).o + +$(DVOMERGE_CLIENT) : $(INC)/dvomerge.h + +$(BIN)/dvomerge_client.$(ARCH) : $(DVOMERGE_CLIENT) DVOCONVERT = \ @@ -99,5 +126,5 @@ $(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY) -INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify +INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify # dependancy rules for binary code ######################### Index: trunk/Ohana/src/dvomerge/doc/failsafe.txt =================================================================== --- trunk/Ohana/src/dvomerge/doc/failsafe.txt (revision 33963) +++ trunk/Ohana/src/dvomerge/doc/failsafe.txt (revision 33963) @@ -0,0 +1,84 @@ + +we have a problem with dvomerge and flaky hosts. the problem is that +an I/O glitch can prevent a table from being written. although we can +(and do?) catch the failure, we do not have a good recovery method. +recovery would involve 2 things: + +1) we should revert the failed table to the original state +2) we should be able to re-run the merge correctly + +point (1) implies that we need a solid copy of the original file before +writing the new output tables. + +point (2) implies that we need to know which tables have been +successfully merged. + +For point (1), since we are writing out the entire table, we can +*move* the original file to a temporary name, and unlink it when the +output is successful. + +For point (2), we should write an ID to the output header that comes +from the input catalog file. then a second attempt will skip any +tables for which the IDs already match. + +Problems & Questions for (2): + +* do we need to have the ID in each input db table? +* we do not have an ID already in existing tables -- I'll need a tool + to write the ID +OR +* can we use something like the name & date of the file to check the +* identity? + +(name, date, size can be determined without reading the file, so that +is quite tempting...) + +I'll still need to do 2 passes, on where I determine which output +tables match the given input table, then check those output tables for +the merge data. + +--- + +Proposal: + +1) Add the following data to the output catalogs: + +OBJID = 675 / +LMRG_DB = 'catdir.2mass' / name of last merged database +LMRG_ST = 80640 / size of last merged cpt file +LMRG_SS = 118080 / size of last merged cps file +LMRG_SM = 331200 / size of last merged cpm file +LMRG_DT = '2012-04-27 16:42:45' / mod-date of last merged cpm file + +This allows us to tell if the current catalog has been merged in yet, +but it does not require us to track any database IDs. If a file has +been merged, we skip it; otherwise, we merge it. + +The proposal may do the wrong thing in the following case: + +dvomerge db1 into db2 +(failure part way through) + +addstar file -D CATDIR db1 + +dvomerge db1 into db2 + +If 'file' overlaps the already-merged sections of db1, they will be +re-ingested. In fact, the first pass need not have failed -- it will +still double merge some data. + +However, it does mean the following does the right thing: + +dvomerge db1 into db2 + +dvomerge db1 into db2 + +(no merge should happen on the second pass) + +dvomerge db1 into db2 +(failure) + +dvomerge db1 into db2 + +(only unmerged entries should be merged on the second pass) + Index: trunk/Ohana/src/dvomerge/doc/parallel.txt =================================================================== --- trunk/Ohana/src/dvomerge/doc/parallel.txt (revision 33963) +++ trunk/Ohana/src/dvomerge/doc/parallel.txt (revision 33963) @@ -0,0 +1,8 @@ + +dvomergeUpdate parallel version: + + * master should update photcode table & image table + * master should call remote clients to update their catalog tables + * write out a temporary IDmap for the remote clients? + * put the inlist loop into a function which can be called by the client or the master + Index: trunk/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- trunk/Ohana/src/dvomerge/include/dvomerge.h (revision 33657) +++ trunk/Ohana/src/dvomerge/include/dvomerge.h (revision 33963) @@ -17,4 +17,14 @@ # include +# define DVO_MAX_PATH 1024 + +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +int HOST_ID; +char *HOSTDIR; + + int VERBOSE; char CATDIR[256]; @@ -26,12 +36,13 @@ int SKY_DEPTH; int NTHREADS; -char *ALTERNATE_PHOTCODE_FILE; +char *ALTERNATE_PHOTCODE_FILE; +int REPLACE_BY_PHOTCODE; SkyRegion UserPatch; // used by MODE CAT typedef struct { - off_t Nmap; - off_t *old; - off_t *new; + unsigned int Nmap; + unsigned int *old; + unsigned int *new; } IDmapType; @@ -51,4 +62,8 @@ void dvomerge_help PROTO((int argc, char **argv)); int dvomerge_args PROTO((int *argc, char **argv)); + +void dvomerge_client_usage PROTO((void)); +void dvomerge_client_help PROTO((int argc, char **argv)); +int dvomerge_client_args PROTO((int *argc, char **argv)); void dvoconvert_usage PROTO((void)); @@ -110,2 +125,8 @@ int dvomergeFromList PROTO((int argc, char **argv)); int dvomergeUpdate_threaded PROTO((int argc, char **argv)); +int dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap)); + +int replace_match PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in)); + +int IDmapSave PROTO((char *filename, IDmapType *IDmap)); +IDmapType *IDmapLoad PROTO((char *filename)); Index: trunk/Ohana/src/dvomerge/src/IDmapIO.c =================================================================== --- trunk/Ohana/src/dvomerge/src/IDmapIO.c (revision 33963) +++ trunk/Ohana/src/dvomerge/src/IDmapIO.c (revision 33963) @@ -0,0 +1,137 @@ +# include "dvomerge.h" + +# define GET_COLUMN(OUT,NAME,TYPE) \ + OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +// write out the IDmap data for clients to read +int IDmapSave(char *filename, IDmapType *IDmap) { + + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_ID_MAP"); + + gfits_define_bintable_column (&theader, "J", "OLD_IDS", "old image IDs", NULL, 1.0, 1.0*0x8000); + gfits_define_bintable_column (&theader, "J", "NEW_IDS", "new image IDs", NULL, 1.0, 1.0*0x8000); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // add the columns to the output array + // XXX does this damage the data (due to byte swaps?) + // XXX does it matter? + gfits_set_bintable_column (&theader, &ftable, "OLD_IDS", IDmap->old, IDmap->Nmap); + gfits_set_bintable_column (&theader, &ftable, "NEW_IDS", IDmap->new, IDmap->Nmap); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open ID map file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for image subset %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for image subset %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for image subset %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for image subset %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing image subset file %s\n", filename); + + return TRUE; +} + +IDmapType *IDmapLoad (char *filename) { + + int Ncol; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + IDmapType *IDmap; + + ALLOCATE (IDmap, IDmapType, 1); + IDmap->Nmap = 0; + IDmap->old = NULL; + IDmap->new = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return (NULL); + } + fclose (f); + + char type[16]; + + GET_COLUMN (IDmap->old, "OLD_IDS", int); + GET_COLUMN (IDmap->new, "NEW_IDS", int); + IDmap->Nmap = Nrow; + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + return IDmap; +} + Index: trunk/Ohana/src/dvomerge/src/args.c =================================================================== --- trunk/Ohana/src/dvomerge/src/args.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/args.c (revision 33963) @@ -15,4 +15,11 @@ remove_argument (N, argc, argv); ALTERNATE_PHOTCODE_FILE = strcreate(argv[N]); + remove_argument (N, argc, argv); + } + + /* replace measurement, don't duplicate */ + REPLACE_BY_PHOTCODE = FALSE; + if ((N = get_argument (*argc, argv, "-replace"))) { + REPLACE_BY_PHOTCODE = TRUE; remove_argument (N, argc, argv); } @@ -43,5 +50,102 @@ } + // XXX for the moment, make this selection manual. it needs to be automatic + // based on the state of the SkyTable + PARALLEL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, argc, argv); + } + if ((*argc < 4) || (*argc > 6)) dvomerge_usage(); + return TRUE; +} + +/*** check for command line options ***/ +int dvomerge_client_args (int *argc, char **argv) { + + int N; + + // by definition, the client is not parallel + PARALLEL = FALSE; + PARALLEL_MANUAL = FALSE; + PARALLEL_SERIAL = FALSE; + + HOST_ID = 0; + if ((N = get_argument (*argc, argv, "-hostID"))) { + remove_argument (N, argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, argc, argv); + } + if (!HOST_ID) dvomerge_client_usage(); + + HOSTDIR = NULL; + if ((N = get_argument (*argc, argv, "-hostdir"))) { + remove_argument (N, argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, argc, argv); + } + if (!HOSTDIR) dvomerge_client_usage(); + + /* extra error messages */ + VERBOSE = FALSE; + if ((N = get_argument (*argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, argc, argv); + } + + /* replace measurement, don't duplicate */ + REPLACE_BY_PHOTCODE = FALSE; + if ((N = get_argument (*argc, argv, "-replace"))) { + REPLACE_BY_PHOTCODE = TRUE; + remove_argument (N, argc, argv); + } + + NTHREADS = 0; + if ((N = get_argument (*argc, argv, "-threads"))) { + remove_argument (N, argc, argv); + NTHREADS = MAX(0, atoi(argv[N])); + remove_argument (N, argc, argv); + } + + /*** provide additional data ***/ + /* restrict to a portion of the sky? */ + UserPatch.Rmin = 0; + UserPatch.Rmax= 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (*argc, argv, "-region"))) { + remove_argument (N, argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, argc, argv); + } + + if ((*argc < 4) || (*argc > 6)) dvomerge_client_usage(); return TRUE; } Index: trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 33963) @@ -1,51 +1,8 @@ # include "dvomerge.h" +// utility functions void sort_IDmap (IDmapType *IDmap); - -off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) { - - // use bisection to find the starting entry by time - - off_t Nlo, Nhi, N; - - // find the last TGT before start - Nlo = 0; Nhi = NimagesTgt; - while (Nhi - Nlo > 10) { - N = 0.5*(Nlo + Nhi); - if (TgtTimes[N] < start) { - Nlo = MAX(N, 0); - } else { - Nhi = MIN(N + 1, NimagesTgt); - } - } - - // check for the matched mosaic starting from Nlo - // we may have to go much beyond Nlo since stop is not sorted - // can we use a sorted version of stop to check when we are beyond the valid range?? - for (N = Nlo; N < NimagesTgt; N++) { - if (TgtTimes[N] < start) continue; - if (TgtTimes[N] > stop) return (-1); - if (TgtCodes[N] != photcode) continue; - return (TgtIndex[N]); - } - return (-1); -} - -// sort two times vectors and an index by first time vector -void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) { - -# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c; \ - tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \ - tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \ - tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \ -} -# define COMPARE(A,B)(S[A] < S[B]) - - OHANA_SORT (N, COMPARE, SWAPFUNC); - -# undef SWAPFUNC -# undef COMPARE - -} +void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N); +off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt); // we have two tables; 'tgt' contains some exposures from 'src' : find them and match a map @@ -71,6 +28,6 @@ } - ALLOCATE (IDmap->old, off_t, NimagesSrc); - ALLOCATE (IDmap->new, off_t, NimagesSrc); + ALLOCATE (IDmap->old, unsigned int, NimagesSrc); + ALLOCATE (IDmap->new, unsigned int, NimagesSrc); IDmap->Nmap = NimagesSrc; @@ -116,4 +73,12 @@ int status; + IDmap->old = NULL; + IDmap->new = NULL; + + // it is OK if there are no images in the database, but there should be no imageIDs to map... + if (in->dbstate == LCK_EMPTY) { + return TRUE; + } + images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped); if (!images) { @@ -122,6 +87,6 @@ } - ALLOCATE (IDmap->old, off_t, Nimages); - ALLOCATE (IDmap->new, off_t, Nimages); + ALLOCATE (IDmap->old, unsigned int, Nimages); + ALLOCATE (IDmap->new, unsigned int, Nimages); IDmap->Nmap = Nimages; @@ -157,5 +122,6 @@ } -// XXX isn't the map just ID_new = ID_old + offset ?? +// map this ID to the new table +// XXX isn't the map just ID_new = ID_old + offset ?? (probably not) off_t dvo_map_image_ID (IDmapType *IDmap, off_t oldID) { @@ -191,4 +157,5 @@ } +// given a table of detections, update their image IDs based on the new map int dvo_update_image_IDs (IDmapType *IDmap, Catalog *catalog) { @@ -198,4 +165,8 @@ oldID = catalog[0].measure[i].imageID; if (oldID == 0) continue; + + if (!IDmap->old) { + fprintf (stderr, "input database has image IDs, but no Image table\n"); + } newID = dvo_map_image_ID (IDmap, oldID); @@ -225,2 +196,48 @@ } +off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) { + + // use bisection to find the starting entry by time + + off_t Nlo, Nhi, N; + + // find the last TGT before start + Nlo = 0; Nhi = NimagesTgt; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (TgtTimes[N] < start) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, NimagesTgt); + } + } + + // check for the matched mosaic starting from Nlo + // we may have to go much beyond Nlo since stop is not sorted + // can we use a sorted version of stop to check when we are beyond the valid range?? + for (N = Nlo; N < NimagesTgt; N++) { + if (TgtTimes[N] < start) continue; + if (TgtTimes[N] > stop) return (-1); + if (TgtCodes[N] != photcode) continue; + return (TgtIndex[N]); + } + return (-1); +} + +// sort two times vectors and an index by first time vector +void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) { + +# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c; \ + tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \ + tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \ + tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \ +} +# define COMPARE(A,B)(S[A] < S[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: trunk/Ohana/src/dvomerge/src/dvomergeCreate.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/dvomergeCreate.c (revision 33963) @@ -124,16 +124,18 @@ depth2 = insky2[0].regions[Ns].depth; + SkyList *outlist = SkyListByPatch (outsky, -1, &UserPatch); + // loop over the populatable output regions - for (i = 0; i < outsky[0].Nregions; i++) { - if (!outsky[0].regions[i].table) continue; - if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name); + for (i = 0; i < outlist[0].Nregions; i++) { + if (!outlist[0].regions[i][0].table) continue; + if (VERBOSE) fprintf (stderr, "output: %s\n", outlist[0].regions[i][0].name); // load / create output catalog - LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", NsecfiltOutput); + LoadCatalog (&outcatalog, outlist[0].regions[i], outlist[0].filename[i], "w", NsecfiltOutput); // combine only tables at equal or larger depth // load in all of the tables from input1 for this region - inlist = SkyListByBounds (insky1, depth1, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax); + inlist = SkyListByBounds (insky1, depth1, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01); for (j = 0; j < inlist[0].Nregions; j++) { if (VERBOSE) fprintf (stderr, "input 1: %s\n", inlist[0].regions[j][0].name); @@ -149,5 +151,5 @@ } dvo_update_image_IDs (&IDmap1, &incatalog); - merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog, secfiltMap1); + merge_catalogs_new (outlist[0].regions[i], &outcatalog, &incatalog, secfiltMap1); dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -156,5 +158,5 @@ // load in all of the tables from input2 for this region - inlist = SkyListByBounds (insky2, depth2, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax); + inlist = SkyListByBounds (insky2, depth2, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01); for (j = 0; j < inlist[0].Nregions; j++) { if (VERBOSE) fprintf (stderr, "input 2: %s\n", inlist[0].regions[j][0].name); @@ -170,5 +172,5 @@ } dvo_update_image_IDs (&IDmap2, &incatalog); - merge_catalogs_old (&outsky[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2); + merge_catalogs_old (outlist[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2); dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -219,19 +221,18 @@ in1DB.mode = dvo_catalog_catmode (CATMODE); in1DB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in1DB.filename); // load the image table - if (in1DB.dbstate == LCK_EMPTY) { - Shutdown ("can't find input (1) image catalog %s", in1DB.filename); - } - if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) { - Shutdown ("can't read input (1) image catalog %s", in1DB.filename); - } - + if (in1DB.dbstate != LCK_EMPTY) { + if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) { + Shutdown ("can't read input (1) image catalog %s", in1DB.filename); + } + } + // convert database table to internal structure & add to output image db + // if in1DB has no images, we will (later) insist that there are no image IDs to map dvo_image_merge_dbs(IDmap1, &outDB, &in1DB); dvo_image_unlock (&in1DB); // unlock input1 - /*** load input2/Images.dat ***/ @@ -239,16 +240,16 @@ in2DB.mode = dvo_catalog_catmode (CATMODE); in2DB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in2DB.filename); /* load the image table */ - if (in2DB.dbstate == LCK_EMPTY) { - Shutdown ("can't find input (2) image catalog %s", in2DB.filename); - } - if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) { - Shutdown ("can't read input (2) image catalog %s", in2DB.filename); + if (in2DB.dbstate != LCK_EMPTY) { + if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) { + Shutdown ("can't read input (2) image catalog %s", in2DB.filename); + } } // convert database table to internal structure & add to output image db + // if in1DB has no images, we will (later) insist that there are no image IDs to map dvo_image_merge_dbs(IDmap2, &outDB, &in2DB); dvo_image_unlock (&in2DB); // unlock input2 Index: trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c (revision 33963) @@ -1,5 +1,5 @@ # include "dvomerge.h" -/*** update the image table ***/ +/*** generate the IDmap for the given databases, and update the output image table ***/ int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) { @@ -12,9 +12,10 @@ inDB.mode = dvo_catalog_catmode (CATMODE); inDB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename); // load the image table if (inDB.dbstate == LCK_EMPTY) { + dvo_image_unlock (&inDB); // unlock input IDmap->Nmap = 0; return TRUE; @@ -52,5 +53,5 @@ } -/*** update the image table ***/ +/*** generate the map for the given 2 databases ***/ int dvomergeImagesGetMap (IDmapType *IDmap, char *input, char *output) { @@ -63,9 +64,10 @@ inDB.mode = dvo_catalog_catmode (CATMODE); inDB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename); // load the image table if (inDB.dbstate == LCK_EMPTY) { + dvo_image_unlock (&inDB); // unlock input IDmap->Nmap = 0; return TRUE; @@ -76,4 +78,5 @@ Shutdown ("can't read input image catalog %s", inDB.filename); } + dvo_image_unlock (&inDB); // unlock input /*** load output/Images.dat ***/ @@ -81,19 +84,18 @@ outDB.mode = dvo_catalog_catmode (CATMODE); outDB.format = dvo_catalog_catformat (CATFORMAT); - status = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + status = dvo_image_lock (&outDB, ImageCat, 5400.0, LCK_XCLD); // shorter timeout? if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename); /* load the image table */ if (outDB.dbstate == LCK_EMPTY) { - Shutdown ("only use -continue for an existing, partially merged database"); + Shutdown ("for dvomerge -continue or dvomerge_client, the output database must exist"); } if (!dvo_image_load (&outDB, VERBOSE, TRUE)) { Shutdown ("can't read output image catalog %s", outDB.filename); } + dvo_image_unlock (&outDB); // unlock output // convert database table to internal structure & add to output image db dvo_image_match_dbs(IDmap, &outDB, &inDB); - dvo_image_unlock (&inDB); // unlock input - dvo_image_unlock (&outDB); // unlock output return TRUE; Index: trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 33963) @@ -4,8 +4,6 @@ int CONTINUE; - off_t i, j; SkyTable *outsky, *insky; - SkyList *outlist, *inlist; - Catalog incatalog, outcatalog; + SkyList *inlist; char filename[256], *input, *output; IDmapType IDmap; @@ -107,54 +105,5 @@ SetPhotcodeTable(NULL); - // loop over the populated input regions - for (i = 0; i < inlist[0].Nregions; i++) { - if (!inlist[0].regions[i][0].table) continue; - if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name); - - // load / create output catalog (if catalog does not exist, it will be created) - LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput); - // skip empty input catalogs - if (!incatalog.Naves_disk) { - dvo_catalog_unlock (&incatalog); - dvo_catalog_free (&incatalog); - continue; - } - - // combine only tables at equal or larger depth - - // load in all of the tables from input for this region - // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding). Since the regions are not infinitely small, - // compare to a slightly reduced footprint - float dPos = 2.0/3600.0; - outlist = SkyListByBounds (outsky, inlist[0].regions[i][0].depth, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos); - for (j = 0; j < outlist[0].Nregions; j++) { - if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name); - - // load input catalog - LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput); - - dvo_update_image_IDs (&IDmap, &incatalog); - merge_catalogs_old (&outsky[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap); - - outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; - - // if we receive a signal which would cause us to exit, wait until the full catalog is written - SetProtect (TRUE); - if (!dvo_catalog_save (&outcatalog, VERBOSE)) { - fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); - exit (1); - } - SetProtect (FALSE); - - dvo_catalog_unlock (&outcatalog); - dvo_catalog_free (&outcatalog); - - fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name); - } - SkyListFree (outlist); - - dvo_catalog_unlock (&incatalog); - dvo_catalog_free (&incatalog); - } + dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap); // save the output sky table copy Index: trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 33963) +++ trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 33963) @@ -0,0 +1,273 @@ +# include "dvomerge.h" +# define DEBUG 1 + +// in parallel mode, the target database may be distributed, while the input database is +// only on a single machine + +// this function only merges tables which are at equal or shallower depth compared to the +// output tables + +int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky, IDmapType *IDmap); + +int dvomergeUpdate_catalogs (char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap) { + + off_t i, j; + SkyList *outlist; + Catalog incatalog, outcatalog; + + if (PARALLEL && !HOST_ID) { + int status = dvomergeUpdate_parallel (input, output, outsky, IDmap); + return status; + } + + // loop over the populated input regions + for (i = 0; i < inlist[0].Nregions; i++) { + if (!inlist[0].regions[i][0].table) continue; + if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name); + + // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding). Since the regions are not infinitely small, + // compare to a slightly reduced footprint + float dPos = 2.0/3600.0; + outlist = SkyListByBounds (outsky, -1, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos); + + // there may be more than one output region for a given input region. are any of these output regions relevant to this machine? + int found = FALSE; + for (j = 0; !found && (j < outlist[0].Nregions); j++) { + found = (found || HostTableTestHost(outlist[0].regions[j], HOST_ID)); + } + + // skip this input table for if no output files are on this machine + if (!found) { + SkyListFree (outlist); + continue; + } + + // get the stats on this input file (for comparison with the output headers) + struct stat instats; + int stat_result = stat (inlist[0].filename[i], &instats); + if (stat_result) { + if (errno == ENOENT) continue; + fprintf (stderr, "cannot read stats on input file %s\n", inlist[0].filename[i]); + perror ("stats error message:"); + exit (2); + } + + // instats.st_size & instats.st_mtime + + // check if any of the output files have NOT yet received data from this input file + + int missed = FALSE; + for (j = 0; !missed && (j < outlist[0].Nregions); j++) { + if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue; + + // set the parameters which guide catalog open/load/create + char hostfile[DVO_MAX_PATH]; + snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name); + char *filename = HOST_ID ? hostfile : outlist[0].filename[j]; + + // get the stats on this input file (for comparison with the output headers) + struct stat outstats; + stat_result = stat (filename, &outstats); + if (stat_result) { + if (errno == ENOENT) { + missed = TRUE; + break; + } + fprintf (stderr, "cannot read stats on output file %s\n", filename); + perror ("stats error message:"); + exit (2); + } + + FILE *fout = fopen (filename, "r"); + if (!fout) { + fprintf (stderr, "problem opening output file to read header %s\n", filename); + perror ("stats error message:"); + exit (2); + } + + Header outheader; + if (!gfits_fread_header (fout, &outheader)) { + fprintf (stderr, "problem reading header for output file %s\n", filename); + exit (2); + } + + fclose (fout); + + // update header of output catalog + long long last_size; + char last_moddate[80]; + gfits_scan (&outheader, "LMRG_SZ", "%lld", 1, &last_size); + gfits_scan (&outheader, "LMRG_DT", "%s", 1, last_moddate); + + time_t last_mod = ohana_date_to_sec (last_moddate); + + if (last_size != instats.st_size) { + missed = TRUE; + break; + } + + if (last_mod != instats.st_mtime) { + missed = TRUE; + break; + } + } + if (!missed) { + fprintf (stderr, "skipping %s, already merged\n", inlist[0].filename[i]); + continue; + } + + // read the input catalog + LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput); + + // skip empty input catalogs + if (!incatalog.Naves_disk) { + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + SkyListFree (outlist); + continue; + } + + char *moddate = ohana_sec_to_date (instats.st_mtime); + + // merge input into the appropriate output tables + for (j = 0; j < outlist[0].Nregions; j++) { + // skip tables for which the output files are not on this machine + if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue; + + if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name); + + // set the parameters which guide catalog open/load/create + char hostfile[DVO_MAX_PATH]; + snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name); + outcatalog.filename = HOST_ID ? hostfile : outlist[0].filename[j]; + + // load input catalog + LoadCatalog (&outcatalog, outlist[0].regions[j], outcatalog.filename, "w", NsecfiltOutput); + + dvo_update_image_IDs (IDmap, &incatalog); + merge_catalogs_old (outlist[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap); + + outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + + // update header of output catalog + long long last_size; + char last_moddate[80]; + int status_size = gfits_scan (&outcatalog.header, "LMRG_SZ", "%lld", 1, &last_size); + int status_date = gfits_scan (&outcatalog.header, "LMRG_DT", "%s", 1, last_moddate); + + // save the previous size of merged entry + if (status_size && status_date) { + gfits_modify (&outcatalog.header, "LMRG_SZ1", "%lld", 1, (long long) last_size); + gfits_modify (&outcatalog.header, "LMRG_DT1", "%s", 1, last_moddate); + } + + // update header of output catalog + gfits_modify (&outcatalog.header, "LMRG_SZ", "%lld", 1, (long long) instats.st_size); + gfits_modify (&outcatalog.header, "LMRG_DT", "%s", 1, moddate); + + if (!dvo_catalog_backup (&outcatalog, TRUE)) { + fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", outlist[0].filename[j]); + exit (1); + } + + // if we receive a signal which would cause us to exit, wait until the full catalog is written + SetProtect (TRUE); + if (!dvo_catalog_save (&outcatalog, VERBOSE)) { + fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]); + exit (1); + } + SetProtect (FALSE); + + if (!dvo_catalog_unlink_backup (&outcatalog, TRUE)) { + fprintf (stderr, "WARNING: failed to remove backup for catalog %s\n", outlist[0].filename[j]); + } + + dvo_catalog_unlock (&outcatalog); + dvo_catalog_free (&outcatalog); + + fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name); + } + SkyListFree (outlist); + free (moddate); + + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + } + return TRUE; +} + +// launch the dvomergeUpdate_client jobs to the parallel hosts +int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky, IDmapType *IDmap) { + + // ensure that the paths are absolute path names + char *absinput = abspath (input, DVO_MAX_PATH); + char *absoutput = abspath (output, DVO_MAX_PATH); + + // name of image subset file: + char IDmapFilename[DVO_MAX_PATH]; + snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput); + + // save IDmap information + if (!IDmapSave (IDmapFilename, IDmap)) { + fprintf (stderr, "ERROR: failure to save the image ID map\n"); + exit (1); + } + + // load the list of hosts + HostTable *table = HostTableLoad (output, outsky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", outsky->hosts, output); + exit (1); + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + // options / arguments that can affect relastro_client -update-objects: + char command[DVO_MAX_PATH]; + snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", + absinput, absoutput, table->hosts[i].hostID, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS + ); + + char tmpline[DVO_MAX_PATH]; + if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } + if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace", command); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running photdbc_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the photdbc_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} Index: trunk/Ohana/src/dvomerge/src/dvomerge_client.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 33963) +++ trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 33963) @@ -0,0 +1,88 @@ +# include "dvomerge.h" + +int main (int argc, char **argv) { + + dvomerge_client_help (argc, argv); + ConfigInit (&argc, argv); + dvomerge_client_args (&argc, argv); + + SkyTable *outsky, *insky; + SkyList *inlist; + char filename[256], *input, *output; + IDmapType *IDmap; + PhotCodeData *inputPhotcodes; + PhotCodeData *outputPhotcodes; + int *secfiltMap = NULL; + int NsecfiltInput, NsecfiltOutput; + + input = argv[1]; + output = argv[3]; + + // we need input and output photcode tables to correctly match secfilt entries + + // load the input photcodes + SetPhotcodeTable(NULL); + sprintf (filename, "%s/Photcodes.dat", input); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error reading input database directory %s\n", input); + exit (1); + } + inputPhotcodes = GetPhotcodeTable(); + NsecfiltInput = GetPhotcodeNsecfilt(); + + // since we are merging the input db into the output db, the output defines the photcode + // table & db layout but, this requires the output to exist. if it does not, fail + // (master should have created the output photcode table) + + // load the output photcodes + SetPhotcodeTable(NULL); + sprintf (filename, "%s/Photcodes.dat", output); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error reading output database directory %s\n", output); + exit (1); + } + outputPhotcodes = GetPhotcodeTable(); + NsecfiltOutput = GetPhotcodeNsecfilt(); + + secfiltMap = GetSecFiltMap(outputPhotcodes, inputPhotcodes); + if (!secfiltMap) { + fprintf (stderr, "failed to map input secfilt photcodes to output photcodes table\n"); + exit (1); + } + + char *absoutput = abspath (output, DVO_MAX_PATH); + char IDmapFilename[DVO_MAX_PATH]; + snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput); + + // save IDmap information + IDmap = IDmapLoad (IDmapFilename); + if (!IDmap) { + fprintf (stderr, "ERROR: failure to save the image ID map\n"); + exit (1); + } + + // load the sky table for the existing database + insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); + if (!insky) { + Shutdown ("can't read SkyTable for %s", input); + } + SkyTableSetFilenames (insky, input, "cpt"); + + // XXX apply this...generate the subset matching the user-selected region + inlist = SkyListByPatch (insky, -1, &UserPatch); + + // generate an output table populated at the desired depth + outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + if (!outsky) { + Shutdown ("can't read or create SkyTable for %s", output); + } + SkyTableSetFilenames (outsky, output, "cpt"); + + // loop over the populatable output tables; check for data in input in the corresponding regions + + SetPhotcodeTable(NULL); + + dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap); + + exit (0); +} Index: trunk/Ohana/src/dvomerge/src/dvoverify.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvoverify.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/dvoverify.c (revision 33963) @@ -44,4 +44,37 @@ CHECKSORTED = TRUE; remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + char *filename = strcreate (argv[N]); + remove_argument (N, &argc, argv); + + int isBad = FALSE; + + if (!VerifyTableFile (filename)) { + fprintf (stderr, "bad average table %s\n", filename); + isBad = TRUE; + } + + // change last 't' to 's': + int Nlast; + Nlast = strlen(filename) - 1; + filename[Nlast] = 's'; + if (!VerifyTableFile (filename)) { + fprintf (stderr, "bad secfilt table %s\n", filename); + isBad = TRUE; + } + + // change last 't' to 's': + Nlast = strlen(filename) - 1; + filename[Nlast] = 'm'; + if (!VerifyTableFile (filename)) { + fprintf (stderr, "bad measure table %s\n", filename); + isBad = TRUE; + } + + if (isBad) exit (1); + exit (0); } Index: trunk/Ohana/src/dvomerge/src/help.c =================================================================== --- trunk/Ohana/src/dvomerge/src/help.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/help.c (revision 33963) @@ -6,4 +6,10 @@ fprintf (stderr, " OR: dvomerge (input) into (output) continue\n"); fprintf (stderr, " OR: dvomerge (input) into (output) from (list)\n"); + fprintf (stderr, " [-region Rmin Rmax Dmin Dmax]\n"); + exit (2); +} + +void dvomerge_client_usage (void) { + fprintf (stderr, "USAGE: dvomerge_client (input) into (output)\n"); fprintf (stderr, " [-region Rmin Rmax Dmin Dmax]\n"); exit (2); @@ -45,4 +51,28 @@ fprintf (stderr, " -h : this list\n\n"); fprintf (stderr, " -region Rmin Rmax Dmin Dmax : region to merge\n\n"); + fprintf (stderr, " -replace : replace existing detections of the same photcode\n"); + fprintf (stderr, " (this should only be used to merge reference dbs)\n\n"); + exit (2); +} + +void dvomerge_client_help (int argc, char **argv) { + + /* check for help request */ + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + return; + +show_help: + + fprintf (stderr, "USAGE\n"); + fprintf (stderr, " dvomerge_client (input) into (output)\n"); + fprintf (stderr, " merge DVO databases\n"); + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : region to merge\n\n"); + fprintf (stderr, " -replace : replace existing detections of the same photcode\n"); + fprintf (stderr, " (this should only be used to merge reference dbs)\n\n"); exit (2); } Index: trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 33657) +++ trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 33963) @@ -111,4 +111,10 @@ if (Nave > 1) sort_coords_index (X2, Y2, N2, Nave); + if (REPLACE_BY_PHOTCODE && output[0].Nmeasure && !output[0].sorted) { + fprintf (stderr, "ERROR: attempt to merge with replace into an unsorted database\n"); + exit (3); + } + + /* set up pointers for linked list of measure */ if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) { @@ -186,4 +192,12 @@ offset = input[0].average[N].measureOffset + Nin; + if (REPLACE_BY_PHOTCODE) { + // index to first measure for this object + int Mout = output[0].average[n].measureOffset; + if (replace_match(&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[offset])) { + input[0].found[N] = Mout; + continue; + } + } /* add to end of measurement list */ add_meas_link (&output[0].average[n], next_meas, Nmeas, NMEAS); Index: trunk/Ohana/src/dvomerge/src/replace_match.c =================================================================== --- trunk/Ohana/src/dvomerge/src/replace_match.c (revision 33963) +++ trunk/Ohana/src/dvomerge/src/replace_match.c (revision 33963) @@ -0,0 +1,57 @@ +# include "dvomerge.h" + +// average and measure pointers to the objects of interest and their first / current measurement +int replace_match (Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in) { + + int Nout; + unsigned int averef; + unsigned int catID; + double Rin, Din; + + // find the matching photcode in the object's list of measurements + for (Nout = 0; Nout < average_out[0].Nmeasure; Nout ++) { + if (measure_out[Nout].photcode != measure_in[0].photcode) continue; + + // set the new measurements + averef = measure_out[Nout].averef; + catID = measure_out[Nout].catID; + measure_out[Nout] = measure_in[0]; + + Rin = average_in[0].R - measure_in[0].dR / 3600.0; + Din = average_in[0].D - measure_in[0].dD / 3600.0; + + /** dR,dD now represent arcsec **/ + measure_out[Nout].dR = 3600.0*(average_out[0].R - Rin); + measure_out[Nout].dD = 3600.0*(average_out[0].D - Din); + measure_out[Nout].dbFlags = 0; // XXX why reset these? + measure_out[Nout].averef = averef; + measure_out[Nout].objID = average_out[0].objID; + measure_out[Nout].catID = catID; + + // rationalize dR + if (measure_out[Nout].dR > +180.0*3600.0) { + // average on high end of boundary, move star up + Rin += 360.0; + measure_out[Nout].dR = 3600.0*(average_out[0].R - Rin); + } + if (measure_out[Nout].dR < -180.0*3600.0) { + // average on low end of boundary, move star down + Rin -= 360.0; + measure_out[Nout].dR = 3600.0*(average_out[0].R - Rin); + } + + // warn on surprisingly distant detections + if (fabs(measure_out[Nout].dR) > 10*RADIUS) { + // ok take declination into account and check again. + double cosD = cos(RAD_DEG*Din); + if (fabs(measure_out[Nout].dR*cosD) > 10*RADIUS) { + + fprintf (stderr, "surprisingly distant detection: %10.6f,%10.6f vs %10.6f,%10.6f\n", + average_out[0].R, average_out[0].D, Rin, Din); + } + } + return TRUE; // found the matched entry + } + return FALSE; // did not find the matched entry +} + Index: trunk/Ohana/src/dvomerge/test/HostTable.dat =================================================================== --- trunk/Ohana/src/dvomerge/test/HostTable.dat (revision 33963) +++ trunk/Ohana/src/dvomerge/test/HostTable.dat (revision 33963) @@ -0,0 +1,5 @@ +# Host Table for parallel DVO +# ID Hostname Catdir + 1 pikake catdir.merge.p1 + 2 pikake catdir.merge.p2 + 3 pikake catdir.merge.p3 Index: trunk/Ohana/src/dvomerge/test/dvomerge.parallel.sh =================================================================== --- trunk/Ohana/src/dvomerge/test/dvomerge.parallel.sh (revision 33963) +++ trunk/Ohana/src/dvomerge/test/dvomerge.parallel.sh (revision 33963) @@ -0,0 +1,7 @@ +#!/bin/csh -f + + rm -rf catdir.merge + rsync -auv catdir.2mass/ catdir.merge/ + cp HostTable.dat catdir.merge/ + dvodist -out catdir.merge + dvomerge -parallel catdir.grizy into catdir.merge Index: trunk/Ohana/src/dvomerge/test/dvomerge.replace.sh =================================================================== --- trunk/Ohana/src/dvomerge/test/dvomerge.replace.sh (revision 33963) +++ trunk/Ohana/src/dvomerge/test/dvomerge.replace.sh (revision 33963) @@ -0,0 +1,8 @@ +#!/bin/csh -f + + rm -rf catdir.merge.save + rm -rf catdir.merge + rsync -auv catdir.2mass/ catdir.merge/ + dvomerge catdir.grizy into catdir.merge + rsync -auv catdir.merge/ catdir.merge.save/ + dvomerge -replace catdir.2mass into catdir.merge