Index: /trunk/Ohana/src/addstar/Makefile =================================================================== --- /trunk/Ohana/src/addstar/Makefile (revision 5238) +++ /trunk/Ohana/src/addstar/Makefile (revision 5239) @@ -17,5 +17,5 @@ # INCS = -I$(INC) -I$(LINC) -I$(XINC) -LIBS = -L$(LLIB) -lohana -lFITS -lz -lm +LIBS = -L$(LLIB) -ldvo -lFITS -lohana -lz -lm CFLAGS = $(INCS) -g LFLAGS = $(LIBS) Index: /trunk/Ohana/src/addstar/doc/Changes.txt =================================================================== --- /trunk/Ohana/src/addstar/doc/Changes.txt (revision 5238) +++ /trunk/Ohana/src/addstar/doc/Changes.txt (revision 5239) @@ -1,2 +1,28 @@ + +2005.10.07 + + I have finished the basic implementation of the update mode. + I have been able to demonstrate substantial improvements in + speed when the number of existing measurements dominates the + total number of measurements and the number of averages is + typically small compared to the number of measurements (ie, + most objects are real, detected in most images, and each new + image supplies many new measurements of objects which exist + and not many of objects which don't exist already). The speed + gain is significant in this context because the average table + is small compared to the measure table; since both update and + full-load methods require the complete average table, there is + no difference in the load time for the average table. + + I was having some memory collision problems, and attempting to + use the ohana_allocate functions reminded me that the libFITS + functions were not supported under ohana_allocate. This was + unhelpful. I bit the bullet and split libohana into libohana + (base functions only, including ohana_allocate) and libdvo + (functions based on the libautocode structures). Doing this + allowed me to make libFITS depend on libohana (including + ohana_allocate). BUT, this forced me to change all LDFLAGS + entries in ohana to swap -lohana -lFITS for -lFITS -lohana, + and to add include in some cases. 2005.10.06 Index: /trunk/Ohana/src/addstar/include/addstar.h =================================================================== --- /trunk/Ohana/src/addstar/include/addstar.h (revision 5238) +++ /trunk/Ohana/src/addstar/include/addstar.h (revision 5239) @@ -189,5 +189,5 @@ int *init_measure_links (Average *average, int Naverage, Measure *measure, int Nmeasure); int *init_missing_links (Average *average, int Naverage, Missing *missing, int Nmissing); -int add_meas_link (Average *average, int *next, int Nmeasure); +int add_meas_link (Average *average, int *next, int Nmeasure, int NMEASURE); int add_miss_link (Average *average, int *next, int Nmissing); int *build_measure_links (Average *average, int Naverage, Measure *measure, int Nmeasure); Index: /trunk/Ohana/src/addstar/src/addstar.c =================================================================== --- /trunk/Ohana/src/addstar/src/addstar.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/addstar.c (revision 5239) @@ -3,4 +3,5 @@ int main (int argc, char **argv) { + int Nm, Na, Ns; int i, Nstars, Noverlap, Nregions, Nsubset, Naverage, Nmeasure; Stars *stars, **subset; @@ -11,5 +12,5 @@ double dtime; - struct timeval start, stop; + struct timeval start, stop, t1, t2; gettimeofday (&start, NULL); @@ -28,4 +29,7 @@ /* we use the image table to lock db access -- perhaps this is not necessary?*/ lock_image_db (&db, ImageCat); + gettimeofday (&stop, NULL); + dtime = DTIME (stop, start); + fprintf (stderr, "mark: lock_image_db: time %9.4f sec\n", dtime); switch (MODE) { @@ -51,10 +55,26 @@ if (VERBOSE) fprintf (stderr, "writing to %d regions\n", Nregions); + gettimeofday (&stop, NULL); + dtime = DTIME (stop, start); + fprintf (stderr, "mark: gstars: time %9.4f sec\n", dtime); + Naverage = Nmeasure = 0; for (i = 0; i < Nregions; i++) { + + gettimeofday (&t1, NULL); + if (!load_pt_catalog (&catalog, ®ions[i])) continue; + Nm = catalog.Nmeasure + catalog.Nmeas_off; + Na = catalog.Naverage; + Ns = Nstars; + + gettimeofday (&t2, NULL); + dtime = DTIME (t2, t1); + fprintf (stderr, "load time %9.4f sec for %5d stars, %6d average, %7d measure\n", dtime, Ns, Na, Nm); Naverage += catalog.Naverage; Nmeasure += catalog.Nmeasure; + + gettimeofday (&t1, NULL); switch (MODE) { @@ -75,4 +95,11 @@ break; } + + gettimeofday (&t2, NULL); + dtime = DTIME (t2, t1); + fprintf (stderr, "match time %9.4f sec for %5d stars, %6d average, %7d measure\n", dtime, Ns, Na, Nm); + + gettimeofday (&t1, NULL); + if (Nsubset == 0) { unlock_catalog (&catalog); @@ -80,5 +107,14 @@ save_pt_catalog (&catalog); } + + gettimeofday (&t2, NULL); + dtime = DTIME (t2, t1); + fprintf (stderr, "save time %9.4f sec for %5d stars, %6d average, %7d measure\n", dtime, Ns, Na, Nm); + } + + gettimeofday (&stop, NULL); + dtime = DTIME (stop, start); + fprintf (stderr, "mark: match: time %9.4f sec\n", dtime); if (CALIBRATE) { FindCalibration (&image); } @@ -102,5 +138,5 @@ gettimeofday (&stop, NULL); dtime = DTIME (stop, start); - fprintf (stderr, "SUCCESS: elapsed time %8.3f sec for %5d stars, %6d average, %7d measure\n", dtime, Nstars, Naverage, Nmeasure); + fprintf (stderr, "SUCCESS: elapsed time %9.4f sec for %5d stars, %6d average, %7d measure\n", dtime, Nstars, Naverage, Nmeasure); exit (0); } Index: /trunk/Ohana/src/addstar/src/build_links.c =================================================================== --- /trunk/Ohana/src/addstar/src/build_links.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/build_links.c (revision 5239) @@ -1,5 +1,6 @@ # include "addstar.h" -/* build the initial links assuming the table is sorted */ +/* build the initial links assuming the table is sorted, + not partial, and has a correct set of average[].offset,Nm values */ int *init_measure_links (Average *average, int Naverage, Measure *measure, int Nmeasure) { @@ -13,6 +14,13 @@ for (j = 0; j < average[i].Nm - 1; j++, N++) { next[N] = N + 1; + if (N >= Nmeasure) { + fprintf (stderr, "WARNING: N out of bounds (1)\n"); + } } next[N] = -1; + if (N >= Nmeasure) { + fprintf (stderr, "WARNING: N out of bounds (2)\n"); + } + if (N >= Nmeasure) { fprintf (stderr, "overflow in init_measure_links\n"); @@ -49,13 +57,34 @@ } -int add_meas_link (Average *average, int *next, int Nmeasure) { +/* average[].offset, average[].Nm are valid within an addstar run */ +int add_meas_link (Average *average, int *next, int Nmeasure, int NMEASURE) { int k, m; + /* if we have trouble, check validity of next[m] : m < Nmeasure */ m = average[0].offset; - for (k = 0; k < average[0].Nm - 1; k++) m = next[m]; + + for (k = 0; k < average[0].Nm - 1; k++) { + m = next[m]; + if (m >= NMEASURE) { + fprintf (stderr, "WARNING: m out of bounds (3)\n"); + } + } + /* set up references */ next[Nmeasure] = -1; - next[m] = Nmeasure; + if (Nmeasure >= NMEASURE) { + fprintf (stderr, "WARNING: Nmeasure out of bounds (1)\n"); + } + + if (m == -1) { + average[0].offset = Nmeasure; + } else { + next[m] = Nmeasure; + if (m >= NMEASURE) { + fprintf (stderr, "WARNING: m out of bounds (4)\n"); + } + } + return (TRUE); } @@ -80,23 +109,46 @@ } +/* construct measure links which are valid FOR THIS LOAD + * - if we have a full load, we will get links which can + * be used by other programs (eg, relphot, etc) + * - if we have a partial load, the links are only valid + * for that partial load + */ + int *build_measure_links (Average *average, int Naverage, Measure *measure, int Nmeasure) { - int m, k, Nm, averef; + int i, m, k, Nm, averef; int *next; ALLOCATE (next, int, Nmeasure); + /* reset the Nm, offset values for average */ + for (i = 0; i < Naverage; i++) { + average[i].offset = -1; + average[i].Nm = 0; + } + for (Nm = 0; Nm < Nmeasure; Nm++) { - next[Nm] = -1; averef = measure[Nm].averef; m = average[averef].offset; + next[Nm] = -1; + + if (m == -1) { /* no links yet for source */ + average[averef].offset = Nm; + average[averef].Nm = 1; + continue; + } + for (k = 0; next[m] != -1; k++) { m = next[m]; - } - if (m == Nm) { - average[averef].Nm = k + 1; - } else { - average[averef].Nm = k + 2; - next[m] = Nm; + if (m >= Nmeasure) { + fprintf (stderr, "WARNING: m out of bounds (1)\n"); + } + } + + average[averef].Nm = k + 2; + next[m] = Nm; + if (m >= Nmeasure) { + fprintf (stderr, "WARNING: m out of bounds (2)\n"); } } Index: /trunk/Ohana/src/addstar/src/find_matches.c =================================================================== --- /trunk/Ohana/src/addstar/src/find_matches.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/find_matches.c (revision 5239) @@ -8,5 +8,5 @@ float dX, dY, dR; int *N1, *N2, *next_meas, *next_miss; - int Nave, NAVE, Nmeas, NMEAS, Nmiss, NMISS, Nmatch, offset; + int Nave, NAVE, Nmeas, NMEAS, Nmiss, NMISS, Nmatch; Coords tcoords; int Nsecfilt, Nsec; @@ -41,5 +41,4 @@ NMEAS = Nmeas = catalog[0].Nmeasure; NMISS = Nmiss = catalog[0].Nmissing; - offset = catalog[0].Nmeas_off; /* project onto rectilinear grid with 1 arcsec pixels */ @@ -130,5 +129,5 @@ /* add to end of measurement list */ - add_meas_link (&catalog[0].average[n], next_meas, Nmeas); + add_meas_link (&catalog[0].average[n], next_meas, Nmeas, NMEAS); /* calculate accurate per-star airmass */ @@ -253,5 +252,5 @@ catalog[0].average[Nave].Xm = NO_MAG; catalog[0].average[Nave].Xg = NO_MAG; - catalog[0].average[Nave].offset = Nmeas + offset; /* this needs to be an absolute reference */ + catalog[0].average[Nave].offset = Nmeas; catalog[0].average[Nave].missing = -1; catalog[0].average[Nave].code = 0; @@ -317,13 +316,4 @@ REALLOCATE (catalog[0].missing, Missing, Nmiss); - /* XXX EAM - temporary examination of next_meas - for (i = 0; i < 5; i++) { - m = catalog[0].average[i].offset; - for (j = 0; j < catalog[0].average[i].Nm; j++) { - fprintf (stderr, "this: %d (%f), next: %d\n", m, catalog[0].measure[m].M_PS, next_meas[m]); - m = next_meas[m]; - } - } */ - if (NOSORT) { catalog[0].sorted = FALSE; Index: /trunk/Ohana/src/addstar/src/find_matches_closest.c =================================================================== --- /trunk/Ohana/src/addstar/src/find_matches_closest.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/find_matches_closest.c (revision 5239) @@ -8,5 +8,5 @@ float dX, dY, dR; int *N1, *N2, *next_meas, *next_miss; - int Nave, NAVE, Nmeas, NMEAS, Nmiss, NMISS, Nmatch, offset; + int Nave, NAVE, Nmeas, NMEAS, Nmiss, NMISS, Nmatch; Coords tcoords; int Nsecfilt, Nsec; @@ -41,5 +41,4 @@ NMEAS = Nmeas = catalog[0].Nmeasure; NMISS = Nmiss = catalog[0].Nmissing; - offset = catalog[0].Nmeas_off; /* project onto rectilinear grid with 1 arcsec pixels */ @@ -145,5 +144,6 @@ /* add to end of measurement list */ - add_meas_link (&catalog[0].average[n], next_meas, Nmeas); + add_meas_link (&catalog[0].average[n], next_meas, Nmeas, NMEAS); + ohana_memcheck (0); /* calculate accurate per-star airmass, if required */ @@ -192,6 +192,6 @@ Nmeas ++; i++; - } - + ohana_memcheck (0); + } ohana_memcheck (0); @@ -250,5 +250,5 @@ catalog[0].average[Nave].Xm = NO_MAG; catalog[0].average[Nave].Xg = NO_MAG; - catalog[0].average[Nave].offset = Nmeas + offset; + catalog[0].average[Nave].offset = Nmeas; catalog[0].average[Nave].missing = -1; catalog[0].average[Nave].code = 0; @@ -314,13 +314,4 @@ REALLOCATE (catalog[0].missing, Missing, Nmiss); - /* XXX EAM - temporary examination of next_meas - for (i = 0; i < 5; i++) { - m = catalog[0].average[i].offset; - for (j = 0; j < catalog[0].average[i].Nm; j++) { - fprintf (stderr, "this: %d (%f), next: %d\n", m, catalog[0].measure[m].M_PS, next_meas[m]); - m = next_meas[m]; - } - } */ - if (NOSORT) { catalog[0].sorted = FALSE; Index: /trunk/Ohana/src/addstar/src/replace_match.c =================================================================== --- /trunk/Ohana/src/addstar/src/replace_match.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/replace_match.c (revision 5239) @@ -14,5 +14,5 @@ measure[i].M_PS = MIN (star[0].M, NO_MAG); measure[i].dM_PS = MIN (star[0].dM, NO_ERR); - star[0].found = average[0].offset + i; + star[0].found = average[0].offset + i; return (TRUE); } Index: /trunk/Ohana/src/addstar/src/update_coords.c =================================================================== --- /trunk/Ohana/src/addstar/src/update_coords.c (revision 5238) +++ /trunk/Ohana/src/addstar/src/update_coords.c (revision 5239) @@ -12,5 +12,5 @@ if (average[0].Nm < 2) return; - /* find the average & sum-square */ + /* find the average & sum-square (does not use reference coordinates) */ m = average[0].offset; /* first measurement of this star */ for (i = 0; i < average[0].Nm; i++) {