Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/doc/allsky.txt =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/doc/allsky.txt (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/doc/allsky.txt (revision 31524) @@ -1,2 +1,34 @@ + +2011.05.10 + + Looking into them memory usage of relphot: + AverageTiny = 32 bytes (including 8 byte padding) + MeasureTiny = 56 bytes (including 8 byte padding) + Image Indexes: + bin : Nmeasure * off_t + clist : Nmeasure * off_t + mlist : Nmeasure * off_t + Mosaic Indexes: + bin : Nmeasure * off_t + clist : Nmeasure * off_t + mlist : Nmeasure * off_t + Total: + 32 bytes * Naverage (values) + 56 bytes * Nmeasure (values) + 64 bytes * Nmeasure (indexes) + + In my test run, I have 3.3M average + 37.6M measure: + 2.1GB values + 2.2GB indexes + + This matches the memory usage graph. But, after the indexes have + been loaded, the memory footprint grows (slowly) by another 1GB as + the processing runs along. + + +using 23153493 of 640663697 stars (192585953 of 289672671 measurements) +using 3306896 of 48399614 stars ( 37582679 of 193863612 measurements) + + 2011.04.12 Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/MosaicOps.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/MosaicOps.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/MosaicOps.c (revision 31524) @@ -485,5 +485,4 @@ ALLOCATE (imageOffset, double, Nmosaic); - ALLOCATE ( Slist, int *, Nmosaic); // array of calibrated star indexes on this mosaic ALLOCATE (NSlist, int, Nmosaic); // number of stars on mosaic @@ -647,11 +646,12 @@ free (starNcount); + free (seclist); + free (NSlist); + free (NSLIST); for (i = 0; i < Nmosaic; i++){ free (Slist[i]); } - free (NSlist); - free (NSLIST); + free (Slist); free (imageOffset); - free (seclist); return (TRUE); } Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/StarOps.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/StarOps.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/StarOps.c (revision 31524) @@ -587,4 +587,6 @@ initstats (STATMODE); if (VERBOSE) fprintf (stderr, OFF_T_FMT" measures marked poor, "OFF_T_FMT" total\n", Ndel, Nave); + free (list); + free (dlist); free (ilist); free (tlist); Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/bcatalog.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/bcatalog.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/bcatalog.c (revision 31524) @@ -5,7 +5,7 @@ int CopyAverageTiny (AverageTiny *averageT, Average *average) { - averageT[0].R = average[0].R; - averageT[0].D = average[0].D; - averageT[0].flags = average[0].flags; + averageT[0].R = average[0].R; + averageT[0].D = average[0].D; + averageT[0].flags = average[0].flags; averageT[0].Nmeasure = average[0].Nmeasure; averageT[0].measureOffset = average[0].measureOffset; @@ -293,13 +293,44 @@ off_t i; + AverageTiny *averageT; + Average *average; + + MeasureTiny *measureT; + Measure *measure; + ALLOCATE (catalog[0].measureT, MeasureTiny, catalog[0].Nmeasure); ALLOCATE (catalog[0].averageT, AverageTiny, catalog[0].Naverage); + average = catalog[0].average; + averageT = catalog[0].averageT; + + measure = catalog[0].measure; + measureT = catalog[0].measureT; + for (i = 0; i < catalog[0].Naverage; i++) { - CopyAverageTiny (&catalog[0].averageT[i], &catalog[0].average[i]); + // CopyAverageTiny (&catalog[0].averageT[i], &catalog[0].average[i]); + averageT[i].R = average[i].R; + averageT[i].D = average[i].D; + averageT[i].flags = average[i].flags; + averageT[i].Nmeasure = average[i].Nmeasure; + averageT[i].measureOffset = average[i].measureOffset; } for (i = 0; i < catalog[0].Nmeasure; i++) { - CopyMeasureTiny (&catalog[0].measureT[i], &catalog[0].measure[i]); + // CopyMeasureTiny (&catalog[0].measureT[i], &catalog[0].measure[i]); + measureT[i].dR = measure[i].dR; + measureT[i].dD = measure[i].dD; + measureT[i].M = measure[i].M; + measureT[i].Mcal = measure[i].Mcal; + measureT[i].dM = measure[i].dM; + measureT[i].airmass = measure[i].airmass; + measureT[i].Xccd = measure[i].Xccd; + measureT[i].Yccd = measure[i].Yccd; + measureT[i].t = measure[i].t; + measureT[i].dt = measure[i].dt; + measureT[i].averef = measure[i].averef; + measureT[i].imageID = measure[i].imageID; + measureT[i].dbFlags = measure[i].dbFlags; + measureT[i].photcode = measure[i].photcode; } Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/load_catalogs.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/load_catalogs.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/load_catalogs.c (revision 31524) @@ -3,5 +3,5 @@ Catalog *load_catalogs (SkyList *skylist, int *Ncatalog) { - int i, Nmeas, Nstar, Nmeas_total, Nstar_total; + off_t i, Nmeas, Nstar, Nmeas_total, Nstar_total; Catalog *catalog, tcatalog; @@ -50,8 +50,8 @@ } if (Nstar < 2) { - fprintf (stderr, "warning: insufficient stars %d\n", Nstar); + fprintf (stderr, "warning: insufficient stars "OFF_T_FMT"\n", Nstar); } - fprintf (stderr, "using %d of %d stars (%d of %d measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total); + fprintf (stderr, "using "OFF_T_FMT" of "OFF_T_FMT" stars ("OFF_T_FMT" of "OFF_T_FMT" measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total); if (Nstar < 1) Shutdown ("%s", "ERROR: no stars match the minimum requirements; exiting \n"); Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/reload_catalogs.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/reload_catalogs.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/reload_catalogs.c (revision 31524) @@ -18,5 +18,6 @@ double time1 = 0.0; double time2 = 0.0; - double time3 = 0.0; + double time3a = 0.0; + double time3b = 0.0; double time4 = 0.0; double time5 = 0.0; @@ -57,9 +58,10 @@ populate_tiny_values(&catalog); + TIMESTAMP(time3a); initImageBins (&catalog, 1); initMosaicBins (&catalog, 1); initGridBins (&catalog, 1); - TIMESTAMP(time3); + TIMESTAMP(time3b); findImages (&catalog, 1); // FX @@ -83,10 +85,11 @@ } - fprintf (stderr, "time1 %f : find catalog\n", time1); - fprintf (stderr, "time2 %f : load catalog\n", time2); - fprintf (stderr, "time3 %f : init imbins\n", time3); - fprintf (stderr, "time4 %f : find images\n", time4); - fprintf (stderr, "time5 %f : set Mrel\n", time5); - fprintf (stderr, "time6 %f : save catalog\n", time6); - fprintf (stderr, "time7 %f : free catalog\n", time7); + fprintf (stderr, "time1 %f : find catalog\n", time1); + fprintf (stderr, "time2 %f : load catalog\n", time2); + fprintf (stderr, "time3a %f : init imbins\n", time3a); + fprintf (stderr, "time3b %f : init imbins\n", time3b); + fprintf (stderr, "time4 %f : find images\n", time4); + fprintf (stderr, "time5 %f : set Mrel\n", time5); + fprintf (stderr, "time6 %f : save catalog\n", time6); + fprintf (stderr, "time7 %f : free catalog\n", time7); } Index: branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/relphot.c =================================================================== --- branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/relphot.c (revision 31493) +++ branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/relphot.c (revision 31524) @@ -207,4 +207,5 @@ dvo_image_save (&dbX, VERBOSE); dvo_image_unlock (&dbX); + MARKTIME("-- save Image.dat.bck: %f sec\n", dtime); }