Index: /branches/eam_branches/relphot.20210521/src/ResetOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ResetOps.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/ResetOps.c (revision 41623) @@ -104,31 +104,6 @@ } -// Used in relphot_objects.c: -void ResetAverageObjects (Catalog *catalog) { - - // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images) - if (!RESET) return; - - int Nsecfilt = catalog->Nsecfilt; - - for (off_t j = 0; j < catalog->Naverage; j++) { - catalog->average[j].flags &= ~photomBitsAverage; // reset all photometry bits (but not astrom) - catalog->average[j].psfQF = NAN; // reset (will be re-calculated here) - catalog->average[j].psfQFperf = NAN; // reset (will be re-calculated here) - catalog->average[j].stargal = NAN; // reset (will be re-calculated here) - catalog->average[j].photFlagsUpper = 0; // reset (will be re-calculated here) - catalog->average[j].photFlagsLower = 0; // reset (will be re-calculated here) - catalog->average[j].NwarpOK = 0; // reset (will be re-calculated here) - for (off_t k = 0; k < Nsecfilt; k++) { - dvo_secfilt_init (&catalog->secfilt[j*Nsecfilt + k], SECFILT_RESET_ALL); - } - } -} - // Used in setMrelFinal.c void ResetAverageAndMeasure (Catalog *catalog) { - - /*** RESET photometry flags and Mcal values ***/ - if (!RESET) return; int Nsecfilt = GetPhotcodeNsecfilt (); @@ -141,4 +116,15 @@ for (off_t i = 0; i < catalog->Naverage; i++) { + catalog[0].average[i].psfQF = NAN; // force recalculation in setMrelCatalog + catalog[0].average[i].psfQFperf = NAN; // force recalculation in setMrelCatalog + catalog[0].average[i].stargal = NAN; // force recalculation in setMrelCatalog + catalog[0].average[i].photFlagsUpper = 0; // reset (will be re-calculated) + catalog[0].average[i].photFlagsLower = 0; // reset (will be re-calculated) + catalog[0].average[i].NwarpOK = 0; // reset (will be re-calculated) + + if (!RESET) continue; + + catalog->average[i].flags &= ~photomBitsAverage; // reset all photometry bits (but not astrom) + for (int Ns = 0; Ns < Nsecfilt; Ns++) { Index: /branches/eam_branches/relphot.20210521/src/args.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/args.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/args.c (revision 41623) @@ -387,8 +387,14 @@ GRID_ZEROPT = FALSE; GRID_MEANFILE = NULL; // this is set by GridCorrectionSave() + GRID_ZPT_MODE = GRID_ZPT_MODE_NONE; // start with grid off if ((N = get_argument (argc, argv, "-grid"))) { remove_argument (N, &argc, argv); GRID_ZEROPT = TRUE; - GRID_ZPT_MODE = GRID_ZPT_MODE_NONE; // start with grid off + } + if ((N = get_argument (argc, argv, "-grid-meanfile"))) { + GRID_ZEROPT = TRUE; + remove_argument (N, &argc, argv); + GRID_MEANFILE = strcreate (argv[N]); // used by relphot -averages and relphot -apply-offsets + remove_argument (N, &argc, argv); } @@ -554,4 +560,6 @@ case SYNTH_PHOT: case UPDATE_AVERAGES: + // note: initialize sets PhotcodeList to the full set of average photcodes + // if the mode is UPDATE_AVERAGES if (argc != 1) relphot_usage(); break; Index: /branches/eam_branches/relphot.20210521/src/reload_catalogs.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/reload_catalogs.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/reload_catalogs.c (revision 41623) @@ -17,13 +17,6 @@ struct timeval start, stop; - double dtime = 0.0; - double time1 = 0.0; - double time2 = 0.0; - double time3a = 0.0; - double time3b = 0.0; - double time4 = 0.0; - double time5 = 0.0; - double time6 = 0.0; - double time7 = 0.0; + double dtime = 0.0, time1 = 0.0, time2 = 0.0, time3 = 0.0; + double time4 = 0.0, time5 = 0.0, time6 = 0.0, time7 = 0.0, time8 = 0.0; // XXX need to decide how to determine PARALLEL mode... @@ -80,22 +73,13 @@ TIMESTAMP(time2); - // XXX relphot_objects.c resets average and secfilt here - populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE); - TIMESTAMP(time3a); + TIMESTAMP(time3); // XXX need to worry about the image subset data initImageBins (&catalog, 1, FALSE); - initMosaicBins (&catalog, 1, FALSE); - initTGroupBins (&catalog, 1); - - initGridBins (); - TIMESTAMP(time3b); + TIMESTAMP(time4); findImages (&catalog, 1, FALSE); - findMosaics (&catalog, 1, FALSE); - findTGroups (&catalog, 1); - - TIMESTAMP(time4); + TIMESTAMP(time5); initMrel (&catalog, 1); @@ -103,7 +87,5 @@ setMflatFromGrid (&catalog); // Mgrid is used to set Mflat; Mgrid is the ignored in setMrelFinal/setMrelCatalog.c setMrelFinal (&catalog, FALSE); - TIMESTAMP(time5); - - // XXX relphot_objects allows !UPDATE and skips + TIMESTAMP(time6); // modify the output format as desired (ignore current format on disk) @@ -125,13 +107,8 @@ free_tiny_values(&catalog); dvo_catalog_free (&catalog); - TIMESTAMP(time6); - - // XXX freeTgroupBins here: + TIMESTAMP(time7); freeImageBins (1, FALSE); - freeMosaicBins (1, FALSE); - freeTGroupBins (1); - freeGridBins (); - TIMESTAMP(time7); + TIMESTAMP(time8); if (hostID) { @@ -140,12 +117,12 @@ } - fprintf (stderr, "time step 1 %10.3f sec : find catalog\n", time1); - fprintf (stderr, "time step 2 %10.3f sec : load catalog\n", time2); - fprintf (stderr, "time step 3 %10.3f sec : make tiny values\n", time3a); - fprintf (stderr, "time step 4 %10.3f sec : init imbins\n", time3b); - fprintf (stderr, "time step 5 %10.3f sec : find images\n", time4); - fprintf (stderr, "time step 6 %10.3f sec : set Mrel\n", time5); - fprintf (stderr, "time step 7 %10.3f sec : save catalog\n", time6); - fprintf (stderr, "time step 8 %10.3f sec : free catalog\n", time7); + fprintf (stderr, "time step 1 %10.3f sec : find catalog\n", time1); + fprintf (stderr, "time step 2 %10.3f sec : load catalog\n", time2); + fprintf (stderr, "time step 3 %10.3f sec : make tiny values\n", time3); + fprintf (stderr, "time step 4 %10.3f sec : init imbins\n", time4); + fprintf (stderr, "time step 5 %10.3f sec : find images\n", time5); + fprintf (stderr, "time step 6 %10.3f sec : set Mrel\n", time6); + fprintf (stderr, "time step 7 %10.3f sec : save catalog\n", time7); + fprintf (stderr, "time step 8 %10.3f sec : free catalog\n", time8); } Index: /branches/eam_branches/relphot.20210521/src/relphot.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/relphot.c (revision 41623) @@ -34,5 +34,5 @@ switch (mode) { case UPDATE_IMAGES: - // calculate zero points for images (may group my exposure [mosaic], night [tgroups]; may measure flat-correction) + // calculate zero points for images (may group by exposure [mosaic], night [tgroups]; may measure flat-correction) // IF CALLED WITH NLOOP == 0, DOES NOT LOAD bcatalog, just loads image table and generates ImageSubset.dat // If called with -update, calls reload_catalogs() just like apply_offsets @@ -72,4 +72,6 @@ exit (2); } + // we do not need to load grid corrections here. the value of + // GRID_MEANFILE, if specified, is passed to the clients reload_catalogs (skylist, 0, NULL); relphot_free (sky, skylist); Index: /branches/eam_branches/relphot.20210521/src/relphot_client.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_client.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/relphot_client.c (revision 41623) @@ -92,4 +92,5 @@ } +/* deprecated (absorbed into MODE_UPDATE) case MODE_UPDATE_OBJECTS: { // take the current set of detections and set the mean magnitudes @@ -99,4 +100,5 @@ break; } +*/ case MODE_SYNTH_PHOT: @@ -106,4 +108,5 @@ break; + case MODE_UPDATE_OBJECTS: default: fprintf (stderr, "impossible!"); Index: /branches/eam_branches/relphot.20210521/src/relphot_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_images.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/relphot_images.c (revision 41623) @@ -161,4 +161,11 @@ } + // once the zero points have been calculated, they are reassigned to the + // individual images. At this point, the mosaic and tgroup values should not + // be used. + freeMosaics (); + MOSAIC_ZEROPT = FALSE; + TGROUP_ZEROPT = FALSE; + reload_images (&db); @@ -175,5 +182,5 @@ if (UPDATE_CATFORMAT) { - // ensure the db format is updated + // ensure the Image db format is updated db.format = dvo_catalog_catformat (UPDATE_CATFORMAT); gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); @@ -186,5 +193,4 @@ freeImages (db.ftable.buffer); - freeMosaics (); freeGridBins (); Index: /branches/eam_branches/relphot.20210521/src/relphot_objects.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_objects.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/relphot_objects.c (revision 41623) @@ -6,23 +6,10 @@ // has been correctly propagated to the measurement, the average will respect that value. -int relphot_objects_parallel (SkyList *sky); - int relphot_objects (SkyList *skylist, int hostID, char *hostpath) { - off_t i; - struct stat filestat; - - Catalog catalog; + FITS_DB db; INITTIME; - // XXX need to decide how to determine PARALLEL mode... - if (PARALLEL && !hostID) { - relphot_objects_parallel (skylist); - return TRUE; - } - - FITS_DB db; - set_db (&db); gfits_db_init (&db); @@ -40,229 +27,14 @@ MARKTIME("loaded images: %f sec\n", dtime); - // XXX can I call reload_catalogs here? + // load grid corrections here (specified by -grid-meanfile) + GridCorrectionLoad (GRID_MEANFILE); - // load the ZP corrections here - if (SYNTH_ZERO_POINTS) SynthZeroPointsLoad (SYNTH_ZERO_POINTS); - - // load data from each region file, only use bright stars - for (i = 0; i < skylist[0].Nregions; i++) { - - // does this host ID match the desired location for the table? - if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; - - // set up the basic catalog info - char hostfile[1024]; - snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); - - dvo_catalog_init (&catalog, TRUE); - catalog.filename = hostID ? hostfile : skylist[0].filename[i]; - - // only update existing db tables - int status = stat (catalog.filename, &filestat); - if ((status == -1) && (errno == ENOENT)) { - if (VERBOSE) fprintf (stderr, "no file %s, skipping\n", catalog.filename); - continue; - } - - catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; - // if we are going to update the format, we should update all tables - if (UPDATE_CATFORMAT) { - catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; - } - catalog.Nsecfilt = GetPhotcodeNsecfilt (); - - if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { - fprintf (stderr, "ERROR: failure reading catalog %s, skipping\n", catalog.filename); - continue; - } - if (!catalog.Naverage_disk) { - if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); - dvo_catalog_unlock (&catalog); - dvo_catalog_free (&catalog); - continue; - } - - // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images) - ResetAverageObjects(&catalog); - - populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE); - - // XXX need to worry about the image subset data - initImageBins (&catalog, 1, FALSE); - initMosaicBins (&catalog, 1, FALSE); - // XXX reload_catalogs.c calls initTGroupBins here - - initGridBins (); - - findImages (&catalog, 1, FALSE); - findMosaics (&catalog, 1, FALSE); - // XXX reload_catalogs.c calls findTGroups here - - initMrel (&catalog, 1); - setMrelFinal (&catalog, TRUE); - - // uncomment for extra verbosity - // MARKTIME("setMrelFinal for "OFF_T_FMT" average "OFF_T_FMT" measure : %f sec\n", catalog.Naverage, catalog.Nmeasure, dtime); - // gettimeofday (&startTimer, NULL); // reset timer - - // XXX if we want to have options for setting warp, chip, stack independently, we need to init only the desired ones - // NOTE flatcorr == NULL, but it should have been applied already by setphot - - if (!UPDATE) { - dvo_catalog_unlock (&catalog); - free_tiny_values(&catalog); - dvo_catalog_free (&catalog); - freeImageBins (1, FALSE); - freeMosaicBins (1, FALSE); - freeGridBins (); - continue; - } - - if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); - - struct timeval now; - gettimeofday (&now, (void *) NULL); - char *moddate = ohana_sec_to_date (now.tv_sec); - gfits_modify (&catalog.header, "RELPHOT", "%s", 1, moddate); - free (moddate); - - // we can optionally convert output format here - // but it would be better to define a dvo crawler program to do this - // catalog.catformat = DVO_FORMAT_PS1_V1; - - // modify the output format as desired (ignore current format on disk) - if (UPDATE_CATFORMAT) { - catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); - } - - SetProtect (TRUE); - if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } - if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } - SetProtect (FALSE); - - free_tiny_values(&catalog); - dvo_catalog_free (&catalog); - - freeImageBins (1, FALSE); - freeMosaicBins (1, FALSE); - freeGridBins (); - } + reload_catalogs (skylist, hostID, hostpath); freeImages(db.ftable.buffer); + freeGridBins (); + gfits_db_free (&db); return (TRUE); } - -// CATDIR is supplied globally -# define DEBUG 1 -int relphot_objects_parallel (SkyList *sky) { - - // launch the setphot_client jobs to the parallel hosts - - // load the list of hosts - HostTable *table = HostTableLoad (CATDIR, sky->hosts); - if (!table) { - fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); - exit (1); - } - - if (BOUNDARY_TREE) { - char *tmppath = abspath(BOUNDARY_TREE, DVO_MAX_PATH); - free (BOUNDARY_TREE); - BOUNDARY_TREE = tmppath; - } - - int i; - for (i = 0; i < table->Nhosts; i++) { - - // ensure that the paths are absolute path names - char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); - free (table->hosts[i].pathname); - table->hosts[i].pathname = tmppath; - - // options / arguments that can affect relphot_client -update-objects: - // VERBOSE, VERBOSE2 - // KEEP_UBERCAL - // RESET (-reset) - // TimeSelect -time - // DophotSelect - // (note that psfQF is applied rigidly at 0.85, as is the galaxy test) - // MAG_LIM - // SIGMA_LIM - // ImagSelect, ImagMin, ImagMax - // MaxDensityUse, MaxDensityValue - - char *command = NULL; - strextend (&command, "relphot_client -update-objects"); - strextend (&command, "-hostID %d", table->hosts[i].hostID); - strextend (&command, "-D CATDIR %s", CATDIR); - strextend (&command, "-hostdir %s", table->hosts[i].pathname); - strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); - strextend (&command, "-statmode %s", STATMODE); - strextend (&command, "-D CAMERA %s", CAMERA); - strextend (&command, "-D STAR_TOOFEW %d", STAR_TOOFEW); - strextend (&command, "-minerror %f", MIN_ERROR); - - if (VERBOSE) { strextend (&command, "-v"); } - if (VERBOSE2) { strextend (&command, "-vv"); } - if (RESET) { strextend (&command, "-reset"); } - if (RESET_ZEROPTS) { strextend (&command, "-reset-zpts"); } - if (RESET_FLATCORR) { strextend (&command, "-reset-flat"); } - if (!KEEP_UBERCAL) { strextend (&command, "-reset-ubercal"); } - if (PRESERVE_PS1) { strextend (&command, "-preserve-ps1"); } - if (IS_DIFF_DB) { strextend (&command, "-is-diff-db"); } - if (UPDATE) { strextend (&command, "-update"); } - if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } - if (BOUNDARY_TREE) { strextend (&command, "-boundary-tree %s", BOUNDARY_TREE); } - if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); } - // if (GRID_ZEROPT) { strextend (&command, "-grid %s", GRID_MEANFILE); } - if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } - if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } - if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } - - if (!(STAGES & STAGE_CHIP)) { strextend (&command, "-skip-chip"); } - if (!(STAGES & STAGE_WARP)) { strextend (&command, "-skip-warp"); } - if (!(STAGES & STAGE_STACK)) { strextend (&command, "-skip-stack"); } - - // if (SET_MREL_VERSION != 1) { strextend (&command, "-set-mrel-version %d", SET_MREL_VERSION); } - - // missing AreaSelect & TimeSelect - - fprintf (stderr, "command: %s\n", command); - - if (PARALLEL_MANUAL) { - free (command); - continue; - } - - if (PARALLEL_SERIAL) { - int status = system (command); - if (status) { - fprintf (stderr, "ERROR running relphot_client\n"); - exit (2); - } - } else { - // launch the job on the remote machine (no handshake) - int errorInfo = 0; - int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); - if (!pid) { - if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); - exit (1); - } - table->hosts[i].pid = pid; // save for future reference - } - free (command); - } - - if (PARALLEL_MANUAL) { - fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); - getchar(); - } - if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { - HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); - } - - FreeHostTable (table); - return TRUE; -} Index: /branches/eam_branches/relphot.20210521/src/setMrelFinal.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/setMrelFinal.c (revision 41622) +++ /branches/eam_branches/relphot.20210521/src/setMrelFinal.c (revision 41623) @@ -15,4 +15,5 @@ // this sets flags in the measureT element, not the measure element setExclusions (catalog, 1, VERBOSE); /* mark by area */ + // XXXXX THis probably does not make sense /* set catalog[0].found[i] = FALSE */ @@ -22,13 +23,4 @@ catalog[0].found_t[i] = FALSE; catalog[0].foundWarp_t[i] = FALSE; - } - - for (i = 0; i < catalog[0].Naverage; i++) { - catalog[0].average[i].psfQF = NAN; // force recalculation below - catalog[0].average[i].psfQFperf = NAN; // force recalculation below - catalog[0].average[i].stargal = NAN; // force recalculation below - catalog[0].average[i].photFlagsUpper = 0; // reset (will be re-calculated here) - catalog[0].average[i].photFlagsLower = 0; // reset (will be re-calculated here) - catalog[0].average[i].NwarpOK = 0; // reset (will be re-calculated here) }