Changeset 41623 for branches/eam_branches/relphot.20210521/src/relphot_objects.c

Timestamp:

May 27, 2021, 3:55:13 PM (5 years ago)

Author:

eugene

Message:

consolidate reload_catalogs and relphot_objects; option to supply grid correction in relphot -averages and relphot -apply-offsets

File:

• branches/eam_branches/relphot.20210521/src/relphot_objects.c (modified) (2 diffs)

branches/eam_branches/relphot.20210521/src/relphot_objects.c

@@ -6 +6 @@
 // has been correctly propagated to the measurement, the average will respect that value.
 
-int relphot_objects_parallel (SkyList *sky);
-
 int relphot_objects (SkyList *skylist, int hostID, char *hostpath) {
 
-  off_t i;
-  struct stat filestat;
-
-  Catalog catalog;
+  FITS_DB db;
 
   INITTIME;
 
-  // XXX need to decide how to determine PARALLEL mode...
-  if (PARALLEL && !hostID) {
-    relphot_objects_parallel (skylist);
-    return TRUE;
-  }
-
-  FITS_DB db;
-      
   set_db (&db);
   gfits_db_init (&db);
@@ -40 +27 @@
   MARKTIME("loaded images: %f sec\n", dtime);
 
-  // XXX can I call reload_catalogs here?
+  // load grid corrections here (specified by -grid-meanfile)
+  GridCorrectionLoad (GRID_MEANFILE);
 
-  // load the ZP corrections here
-  if (SYNTH_ZERO_POINTS) SynthZeroPointsLoad (SYNTH_ZERO_POINTS);
-
-  // load data from each region file, only use bright stars
-  for (i = 0; i < skylist[0].Nregions; i++) {
-
-    // does this host ID match the desired location for the table?
-    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
-
-    // set up the basic catalog info
-    char hostfile[1024];
-    snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
-
-    dvo_catalog_init (&catalog, TRUE);
-    catalog.filename    = hostID ? hostfile : skylist[0].filename[i];
-
-    // only update existing db tables
-    int status = stat (catalog.filename, &filestat);
-    if ((status == -1) && (errno == ENOENT)) {
-      if (VERBOSE) fprintf (stderr, "no file %s, skipping\n", catalog.filename);
-      continue;
-    }
-
-    catalog.catflags    = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
-    // if we are going to update the format, we should update all tables
-    if (UPDATE_CATFORMAT) {
-      catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
-    }
-    catalog.Nsecfilt    = GetPhotcodeNsecfilt ();
-
-    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
-      fprintf (stderr, "ERROR: failure reading catalog %s, skipping\n", catalog.filename);
-      continue;
-    }
-    if (!catalog.Naverage_disk) {
-      if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
-      dvo_catalog_unlock (&catalog);
-      dvo_catalog_free (&catalog);
-      continue;
-    }
-
-    // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images)
-    ResetAverageObjects(&catalog);
-
-    populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE);
-
-    // XXX need to worry about the image subset data
-    initImageBins  (&catalog, 1, FALSE);
-    initMosaicBins (&catalog, 1, FALSE);
-    // XXX reload_catalogs.c calls initTGroupBins here 
-
-    initGridBins   (); 
-
-    findImages (&catalog, 1, FALSE);
-    findMosaics (&catalog, 1, FALSE);
-    // XXX reload_catalogs.c calls findTGroups here 
-
-    initMrel (&catalog, 1);
-    setMrelFinal (&catalog, TRUE);
-
-    // uncomment for extra verbosity
-    // MARKTIME("setMrelFinal for "OFF_T_FMT" average "OFF_T_FMT" measure : %f sec\n", catalog.Naverage, catalog.Nmeasure, dtime);
-    // gettimeofday (&startTimer, NULL); // reset timer
-
-    // XXX if we want to have options for setting warp, chip, stack independently, we need to init only the desired ones
-    // NOTE flatcorr == NULL, but it should have been applied already by setphot
-
-    if (!UPDATE) {
-      dvo_catalog_unlock (&catalog);
-      free_tiny_values(&catalog);
-      dvo_catalog_free (&catalog);
-      freeImageBins (1, FALSE);
-      freeMosaicBins (1, FALSE);
-      freeGridBins ();
-      continue;
-    }
-    
-    if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename);
-    
-    struct timeval now;
-    gettimeofday (&now, (void *) NULL);
-    char *moddate = ohana_sec_to_date (now.tv_sec);
-    gfits_modify (&catalog.header, "RELPHOT", "%s", 1, moddate);      
-    free (moddate);
-
-    // we can optionally convert output format here
-    // but it would be better to define a dvo crawler program to do this
-    // catalog.catformat = DVO_FORMAT_PS1_V1;
-
-    // modify the output format as desired (ignore current format on disk)
-    if (UPDATE_CATFORMAT) {
-      catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
-    }
-    
-    SetProtect (TRUE);
-    if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }
-    if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); }
-    SetProtect (FALSE);
-
-    free_tiny_values(&catalog);
-    dvo_catalog_free (&catalog);
-
-    freeImageBins (1, FALSE);
-    freeMosaicBins (1, FALSE);
-    freeGridBins ();
-  }
+  reload_catalogs (skylist, hostID, hostpath);
 
   freeImages(db.ftable.buffer);
+  freeGridBins ();
+
   gfits_db_free (&db);
 
   return (TRUE);
 }
-
-// CATDIR is supplied globally
-# define DEBUG 1
-int relphot_objects_parallel (SkyList *sky) {
-
-  // launch the setphot_client jobs to the parallel hosts
-
-  // load the list of hosts
-  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
-  if (!table) {
-    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
-    exit (1);
-  }    
-
-  if (BOUNDARY_TREE) {
-    char *tmppath = abspath(BOUNDARY_TREE, DVO_MAX_PATH);
-    free (BOUNDARY_TREE);
-    BOUNDARY_TREE = tmppath;
-  }
-
-  int i;
-  for (i = 0; i < table->Nhosts; i++) {
-
-    // ensure that the paths are absolute path names
-    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
-    free (table->hosts[i].pathname);
-    table->hosts[i].pathname = tmppath;
-
-    // options / arguments that can affect relphot_client -update-objects:
-    // VERBOSE, VERBOSE2
-    // KEEP_UBERCAL
-    // RESET (-reset)
-    // TimeSelect -time
-    // DophotSelect
-    // (note that psfQF is applied rigidly at 0.85, as is the galaxy test)
-    // MAG_LIM
-    // SIGMA_LIM
-    // ImagSelect, ImagMin, ImagMax
-    // MaxDensityUse, MaxDensityValue
-
-    char *command = NULL;
-    strextend (&command, "relphot_client -update-objects");
-    strextend (&command, "-hostID %d", table->hosts[i].hostID);
-    strextend (&command, "-D CATDIR %s", CATDIR);
-    strextend (&command, "-hostdir %s", table->hosts[i].pathname);
-    strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
-    strextend (&command, "-statmode %s", STATMODE);
-    strextend (&command, "-D CAMERA %s", CAMERA);
-    strextend (&command, "-D STAR_TOOFEW %d", STAR_TOOFEW);
-    strextend (&command, "-minerror %f", MIN_ERROR);
-
-    if (VERBOSE)           { strextend (&command, "-v"); }
-    if (VERBOSE2)          { strextend (&command, "-vv"); }
-    if (RESET)             { strextend (&command, "-reset"); }
-    if (RESET_ZEROPTS)     { strextend (&command, "-reset-zpts"); }
-    if (RESET_FLATCORR)    { strextend (&command, "-reset-flat"); }
-    if (!KEEP_UBERCAL)     { strextend (&command, "-reset-ubercal"); }
-    if (PRESERVE_PS1)      { strextend (&command, "-preserve-ps1"); }
-    if (IS_DIFF_DB)        { strextend (&command, "-is-diff-db"); }
-    if (UPDATE)            { strextend (&command, "-update"); }
-    if (UPDATE_CATFORMAT)  { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
-    if (BOUNDARY_TREE)     { strextend (&command, "-boundary-tree %s", BOUNDARY_TREE); }
-    if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); }
-    // if (GRID_ZEROPT)       { strextend (&command, "-grid %s", GRID_MEANFILE); } 
-    if (USE_BASIC_CHECK)   { strextend (&command, "-basic-image-search"); }
-    if (USE_ALL_IMAGES)    { strextend (&command, "-use-all-images"); }
-    if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); }
-
-    if (!(STAGES & STAGE_CHIP))  { strextend (&command, "-skip-chip"); }
-    if (!(STAGES & STAGE_WARP))  { strextend (&command, "-skip-warp"); }
-    if (!(STAGES & STAGE_STACK)) { strextend (&command, "-skip-stack"); }
-
-    // if (SET_MREL_VERSION != 1) { strextend (&command, "-set-mrel-version %d", SET_MREL_VERSION); } 
-
-    // missing AreaSelect & TimeSelect
-
-    fprintf (stderr, "command: %s\n", command);
-
-    if (PARALLEL_MANUAL) {
-      free (command);
-      continue;
-    }
-
-    if (PARALLEL_SERIAL) {
-      int status = system (command);
-      if (status) {
-        fprintf (stderr, "ERROR running relphot_client\n");
-        exit (2);
-      }
-    } else {
-      // launch the job on the remote machine (no handshake)
-      int errorInfo = 0;
-      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
-      if (!pid) {
-        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
-        exit (1);
-      }
-      table->hosts[i].pid = pid; // save for future reference
-    }
-    free (command);
-  }
-
-  if (PARALLEL_MANUAL) {
-    fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
-    getchar();
-  }
-  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
-  }
-
-  FreeHostTable (table);
-  return TRUE;
-}