IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41623


Ignore:
Timestamp:
May 27, 2021, 3:55:13 PM (5 years ago)
Author:
eugene
Message:

consolidate reload_catalogs and relphot_objects; option to supply grid correction in relphot -averages and relphot -apply-offsets

Location:
branches/eam_branches/relphot.20210521/src
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/relphot.20210521/src/ResetOps.c

    r41620 r41623  
    104104}
    105105
    106 // Used in relphot_objects.c:
    107 void ResetAverageObjects (Catalog *catalog) {
    108 
    109   // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images)
    110   if (!RESET) return;
    111 
    112   int Nsecfilt = catalog->Nsecfilt;
    113 
    114   for (off_t j = 0; j < catalog->Naverage; j++) {
    115     catalog->average[j].flags    &= ~photomBitsAverage; // reset all photometry bits (but not astrom)
    116     catalog->average[j].psfQF     = NAN;     // reset (will be re-calculated here)
    117     catalog->average[j].psfQFperf = NAN;        // reset (will be re-calculated here)
    118     catalog->average[j].stargal   = NAN;        // reset (will be re-calculated here)
    119     catalog->average[j].photFlagsUpper = 0;     // reset (will be re-calculated here)
    120     catalog->average[j].photFlagsLower = 0;     // reset (will be re-calculated here)
    121     catalog->average[j].NwarpOK        = 0;     // reset (will be re-calculated here)
    122     for (off_t k = 0; k < Nsecfilt; k++) {
    123       dvo_secfilt_init (&catalog->secfilt[j*Nsecfilt + k], SECFILT_RESET_ALL);
    124     }
    125   }
    126 }
    127 
    128106// Used in setMrelFinal.c
    129107void ResetAverageAndMeasure (Catalog *catalog) {
    130 
    131   /*** RESET photometry flags and Mcal values ***/
    132   if (!RESET) return;
    133108
    134109  int Nsecfilt = GetPhotcodeNsecfilt ();
    … …  
    141116
    142117  for (off_t i = 0; i < catalog->Naverage; i++) {
     118    catalog[0].average[i].psfQF     = NAN;      // force recalculation in setMrelCatalog
     119    catalog[0].average[i].psfQFperf = NAN;      // force recalculation in setMrelCatalog
     120    catalog[0].average[i].stargal   = NAN;      // force recalculation in setMrelCatalog
     121    catalog[0].average[i].photFlagsUpper = 0;   // reset (will be re-calculated)
     122    catalog[0].average[i].photFlagsLower = 0;   // reset (will be re-calculated)
     123    catalog[0].average[i].NwarpOK        = 0;   // reset (will be re-calculated)
     124
     125    if (!RESET) continue;
     126
     127    catalog->average[i].flags    &= ~photomBitsAverage; // reset all photometry bits (but not astrom)
     128
    143129    for (int Ns = 0; Ns < Nsecfilt; Ns++) {
    144130       

  • branches/eam_branches/relphot.20210521/src/args.c

    r41620 r41623  
    387387  GRID_ZEROPT = FALSE;
    388388  GRID_MEANFILE = NULL; // this is set by GridCorrectionSave()
     389  GRID_ZPT_MODE = GRID_ZPT_MODE_NONE; // start with grid off
    389390  if ((N = get_argument (argc, argv, "-grid"))) {
    390391    remove_argument (N, &argc, argv);
    391392    GRID_ZEROPT = TRUE;
    392     GRID_ZPT_MODE  = GRID_ZPT_MODE_NONE; // start with grid off
     393  }
     394  if ((N = get_argument (argc, argv, "-grid-meanfile"))) {
     395    GRID_ZEROPT = TRUE;
     396    remove_argument (N, &argc, argv);
     397    GRID_MEANFILE = strcreate (argv[N]); // used by relphot -averages and relphot -apply-offsets
     398    remove_argument (N, &argc, argv);
    393399  }
    394400
    … …  
    554560    case SYNTH_PHOT:
    555561    case UPDATE_AVERAGES:
     562      // note: initialize sets PhotcodeList to the full set of average photcodes
     563      // if the mode is UPDATE_AVERAGES
    556564      if (argc != 1) relphot_usage();
    557565      break;

  • branches/eam_branches/relphot.20210521/src/reload_catalogs.c

    r41620 r41623  
    1717
    1818  struct timeval start, stop;
    19   double dtime = 0.0;
    20   double time1 = 0.0;
    21   double time2 = 0.0;
    22   double time3a = 0.0;
    23   double time3b = 0.0;
    24   double time4 = 0.0;
    25   double time5 = 0.0;
    26   double time6 = 0.0;
    27   double time7 = 0.0;
     19  double dtime = 0.0, time1 = 0.0, time2 = 0.0, time3 = 0.0;
     20  double time4 = 0.0, time5 = 0.0, time6 = 0.0, time7 = 0.0, time8 = 0.0;
    2821
    2922  // XXX need to decide how to determine PARALLEL mode...
    … …  
    8073    TIMESTAMP(time2);
    8174
    82     // XXX relphot_objects.c resets average and secfilt here
    83 
    8475    populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE);
    85     TIMESTAMP(time3a);
     76    TIMESTAMP(time3);
    8677
    8778    // XXX need to worry about the image subset data
    8879    initImageBins  (&catalog, 1, FALSE);
    89     initMosaicBins (&catalog, 1, FALSE);
    90     initTGroupBins (&catalog, 1);
    91 
    92     initGridBins   ();
    93     TIMESTAMP(time3b);
     80    TIMESTAMP(time4);
    9481
    9582    findImages (&catalog, 1, FALSE);
    96     findMosaics (&catalog, 1, FALSE);
    97     findTGroups (&catalog, 1);
    98 
    99     TIMESTAMP(time4);
     83    TIMESTAMP(time5);
    10084
    10185    initMrel (&catalog, 1);
    … …  
    10387    setMflatFromGrid (&catalog); // Mgrid is used to set Mflat; Mgrid is the ignored in setMrelFinal/setMrelCatalog.c
    10488    setMrelFinal (&catalog, FALSE);
    105     TIMESTAMP(time5);
    106 
    107     // XXX relphot_objects allows !UPDATE and skips
     89    TIMESTAMP(time6);
    10890
    10991    // modify the output format as desired (ignore current format on disk)
    … …  
    125107    free_tiny_values(&catalog);
    126108    dvo_catalog_free (&catalog);
    127     TIMESTAMP(time6);
    128 
    129     // XXX freeTgroupBins here:
     109    TIMESTAMP(time7);
    130110
    131111    freeImageBins (1, FALSE);
    132     freeMosaicBins (1, FALSE);
    133     freeTGroupBins (1);
    134     freeGridBins ();
    135     TIMESTAMP(time7);
     112    TIMESTAMP(time8);
    136113
    137114    if (hostID) {
    … …  
    140117  }
    141118
    142   fprintf (stderr, "time step 1  %10.3f sec : find catalog\n", time1);
    143   fprintf (stderr, "time step 2  %10.3f sec : load catalog\n", time2);
    144   fprintf (stderr, "time step 3  %10.3f sec : make tiny values\n", time3a);
    145   fprintf (stderr, "time step 4  %10.3f sec : init imbins\n",  time3b);
    146   fprintf (stderr, "time step 5  %10.3f sec : find images\n",  time4);
    147   fprintf (stderr, "time step 6  %10.3f sec : set Mrel\n",     time5);
    148   fprintf (stderr, "time step 7  %10.3f sec : save catalog\n", time6);
    149   fprintf (stderr, "time step 8  %10.3f sec : free catalog\n", time7);
     119  fprintf (stderr, "time step 1  %10.3f sec : find catalog\n",     time1);
     120  fprintf (stderr, "time step 2  %10.3f sec : load catalog\n",     time2);
     121  fprintf (stderr, "time step 3  %10.3f sec : make tiny values\n", time3);
     122  fprintf (stderr, "time step 4  %10.3f sec : init imbins\n",      time4);
     123  fprintf (stderr, "time step 5  %10.3f sec : find images\n",      time5);
     124  fprintf (stderr, "time step 6  %10.3f sec : set Mrel\n",         time6);
     125  fprintf (stderr, "time step 7  %10.3f sec : save catalog\n",     time7);
     126  fprintf (stderr, "time step 8  %10.3f sec : free catalog\n",     time8);
    150127}
    151128

  • branches/eam_branches/relphot.20210521/src/relphot.c

    r41606 r41623  
    3434  switch (mode) {
    3535    case UPDATE_IMAGES:
    36       // calculate zero points for images (may group my exposure [mosaic], night [tgroups]; may measure flat-correction)
     36      // calculate zero points for images (may group by exposure [mosaic], night [tgroups]; may measure flat-correction)
    3737      // IF CALLED WITH NLOOP == 0, DOES NOT LOAD bcatalog, just loads image table and generates ImageSubset.dat
    3838      // If called with -update, calls reload_catalogs() just like apply_offsets
    … …  
    7272        exit (2);
    7373      }
     74      // we do not need to load grid corrections here.  the value of
     75      // GRID_MEANFILE, if specified, is passed to the clients
    7476      reload_catalogs (skylist, 0, NULL);
    7577      relphot_free (sky, skylist);

  • branches/eam_branches/relphot.20210521/src/relphot_client.c

    r41615 r41623  
    9292    }
    9393
     94/* deprecated (absorbed into MODE_UPDATE)
    9495    case MODE_UPDATE_OBJECTS: {
    9596      // take the current set of detections and set the mean magnitudes
    … …  
    99100      break;
    100101    }
     102*/
    101103
    102104    case MODE_SYNTH_PHOT:
    … …  
    106108      break;
    107109
     110    case MODE_UPDATE_OBJECTS:
    108111    default:
    109112      fprintf (stderr, "impossible!");

  • branches/eam_branches/relphot.20210521/src/relphot_images.c

    r41620 r41623  
    161161  }
    162162
     163  // once the zero points have been calculated, they are reassigned to the
     164  // individual images.  At this point, the mosaic and tgroup values should not
     165  // be used.
     166  freeMosaics ();
     167  MOSAIC_ZEROPT = FALSE;
     168  TGROUP_ZEROPT = FALSE;
     169
    163170  reload_images (&db);
    164171
    … …  
    175182
    176183  if (UPDATE_CATFORMAT) {
    177     // ensure the db format is updated
     184    // ensure the Image db format is updated
    178185    db.format = dvo_catalog_catformat (UPDATE_CATFORMAT);
    179186    gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT);
    … …  
    186193
    187194  freeImages (db.ftable.buffer);
    188   freeMosaics ();
    189195  freeGridBins ();
    190196

  • branches/eam_branches/relphot.20210521/src/relphot_objects.c

    r41620 r41623  
    66// has been correctly propagated to the measurement, the average will respect that value.
    77
    8 int relphot_objects_parallel (SkyList *sky);
    9 
    108int relphot_objects (SkyList *skylist, int hostID, char *hostpath) {
    119
    12   off_t i;
    13   struct stat filestat;
    14 
    15   Catalog catalog;
     10  FITS_DB db;
    1611
    1712  INITTIME;
    1813
    19   // XXX need to decide how to determine PARALLEL mode...
    20   if (PARALLEL && !hostID) {
    21     relphot_objects_parallel (skylist);
    22     return TRUE;
    23   }
    24 
    25   FITS_DB db;
    26      
    2714  set_db (&db);
    2815  gfits_db_init (&db);
    … …  
    4027  MARKTIME("loaded images: %f sec\n", dtime);
    4128
    42   // XXX can I call reload_catalogs here?
     29  // load grid corrections here (specified by -grid-meanfile)
     30  GridCorrectionLoad (GRID_MEANFILE);
    4331
    44   // load the ZP corrections here
    45   if (SYNTH_ZERO_POINTS) SynthZeroPointsLoad (SYNTH_ZERO_POINTS);
    46 
    47   // load data from each region file, only use bright stars
    48   for (i = 0; i < skylist[0].Nregions; i++) {
    49 
    50     // does this host ID match the desired location for the table?
    51     if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
    52 
    53     // set up the basic catalog info
    54     char hostfile[1024];
    55     snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
    56 
    57     dvo_catalog_init (&catalog, TRUE);
    58     catalog.filename    = hostID ? hostfile : skylist[0].filename[i];
    59 
    60     // only update existing db tables
    61     int status = stat (catalog.filename, &filestat);
    62     if ((status == -1) && (errno == ENOENT)) {
    63       if (VERBOSE) fprintf (stderr, "no file %s, skipping\n", catalog.filename);
    64       continue;
    65     }
    66 
    67     catalog.catflags    = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
    68     // if we are going to update the format, we should update all tables
    69     if (UPDATE_CATFORMAT) {
    70       catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
    71     }
    72     catalog.Nsecfilt    = GetPhotcodeNsecfilt ();
    73 
    74     if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
    75       fprintf (stderr, "ERROR: failure reading catalog %s, skipping\n", catalog.filename);
    76       continue;
    77     }
    78     if (!catalog.Naverage_disk) {
    79       if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
    80       dvo_catalog_unlock (&catalog);
    81       dvo_catalog_free (&catalog);
    82       continue;
    83     }
    84 
    85     // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images)
    86     ResetAverageObjects(&catalog);
    87 
    88     populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE);
    89 
    90     // XXX need to worry about the image subset data
    91     initImageBins  (&catalog, 1, FALSE);
    92     initMosaicBins (&catalog, 1, FALSE);
    93     // XXX reload_catalogs.c calls initTGroupBins here
    94 
    95     initGridBins   ();
    96 
    97     findImages (&catalog, 1, FALSE);
    98     findMosaics (&catalog, 1, FALSE);
    99     // XXX reload_catalogs.c calls findTGroups here
    100 
    101     initMrel (&catalog, 1);
    102     setMrelFinal (&catalog, TRUE);
    103 
    104     // uncomment for extra verbosity
    105     // MARKTIME("setMrelFinal for "OFF_T_FMT" average "OFF_T_FMT" measure : %f sec\n", catalog.Naverage, catalog.Nmeasure, dtime);
    106     // gettimeofday (&startTimer, NULL); // reset timer
    107 
    108     // XXX if we want to have options for setting warp, chip, stack independently, we need to init only the desired ones
    109     // NOTE flatcorr == NULL, but it should have been applied already by setphot
    110 
    111     if (!UPDATE) {
    112       dvo_catalog_unlock (&catalog);
    113       free_tiny_values(&catalog);
    114       dvo_catalog_free (&catalog);
    115       freeImageBins (1, FALSE);
    116       freeMosaicBins (1, FALSE);
    117       freeGridBins ();
    118       continue;
    119     }
    120    
    121     if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename);
    122    
    123     struct timeval now;
    124     gettimeofday (&now, (void *) NULL);
    125     char *moddate = ohana_sec_to_date (now.tv_sec);
    126     gfits_modify (&catalog.header, "RELPHOT", "%s", 1, moddate);     
    127     free (moddate);
    128 
    129     // we can optionally convert output format here
    130     // but it would be better to define a dvo crawler program to do this
    131     // catalog.catformat = DVO_FORMAT_PS1_V1;
    132 
    133     // modify the output format as desired (ignore current format on disk)
    134     if (UPDATE_CATFORMAT) {
    135       catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
    136     }
    137    
    138     SetProtect (TRUE);
    139     if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }
    140     if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); }
    141     SetProtect (FALSE);
    142 
    143     free_tiny_values(&catalog);
    144     dvo_catalog_free (&catalog);
    145 
    146     freeImageBins (1, FALSE);
    147     freeMosaicBins (1, FALSE);
    148     freeGridBins ();
    149   }
     32  reload_catalogs (skylist, hostID, hostpath);
    15033
    15134  freeImages(db.ftable.buffer);
     35  freeGridBins ();
     36
    15237  gfits_db_free (&db);
    15338
    15439  return (TRUE);
    15540}
    156 
    157 // CATDIR is supplied globally
    158 # define DEBUG 1
    159 int relphot_objects_parallel (SkyList *sky) {
    160 
    161   // launch the setphot_client jobs to the parallel hosts
    162 
    163   // load the list of hosts
    164   HostTable *table = HostTableLoad (CATDIR, sky->hosts);
    165   if (!table) {
    166     fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
    167     exit (1);
    168   }   
    169 
    170   if (BOUNDARY_TREE) {
    171     char *tmppath = abspath(BOUNDARY_TREE, DVO_MAX_PATH);
    172     free (BOUNDARY_TREE);
    173     BOUNDARY_TREE = tmppath;
    174   }
    175 
    176   int i;
    177   for (i = 0; i < table->Nhosts; i++) {
    178 
    179     // ensure that the paths are absolute path names
    180     char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
    181     free (table->hosts[i].pathname);
    182     table->hosts[i].pathname = tmppath;
    183 
    184     // options / arguments that can affect relphot_client -update-objects:
    185     // VERBOSE, VERBOSE2
    186     // KEEP_UBERCAL
    187     // RESET (-reset)
    188     // TimeSelect -time
    189     // DophotSelect
    190     // (note that psfQF is applied rigidly at 0.85, as is the galaxy test)
    191     // MAG_LIM
    192     // SIGMA_LIM
    193     // ImagSelect, ImagMin, ImagMax
    194     // MaxDensityUse, MaxDensityValue
    195 
    196     char *command = NULL;
    197     strextend (&command, "relphot_client -update-objects");
    198     strextend (&command, "-hostID %d", table->hosts[i].hostID);
    199     strextend (&command, "-D CATDIR %s", CATDIR);
    200     strextend (&command, "-hostdir %s", table->hosts[i].pathname);
    201     strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
    202     strextend (&command, "-statmode %s", STATMODE);
    203     strextend (&command, "-D CAMERA %s", CAMERA);
    204     strextend (&command, "-D STAR_TOOFEW %d", STAR_TOOFEW);
    205     strextend (&command, "-minerror %f", MIN_ERROR);
    206 
    207     if (VERBOSE)           { strextend (&command, "-v"); }
    208     if (VERBOSE2)          { strextend (&command, "-vv"); }
    209     if (RESET)             { strextend (&command, "-reset"); }
    210     if (RESET_ZEROPTS)     { strextend (&command, "-reset-zpts"); }
    211     if (RESET_FLATCORR)    { strextend (&command, "-reset-flat"); }
    212     if (!KEEP_UBERCAL)     { strextend (&command, "-reset-ubercal"); }
    213     if (PRESERVE_PS1)      { strextend (&command, "-preserve-ps1"); }
    214     if (IS_DIFF_DB)        { strextend (&command, "-is-diff-db"); }
    215     if (UPDATE)            { strextend (&command, "-update"); }
    216     if (UPDATE_CATFORMAT)  { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
    217     if (BOUNDARY_TREE)     { strextend (&command, "-boundary-tree %s", BOUNDARY_TREE); }
    218     if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); }
    219     // if (GRID_ZEROPT)       { strextend (&command, "-grid %s", GRID_MEANFILE); }
    220     if (USE_BASIC_CHECK)   { strextend (&command, "-basic-image-search"); }
    221     if (USE_ALL_IMAGES)    { strextend (&command, "-use-all-images"); }
    222     if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); }
    223 
    224     if (!(STAGES & STAGE_CHIP))  { strextend (&command, "-skip-chip"); }
    225     if (!(STAGES & STAGE_WARP))  { strextend (&command, "-skip-warp"); }
    226     if (!(STAGES & STAGE_STACK)) { strextend (&command, "-skip-stack"); }
    227 
    228     // if (SET_MREL_VERSION != 1) { strextend (&command, "-set-mrel-version %d", SET_MREL_VERSION); }
    229 
    230     // missing AreaSelect & TimeSelect
    231 
    232     fprintf (stderr, "command: %s\n", command);
    233 
    234     if (PARALLEL_MANUAL) {
    235       free (command);
    236       continue;
    237     }
    238 
    239     if (PARALLEL_SERIAL) {
    240       int status = system (command);
    241       if (status) {
    242         fprintf (stderr, "ERROR running relphot_client\n");
    243         exit (2);
    244       }
    245     } else {
    246       // launch the job on the remote machine (no handshake)
    247       int errorInfo = 0;
    248       int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
    249       if (!pid) {
    250         if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
    251         exit (1);
    252       }
    253       table->hosts[i].pid = pid; // save for future reference
    254     }
    255     free (command);
    256   }
    257 
    258   if (PARALLEL_MANUAL) {
    259     fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
    260     getchar();
    261   }
    262   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
    263     HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
    264   }
    265 
    266   FreeHostTable (table);
    267   return TRUE;
    268 }     

  • branches/eam_branches/relphot.20210521/src/setMrelFinal.c

    r41620 r41623  
    1515  // this sets flags in the measureT element, not the measure element
    1616  setExclusions (catalog, 1, VERBOSE);  /* mark by area */
     17  // XXXXX THis probably does not make sense
    1718
    1819  /* set catalog[0].found[i] = FALSE */
    … …  
    2223    catalog[0].found_t[i] = FALSE;
    2324    catalog[0].foundWarp_t[i] = FALSE;
    24   }
    25 
    26   for (i = 0; i < catalog[0].Naverage; i++) {
    27     catalog[0].average[i].psfQF     = NAN;      // force recalculation below
    28     catalog[0].average[i].psfQFperf = NAN;      // force recalculation below
    29     catalog[0].average[i].stargal   = NAN;      // force recalculation below
    30     catalog[0].average[i].photFlagsUpper = 0;   // reset (will be re-calculated here)
    31     catalog[0].average[i].photFlagsLower = 0;   // reset (will be re-calculated here)
    32     catalog[0].average[i].NwarpOK        = 0;   // reset (will be re-calculated here)
    3325  }
    3426
Note: See TracChangeset for help on using the changeset viewer.