IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40881


Ignore:
Timestamp:
Sep 3, 2019, 11:38:21 AM (7 years ago)
Author:
eugene
Message:

reorg loadukirt_uhs to add from an outer loop

Location:
branches/eam_branches/ohana.20190329/src/addstar
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20190329/src/addstar/include/ukirt_uhs.h

    r40833 r40881  
     1
     2# define BUFFER_SIZE 0x100000
     3// # define BUFFER_SIZE 900
    14
    25// measure[1] = J
    … …  
    1720// AddstarClientOptions args_loadukirt_uhs_client (int *argc, char **argv, AddstarClientOptions options);
    1821
    19 int loadukirt_uhs_table (int Nstart, int Nend, SkyList *skylistInput, HostTable *hosts, char **filename, AddstarClientOptions *options);
     22int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options);
    2023
    2124UKIRT_UHS_Stars *loadukirt_uhs_make_subset (UKIRT_UHS_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset);
    … …  
    3942// UKIRT_UHS_Stars *loadukirt_uhs_load_stars (char *filename, int *nstars);
    4043
    41 UKIRT_UHS_Stars *loadukirt_uhs_readstars (char *filename, UKIRT_UHS_Stars *stars, int *nstars, AddstarClientOptions *options);
     44UKIRT_UHS_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars);
    4245
    4346int loadukirt_uhs_sortStars (UKIRT_UHS_Stars *stars, int Nstars);

  • branches/eam_branches/ohana.20190329/src/addstar/src/args_loadukirt_uhs.c

    r40832 r40881  
    116116  DUMP = NULL;
    117117
    118   if (*argc < 2) {
    119     fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile) [..more files]\n");
     118  if (*argc != 2) {
     119    fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile)\n");
    120120    exit (2);
    121121  }

  • branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs.c

    r40832 r40881  
    2727  skylist = SkyListByPatch (sky, -1, &UserPatch);
    2828
    29   int Nstart = 1;
    30   int Nend = argc;
    31   while (Nstart < Nend) {
    32     Nstart = loadukirt_uhs_table (Nstart, Nend, skylist, NULL, argv, &options);
    33   }
     29  loadukirt_uhs_table (skylist, argv[1], &options);
    3430
    3531  SkyTableFree (sky);

  • branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_readstars.c

    r40880 r40881  
    11# include "addstar.h"
    22# include "ukirt_uhs.h"
     3# define NSTARS_MAX 10000000
     4// # define NSTARS_MAX 10
    35
    46/* this function reads the values of interest from the UKIRT UHS CSV files:
    … …  
    7375    lineStatus &= readStatus; FIELD[Nelem] = dvalue; }
    7476
    75 UKIRT_UHS_Stars *loadukirt_uhs_readstars (char *filename, UKIRT_UHS_Stars *stars, int *nstars, AddstarClientOptions *options) {
     77static int Nline_read = 0; // track number of lines read so far (use to skip lines as well)
     78static int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark)
     79
     80UKIRT_UHS_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) {
    7681
    7782  int codeJ = GetPhotcodeCodebyName ("UKIRT_UHS_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_UHS_J");
    … …  
    8085  // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00");
    8186  // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n");
    82 
    83   // read in the full FITS files
    84   FILE *f = fopen (filename, "r");
    85   if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename);
    86 
    87   int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark)
    8887
    8988  int Nelem = 0;      // number of valid rows read (vector elements)
    … …  
    110109  ALLOCATE_PTR (jSeqNum,      int,      NELEM);
    111110
    112   // we allocate one extra byte into which we never read so there will always be a NULL terminating the string
    113   ALLOCATE_PTR (buffer, char, 0x10001);
    114   bzero (buffer, 0x10001);
    115 
    116   int Nline_read = 0; // track number of lines read so far (use to skip lines as well)
    117 
    118111  // we have a working buffer read from the file. we parse the lines in the working buffer
    119112  // until we reach the last chunk without an EOL char.  at that point, we shift the start
    … …  
    122115  // we treat \n\r pair as a single EOL char to handle mac files:
    123116
    124   int Nstart = 0; // location of the last valid byte in the buffer (start filling here)
     117  int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here)
    125118  int EndOfFile = FALSE;
    126   while (!EndOfFile) {
    127     int Nbytes = 0x10000 - Nstart;
     119  while (!EndOfFile && (Nelem < NSTARS_MAX)) {
     120    int Nbytes = BUFFER_SIZE - Nstart;
    128121    // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string
    129122    bzero (&buffer[Nstart], Nbytes + 1);
    … …  
    133126      break;
    134127    }
     128
    135129    // we still need to parse the rest of the buffer, but there might not be an EOL on the last line
    136130    if (Nread == 0) {
    … …  
    187181
    188182      // the start of the line is the 1st element (fields are 1-counting)
    189       jPARSE ( 1,  0, sourceID,     "sourceID");
     183      jPARSE ( 1,  1, sourceID,     "sourceID");
    190184      iPARSE ( 3,  1, frameSetID,   "frameSetID");
    191185      dPARSE ( 4,  3, ra,           "ra");
    … …  
    255249  int NSTARS = Nstars + 0.1*NstarsIn;
    256250
    257   if (!stars) {
    258     ALLOCATE (stars, UKIRT_UHS_Stars, NSTARS);
    259   } else {
    260     REALLOCATE (stars, UKIRT_UHS_Stars, NSTARS);
    261   }
     251  ALLOCATE_PTR (stars, UKIRT_UHS_Stars, NSTARS);
    262252
    263253  for (int i = 0; i < NstarsIn; i++) {
    … …  
    336326  FREE (jSeqNum);
    337327
    338   free (buffer);
    339 
    340328  *nstars = Nstars;
     329  *nstart = Nstart;
    341330  return (stars);
    342331}

  • branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_table.c

    r40832 r40881  
    11# include "addstar.h"
    22# include "ukirt_uhs.h"
    3 # define NSTARS_MAX 1000000
    43
    5 int loadukirt_uhs_table (int Nstart, int Nend, SkyList *skylistInput, HostTable *hosts, char **filename, AddstarClientOptions *options) {
    6   OHANA_UNUSED_PARAM(hosts);
     4// one pass reads a single file at a time
     5int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options) {
    76 
    8   int i;
     7  // open the input file here and pass the pointer below
     8  FILE *f = fopen (filename, "r");
     9  if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename);
    910
    10   int Nstars = 0;
    11   UKIRT_UHS_Stars *stars = NULL;
    12   for (i = Nstart; (Nstars < NSTARS_MAX) && (i < Nend); i++) {
    13     // read the next file and append to the current set of stars
    14     fprintf (stderr, "loading %s\n", filename[i]);
    15     stars = loadukirt_uhs_readstars (filename[i], stars, &Nstars, options);
    16   }
    17   Nstart = i; // we pass back the entry for the next file to be read
    18   if (!stars) return Nstart;
     11  // we allocate one extra byte into which we never read so there will always be a NULL terminating the string
     12  ALLOCATE_PTR (buffer, char, BUFFER_SIZE + 1);
     13  bzero (buffer, BUFFER_SIZE + 1);
    1914
    20   fprintf (stderr, "writing %d stars to dvo\n", Nstars);
     15  // starting point of valid data in buffer; this is updated on each pass to readstars
     16  int Nstart = 0;
    2117
    22   // sort the stars by RA
    23   loadukirt_uhs_sortStars (stars, Nstars);
     18  while (1) {
    2419
    25   // scan through the stars, loading the containing catalogs
    26   // skip through table for unsaved stars
    27   for (i = 0; i < Nstars; i++) {
    28     if (stars[i].flag) continue;
     20    // read the chunk from the file
     21    fprintf (stderr, "loading %s\n", filename);
    2922
    30     // scan forward until we read the UserPatch
    31     if (stars[i].average.R < UserPatch.Rmin) continue;
    32     if (stars[i].average.R > UserPatch.Rmax) break;
    33     if (stars[i].average.D < UserPatch.Dmin) continue;
    34     if (stars[i].average.D > UserPatch.Dmax) continue;
     23    // on each pass, we read a new chunk of data and receive Nstars
    3524
    36     // identify the relevant catalog
    37     SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D);
    38     if (skylist[0].Nregions == 0) {
     25    int Nstars = 0;
     26    UKIRT_UHS_Stars *stars = loadukirt_uhs_readstars (f, buffer, &Nstart, options, &Nstars);
     27    if (!Nstars) {
     28      fclose (f);
     29      free (buffer);
     30      return TRUE; // end of file reached
     31    }
     32
     33    fprintf (stderr, "writing %d stars to dvo\n", Nstars);
     34
     35    // sort the stars by RA
     36    loadukirt_uhs_sortStars (stars, Nstars);
     37
     38    // scan through the stars, loading the containing catalogs
     39    // skip through table for unsaved stars
     40    for (int i = 0; i < Nstars; i++) {
     41      if (stars[i].flag) continue;
     42
     43      // scan forward until we read the UserPatch
     44      if (stars[i].average.R < UserPatch.Rmin) continue;
     45      if (stars[i].average.R > UserPatch.Rmax) break;
     46      if (stars[i].average.D < UserPatch.Dmin) continue;
     47      if (stars[i].average.D > UserPatch.Dmax) continue;
     48
     49      // identify the relevant catalog
     50      SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D);
     51      if (skylist[0].Nregions == 0) {
     52        SkyListFree (skylist);
     53        continue;
     54      }
     55      SkyRegion *region = skylist[0].regions[0];
     56
     57      // select stars matching this region
     58      int Nsubset;
     59      UKIRT_UHS_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset);
     60
     61      // In parallel mode, write out the subset to a disk file.  Block until a remote host
     62      // is available.  In serial mode, just match against the appropriate region and save
     63      // NOTE: disable parallel mode for now:
     64      // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options);
     65      loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options);
     66      free (subset);
    3967      SkyListFree (skylist);
    40       continue;
    4168    }
    42     SkyRegion *region = skylist[0].regions[0];
    43 
    44     // select stars matching this region
    45     int Nsubset;
    46     UKIRT_UHS_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset);
    47 
    48     // In parallel mode, write out the subset to a disk file.  Block until a remote host
    49     // is available.  In serial mode, just match against the appropriate region and save
    50     // NOTE: disable parallel mode for now:
    51     // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options);
    52     loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options);
    53     free (subset);
    54     SkyListFree (skylist);
     69    free (stars);
    5570  }
    5671
    … …  
    5974  // harvest_all ();
    6075
    61   free (stars);
    62   return Nstart;
     76  // we should not actually reach this point
     77  return FALSE;
    6378}
    6479
     80/* modifying to read in smaller blocks of the input file at a time
     81
     82   each pass on readstars needs to ...
     83
     84*/
     85
Note: See TracChangeset for help on using the changeset viewer.