Index: /branches/eam_branches/ohana.20190329/src/addstar/include/ukirt_uhs.h =================================================================== --- /branches/eam_branches/ohana.20190329/src/addstar/include/ukirt_uhs.h (revision 40880) +++ /branches/eam_branches/ohana.20190329/src/addstar/include/ukirt_uhs.h (revision 40881) @@ -1,2 +1,5 @@ + +# define BUFFER_SIZE 0x100000 +// # define BUFFER_SIZE 900 // measure[1] = J @@ -17,5 +20,5 @@ // AddstarClientOptions args_loadukirt_uhs_client (int *argc, char **argv, AddstarClientOptions options); -int loadukirt_uhs_table (int Nstart, int Nend, SkyList *skylistInput, HostTable *hosts, char **filename, AddstarClientOptions *options); +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options); UKIRT_UHS_Stars *loadukirt_uhs_make_subset (UKIRT_UHS_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset); @@ -39,5 +42,5 @@ // UKIRT_UHS_Stars *loadukirt_uhs_load_stars (char *filename, int *nstars); -UKIRT_UHS_Stars *loadukirt_uhs_readstars (char *filename, UKIRT_UHS_Stars *stars, int *nstars, AddstarClientOptions *options); +UKIRT_UHS_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); int loadukirt_uhs_sortStars (UKIRT_UHS_Stars *stars, int Nstars); Index: /branches/eam_branches/ohana.20190329/src/addstar/src/args_loadukirt_uhs.c =================================================================== --- /branches/eam_branches/ohana.20190329/src/addstar/src/args_loadukirt_uhs.c (revision 40880) +++ /branches/eam_branches/ohana.20190329/src/addstar/src/args_loadukirt_uhs.c (revision 40881) @@ -116,6 +116,6 @@ DUMP = NULL; - if (*argc < 2) { - fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile) [..more files]\n"); + if (*argc != 2) { + fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile)\n"); exit (2); } Index: /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs.c =================================================================== --- /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs.c (revision 40880) +++ /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs.c (revision 40881) @@ -27,9 +27,5 @@ skylist = SkyListByPatch (sky, -1, &UserPatch); - int Nstart = 1; - int Nend = argc; - while (Nstart < Nend) { - Nstart = loadukirt_uhs_table (Nstart, Nend, skylist, NULL, argv, &options); - } + loadukirt_uhs_table (skylist, argv[1], &options); SkyTableFree (sky); Index: /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_readstars.c =================================================================== --- /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_readstars.c (revision 40880) +++ /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_readstars.c (revision 40881) @@ -1,4 +1,6 @@ # include "addstar.h" # include "ukirt_uhs.h" +# define NSTARS_MAX 10000000 +// # define NSTARS_MAX 10 /* this function reads the values of interest from the UKIRT UHS CSV files: @@ -73,5 +75,8 @@ lineStatus &= readStatus; FIELD[Nelem] = dvalue; } -UKIRT_UHS_Stars *loadukirt_uhs_readstars (char *filename, UKIRT_UHS_Stars *stars, int *nstars, AddstarClientOptions *options) { +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_UHS_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { int codeJ = GetPhotcodeCodebyName ("UKIRT_UHS_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_UHS_J"); @@ -80,10 +85,4 @@ // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); - - // read in the full FITS files - FILE *f = fopen (filename, "r"); - if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename); - - int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark) int Nelem = 0; // number of valid rows read (vector elements) @@ -110,10 +109,4 @@ ALLOCATE_PTR (jSeqNum, int, NELEM); - // we allocate one extra byte into which we never read so there will always be a NULL terminating the string - ALLOCATE_PTR (buffer, char, 0x10001); - bzero (buffer, 0x10001); - - int Nline_read = 0; // track number of lines read so far (use to skip lines as well) - // we have a working buffer read from the file. we parse the lines in the working buffer // until we reach the last chunk without an EOL char. at that point, we shift the start @@ -122,8 +115,8 @@ // we treat \n\r pair as a single EOL char to handle mac files: - int Nstart = 0; // location of the last valid byte in the buffer (start filling here) + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) int EndOfFile = FALSE; - while (!EndOfFile) { - int Nbytes = 0x10000 - Nstart; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string bzero (&buffer[Nstart], Nbytes + 1); @@ -133,4 +126,5 @@ break; } + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line if (Nread == 0) { @@ -187,5 +181,5 @@ // the start of the line is the 1st element (fields are 1-counting) - jPARSE ( 1, 0, sourceID, "sourceID"); + jPARSE ( 1, 1, sourceID, "sourceID"); iPARSE ( 3, 1, frameSetID, "frameSetID"); dPARSE ( 4, 3, ra, "ra"); @@ -255,9 +249,5 @@ int NSTARS = Nstars + 0.1*NstarsIn; - if (!stars) { - ALLOCATE (stars, UKIRT_UHS_Stars, NSTARS); - } else { - REALLOCATE (stars, UKIRT_UHS_Stars, NSTARS); - } + ALLOCATE_PTR (stars, UKIRT_UHS_Stars, NSTARS); for (int i = 0; i < NstarsIn; i++) { @@ -336,7 +326,6 @@ FREE (jSeqNum); - free (buffer); - *nstars = Nstars; + *nstart = Nstart; return (stars); } Index: /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_table.c =================================================================== --- /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_table.c (revision 40880) +++ /branches/eam_branches/ohana.20190329/src/addstar/src/loadukirt_uhs_table.c (revision 40881) @@ -1,56 +1,71 @@ # include "addstar.h" # include "ukirt_uhs.h" -# define NSTARS_MAX 1000000 -int loadukirt_uhs_table (int Nstart, int Nend, SkyList *skylistInput, HostTable *hosts, char **filename, AddstarClientOptions *options) { - OHANA_UNUSED_PARAM(hosts); +// one pass reads a single file at a time +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options) { - int i; + // open the input file here and pass the pointer below + FILE *f = fopen (filename, "r"); + if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename); - int Nstars = 0; - UKIRT_UHS_Stars *stars = NULL; - for (i = Nstart; (Nstars < NSTARS_MAX) && (i < Nend); i++) { - // read the next file and append to the current set of stars - fprintf (stderr, "loading %s\n", filename[i]); - stars = loadukirt_uhs_readstars (filename[i], stars, &Nstars, options); - } - Nstart = i; // we pass back the entry for the next file to be read - if (!stars) return Nstart; + // we allocate one extra byte into which we never read so there will always be a NULL terminating the string + ALLOCATE_PTR (buffer, char, BUFFER_SIZE + 1); + bzero (buffer, BUFFER_SIZE + 1); - fprintf (stderr, "writing %d stars to dvo\n", Nstars); + // starting point of valid data in buffer; this is updated on each pass to readstars + int Nstart = 0; - // sort the stars by RA - loadukirt_uhs_sortStars (stars, Nstars); + while (1) { - // scan through the stars, loading the containing catalogs - // skip through table for unsaved stars - for (i = 0; i < Nstars; i++) { - if (stars[i].flag) continue; + // read the chunk from the file + fprintf (stderr, "loading %s\n", filename); - // scan forward until we read the UserPatch - if (stars[i].average.R < UserPatch.Rmin) continue; - if (stars[i].average.R > UserPatch.Rmax) break; - if (stars[i].average.D < UserPatch.Dmin) continue; - if (stars[i].average.D > UserPatch.Dmax) continue; + // on each pass, we read a new chunk of data and receive Nstars - // identify the relevant catalog - SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D); - if (skylist[0].Nregions == 0) { + int Nstars = 0; + UKIRT_UHS_Stars *stars = loadukirt_uhs_readstars (f, buffer, &Nstart, options, &Nstars); + if (!Nstars) { + fclose (f); + free (buffer); + return TRUE; // end of file reached + } + + fprintf (stderr, "writing %d stars to dvo\n", Nstars); + + // sort the stars by RA + loadukirt_uhs_sortStars (stars, Nstars); + + // scan through the stars, loading the containing catalogs + // skip through table for unsaved stars + for (int i = 0; i < Nstars; i++) { + if (stars[i].flag) continue; + + // scan forward until we read the UserPatch + if (stars[i].average.R < UserPatch.Rmin) continue; + if (stars[i].average.R > UserPatch.Rmax) break; + if (stars[i].average.D < UserPatch.Dmin) continue; + if (stars[i].average.D > UserPatch.Dmax) continue; + + // identify the relevant catalog + SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D); + if (skylist[0].Nregions == 0) { + SkyListFree (skylist); + continue; + } + SkyRegion *region = skylist[0].regions[0]; + + // select stars matching this region + int Nsubset; + UKIRT_UHS_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset); + + // In parallel mode, write out the subset to a disk file. Block until a remote host + // is available. In serial mode, just match against the appropriate region and save + // NOTE: disable parallel mode for now: + // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options); + loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options); + free (subset); SkyListFree (skylist); - continue; } - SkyRegion *region = skylist[0].regions[0]; - - // select stars matching this region - int Nsubset; - UKIRT_UHS_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset); - - // In parallel mode, write out the subset to a disk file. Block until a remote host - // is available. In serial mode, just match against the appropriate region and save - // NOTE: disable parallel mode for now: - // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options); - loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options); - free (subset); - SkyListFree (skylist); + free (stars); } @@ -59,6 +74,12 @@ // harvest_all (); - free (stars); - return Nstart; + // we should not actually reach this point + return FALSE; } +/* modifying to read in smaller blocks of the input file at a time + + each pass on readstars needs to ... + +*/ +