Index: /trunk/Ohana/src/addstar/Makefile =================================================================== --- /trunk/Ohana/src/addstar/Makefile (revision 7690) +++ /trunk/Ohana/src/addstar/Makefile (revision 7691) @@ -93,8 +93,16 @@ LOAD-2MASS = \ $(SRC)/load2mass.$(ARCH).o \ -$(SRC)/load2mass_as_data.$(ARCH).o \ +$(SRC)/load2mass_as_rawdata.$(ARCH).o \ $(SRC)/get2mass_ops.$(ARCH).o \ -$(SRC)/ConfigInit.$(ARCH).o \ -$(SRC)/args.$(ARCH).o \ +$(SRC)/find_matches_refstars.$(ARCH).o \ +$(SRC)/load_pt_catalog.$(ARCH).o \ +$(SRC)/args_load2mass.$(ARCH).o \ +$(SRC)/sort_lists.$(ARCH).o \ +$(SRC)/replace_match.$(ARCH).o \ +$(SRC)/update_coords.$(ARCH).o \ +$(SRC)/gcatalog.$(ARCH).o \ +$(SRC)/mkcatalog.$(ARCH).o \ +$(SRC)/wcatalog.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ $(SRC)/Shutdown.$(ARCH).o \ $(SRC)/SetSignals.$(ARCH).o @@ -244,4 +252,8 @@ scan2mass.install : $(DESTBIN)/scan2mass +load2mass : $(BIN)/load2mass.$(ARCH) +$(BIN)/load2mass.$(ARCH) : $(LOAD-2MASS) +load2mass.install : $(DESTBIN)/load2mass + all: addstar addstarc addstard addstart Index: /trunk/Ohana/src/addstar/include/2mass.h =================================================================== --- /trunk/Ohana/src/addstar/include/2mass.h (revision 7690) +++ /trunk/Ohana/src/addstar/include/2mass.h (revision 7691) @@ -30,3 +30,7 @@ char *skipNbounds (char *line, char bound, int Nbound, int Nbyte); -e_time get2mass_time (char *ptr, char *buffer, int Nbound, int Nbyte); +e_time get2mass_time (char *ptr, int Nbound, int Nbyte); + +int load2mass_as_rawdata (SkyTable *skytable, char *filename, AddstarClientOptions options); +SkyTable *load2mass_acc (char *path, char *accel); +int get2mass_3star (Stars *star, char *line, int Nmax); Index: /trunk/Ohana/src/addstar/include/addstar.h =================================================================== --- /trunk/Ohana/src/addstar/include/addstar.h (revision 7690) +++ /trunk/Ohana/src/addstar/include/addstar.h (revision 7691) @@ -138,5 +138,4 @@ Stars *grefstars PROTO((char *file, int photcode, int *Nstars)); Stars *gstars PROTO((char *file, int *NSTARS, int photcode, Image *image)); -void help PROTO(()); int in_image PROTO((double r, double d, Image *image)); int load_pt_catalog PROTO((Catalog *catalog, SkyRegion *region)); /*** choose new name ***/ @@ -191,4 +190,5 @@ AddstarClientOptions args_client (int argc, char **argv, AddstarClientOptions options); +AddstarClientOptions args_load2mass (int argc, char **argv, AddstarClientOptions options); void args_server (int argc, char **argv); Index: /trunk/Ohana/src/addstar/src/Shutdown.c =================================================================== --- /trunk/Ohana/src/addstar/src/Shutdown.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/Shutdown.c (revision 7691) @@ -22,5 +22,5 @@ SetProtect (TRUE); - gfits_db_close (db); + if (db != NULL) gfits_db_close (db); fprintf (stderr, "ERROR: addstar halted\n"); exit (1); Index: /trunk/Ohana/src/addstar/src/args.c =================================================================== --- /trunk/Ohana/src/addstar/src/args.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/args.c (revision 7691) @@ -1,4 +1,4 @@ # include "addstar.h" -void help (void); +static void help (void); AddstarClientOptions args (int argc, char **argv, AddstarClientOptions options) { @@ -283,5 +283,5 @@ } -void help () { +static void help () { fprintf (stderr, "USAGE\n"); Index: /trunk/Ohana/src/addstar/src/args_client.c =================================================================== --- /trunk/Ohana/src/addstar/src/args_client.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/args_client.c (revision 7691) @@ -1,4 +1,4 @@ # include "addstar.h" -void help (void); +static void help (void); AddstarClientOptions args_client (int argc, char **argv, AddstarClientOptions options) { @@ -212,5 +212,5 @@ } -void help () { +static void help () { fprintf (stderr, "USAGE\n"); Index: /trunk/Ohana/src/addstar/src/args_load2mass.c =================================================================== --- /trunk/Ohana/src/addstar/src/args_load2mass.c (revision 7691) +++ /trunk/Ohana/src/addstar/src/args_load2mass.c (revision 7691) @@ -0,0 +1,110 @@ +# include "addstar.h" +static void help (void); + +AddstarClientOptions args_load2mass (int argc, char **argv, AddstarClientOptions options) { + + int N; + + /* check for help request */ + if (get_argument (argc, argv, "-help") || + get_argument (argc, argv, "-h")) { + help (); + } + + /*** check for command line options ***/ + + /* basic mode: image, list, refcat */ + options.mode = M_REFCAT; + + /*** provide additional data ***/ + /* restrict to a portion of the sky? (UNUSED) */ + UserPatch.Rmin = 0; + UserPatch.Rmax= 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + /* override any header PHOTCODE values */ + options.photcode = 0; + if ((N = get_argument (argc, argv, "-p"))) { + remove_argument (N, &argc, argv); + options.photcode = GetPhotcodeCodebyName (argv[N]); + remove_argument (N, &argc, argv); + } + + options.timeref = 0; + options.mosaic = FALSE; + options.existing_regions = FALSE; + options.skip_missed = FALSE; + options.closest = FALSE; + + /* only add to existing objects */ + options.only_match = FALSE; + if ((N = get_argument (argc, argv, "-only-match"))) { + options.only_match = TRUE; + remove_argument (N, &argc, argv); + } + /* replace measurement, don't duplicate (ref/cat only) */ + options.replace = FALSE; + if ((N = get_argument (argc, argv, "-replace"))) { + options.replace = TRUE; + remove_argument (N, &argc, argv); + } + + /* select quality flags */ + SELECT_2MASS_QUALITY = NULL; + if ((N = get_argument (argc, argv, "-2massquality"))) { + remove_argument (N, &argc, argv); + SELECT_2MASS_QUALITY = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + /* extra error messages */ + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + /* other defaults */ + options.nosort = FALSE; + options.update = FALSE; + options.only_images = FALSE; + options.calibrate = FALSE; + options.quality_airmass = FALSE; + ACCEPT_ASTROM = FALSE; + FORCE_READ = FALSE; + TEXTMODE = FALSE; + SUBPIX = FALSE; + DUMP = NULL; + + if (argc != 1) { + fprintf (stderr, "USAGE: load2mass\n"); + exit (2); + } + return (options); +} + +static void help () { + + fprintf (stderr, "USAGE\n"); + fprintf (stderr, " load2mass"); + fprintf (stderr, " add data from 2MASS catalog to fullsky\n\n"); + + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -p (photcode) : specify photcode (override header)\n"); + fprintf (stderr, " -only-match : only add measurements to existing objects\n"); + fprintf (stderr, " -replace : replace time/photcode measurements (no duplication)\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} Index: /trunk/Ohana/src/addstar/src/args_server.c =================================================================== --- /trunk/Ohana/src/addstar/src/args_server.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/args_server.c (revision 7691) @@ -1,4 +1,4 @@ # include "addstar.h" -void help (void); +static void help (void); void args_server (int argc, char **argv) { @@ -57,5 +57,5 @@ } -void help () { +static void help () { fprintf (stderr, "USAGE\n"); Index: /trunk/Ohana/src/addstar/src/find_matches_refstars.c =================================================================== --- /trunk/Ohana/src/addstar/src/find_matches_refstars.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/find_matches_refstars.c (revision 7691) @@ -322,4 +322,13 @@ } + free (X1); + free (Y1); + free (N1); + free (X2); + free (Y2); + free (N2); + free (next); + free (next_miss); + catalog[0].Naverage = Nave; catalog[0].Nmeasure = Nmeas; Index: /trunk/Ohana/src/addstar/src/get2mass_ops.c =================================================================== --- /trunk/Ohana/src/addstar/src/get2mass_ops.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/get2mass_ops.c (revision 7691) @@ -9,4 +9,8 @@ int get2mass_setup (int photcode) { + NAMED_PHOTCODE (TM_J, "2MASS_J"); + NAMED_PHOTCODE (TM_H, "2MASS_H"); + NAMED_PHOTCODE (TM_K, "2MASS_K"); + FilterSkip = TimeSkip = Qentry = 0; Photcode = photcode; @@ -31,33 +35,99 @@ } +// fill in the data for a single star. takes a pointer to the start of the line int get2mass_star (Stars *star, char *line, int Nmax) { - ptr = skipNbounds (p, '|', FilterSkip, Nmax); + char *ptr, qc; + double M, dM; + e_time time; + + ptr = skipNbounds (line, '|', FilterSkip, Nmax); if (ptr == NULL) Shutdown ("format error in 2mass"); M = strtod (ptr, NULL); ptr = skipNbounds (ptr, '|', 1, Nmax - (ptr - line)); dM = strtod (ptr, NULL); - time = get2mass_time (ptr, buffer, TimeSkip, Nbyte); + time = get2mass_time (ptr, TimeSkip, Nmax - (ptr - line)); /* filter on the ph_qual flag for this filter (field 19) */ if (SELECT_2MASS_QUALITY != NULL) { - ptr = skipNbounds (p, '|', 18, Nmax - (ptr - line)); + ptr = skipNbounds (line, '|', 18, Nmax); qc = ptr[Qentry]; if (strchr (SELECT_2MASS_QUALITY, qc) == NULL) return (FALSE); } - stars[0].M = M; - stars[0].dM = dM; - stars[0].code = photcode; - stars[0].t = time; - stars[0].found = -1; - - return TRUE; -} - -int get2mass_coords (Stars *star, double *R, double *D, int Nmax) { - - *R = strtod (p, NULL); - ptr = skipNbounds (p, '|', 1, Nmax); + star[0].M = M; + star[0].dM = dM; + star[0].code = Photcode; + star[0].t = time; + star[0].found = -1; + + return TRUE; +} + +// fill in the data for a JHK triplet star. takes a pointer to the start of the line +int get2mass_3star (Stars *star, char *line, int Nmax) { + + char *ptr; + char Jquality, Hquality, Kquality; + double J, dJ, H, dH, K, dK; + e_time time; + + ptr = line; + if (ptr == NULL) Shutdown ("format error in 2mass"); + + ptr = skipNbounds (ptr, '|', 6, Nmax - (ptr - line)); + J = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 1, Nmax - (ptr - line)); + dJ = strtod (ptr, NULL); + + ptr = skipNbounds (ptr, '|', 3, Nmax - (ptr - line)); + H = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 1, Nmax - (ptr - line)); + dH = strtod (ptr, NULL); + + ptr = skipNbounds (ptr, '|', 3, Nmax - (ptr - line)); + K = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 1, Nmax - (ptr - line)); + dK = strtod (ptr, NULL); + + /* filter on the ph_qual flag for this filter (field 19) */ + if (SELECT_2MASS_QUALITY != NULL) { + ptr = skipNbounds (ptr, '|', 3, Nmax - (ptr - line)); + Jquality = (strchr (SELECT_2MASS_QUALITY, ptr[0]) != NULL); + Hquality = (strchr (SELECT_2MASS_QUALITY, ptr[1]) != NULL); + Kquality = (strchr (SELECT_2MASS_QUALITY, ptr[2]) != NULL); + time = get2mass_time (ptr, 18, Nmax - (ptr - line)); + } else { + time = get2mass_time (ptr, 20, Nmax - (ptr - line)); + } + + star[0].M = J; + star[0].dM = dJ; + star[0].code = TM_J; + star[0].t = time; + star[0].found = -1; + + star[1].M = H; + star[1].dM = dH; + star[1].code = TM_H; + star[1].t = time; + star[1].found = -1; + + star[2].M = K; + star[2].dM = dK; + star[2].code = TM_K; + star[2].t = time; + star[2].found = -1; + + return TRUE; +} + +// fill in the coords for a single star. takes a pointer to the start of the line +int get2mass_coords (char *line, double *R, double *D, int Nmax) { + + char *ptr; + + *R = strtod (line, NULL); + ptr = skipNbounds (line, '|', 1, Nmax); *D = strtod (ptr, NULL); if (*D > 90) Shutdown ("weird DEC value: something is wrong"); Index: /trunk/Ohana/src/addstar/src/load2mass.c =================================================================== --- /trunk/Ohana/src/addstar/src/load2mass.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/load2mass.c (revision 7691) @@ -6,32 +6,23 @@ char *path; int i, N, Nrefcat; - SkyTable *sky; - - if (argc == 2) { - load2mass_as_data (argv[1]); - exit (0); - } + SkyTable *sky, *sky2mass; + AddstarClientOptions options; // need to construct these options with args_load2mass... options = ConfigInit (&argc, argv); - options = args (argc, argv, options); + options = args_load2mass (argc, argv, options); sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); - if (argc != 1) { - fprintf (stderr, "USAGE: load2mass [options: see addstar]\n"); - exit (2); - } - path = TWO_MASS_DIR_AS; // the accel.dat file has the raw filenames // test if the file exists, or else try the .gz version - sky = load2mass_acc (path, "accel.dat"); + sky2mass = load2mass_acc (path, "accel.dat"); - for (i = 0; i < sky[0].Nregions; i++) { + for (i = 0; i < sky2mass[0].Nregions; i++) { fprintf (stderr, "loading %s\n", sky[0].filename[i]); - load2mass_as_data (skytable, sky[0].filename[i], options); + load2mass_as_rawdata (sky, sky2mass[0].filename[i], options); } exit (0); Index: /trunk/Ohana/src/addstar/src/load2mass_as_rawdata.c =================================================================== --- /trunk/Ohana/src/addstar/src/load2mass_as_rawdata.c (revision 7691) +++ /trunk/Ohana/src/addstar/src/load2mass_as_rawdata.c (revision 7691) @@ -0,0 +1,196 @@ +# include "addstar.h" +# include "2mass.h" + +/* unlike the DR2 data, the AS data is NOT fixed bytes/row + * we need to handle fractional lines at the end of each read block + */ + +/* read in chunks of ~16MB */ +# define NBYTE 0x4000000 +# define NBREC 330 +# define DEBUG 1 + +int load2mass_as_rawdata (SkyTable *skytable, char *filename, AddstarClientOptions options) { + + int i, j, verbose; + int Nstars, NSTARS, Ntstars, NTSTARS; + int Nbyte, Nextra, Ntotal, offset; + + double Rmin, Rmax, Dmin, Dmax; + + FILE *f; + char *buffer, *p, *q; + + Stars *stars, **subset; + TMStars *tstars; + SkyList *skylist; + SkyRegion *region; + Catalog catalog; + + get2mass_setup (options.photcode); + + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + + ALLOCATE (buffer, char, NBYTE); + + // scan through the entire 2MASS file + f = fopen (filename, "r"); + if (f == NULL) Shutdown ("can't read 2mass data file: %s", filename); + // test if this is a raw datafile or gzipped... + + Nextra = 0; // number excess bytes from lsat partial row + Ntotal = 0; // track the total number of bytes read + while ((Nbyte = fread (&buffer[Nextra], 1, NBYTE-Nextra, f)) != 0) { + if (Nbyte == -1) Shutdown ("error reading from raw file %s", filename); + if (DEBUG) fprintf (stderr, "read %d bytes", Nbyte); + + Nbyte += Nextra; + + if (VERBOSE) fprintf (stderr, "."); + + /* find bounds on first complete line */ + p = buffer; + q = memchr (p, '\n', Nbyte); + if (q == NULL) Shutdown ("incomplete line at end of file\n"); + offset = p - buffer; // offset within this scan + + Ntstars = 0; + NTSTARS = 10000; + ALLOCATE (tstars, TMStars, NTSTARS); + + Rmin = 360.0; + Rmax = 0.0; + Dmin = +90.0; + Dmax = -90.0; + + // scan through entire buffer for star coords + while (1) { + get2mass_coords (p, &tstars[Ntstars].R, &tstars[Ntstars].D, Nbyte - offset); + tstars[Ntstars].offset = offset; // offset within scan + tstars[Ntstars].flag = FALSE; + + if (VERBOSE) { + Rmin = MIN (Rmin, tstars[Ntstars].R); + Rmax = MAX (Rmax, tstars[Ntstars].R); + Dmin = MIN (Dmin, tstars[Ntstars].D); + Dmax = MAX (Dmax, tstars[Ntstars].D); + } + + Ntstars ++; + CHECK_REALLOCATE (tstars, TMStars, NTSTARS, Ntstars, 10000); + + /* start of the next line */ + p = q + 1; + offset = p - buffer; // offset within this scan + if (offset == Nbyte) { + // last line in buffer is a complete line + Nextra = 0; + break; + } + /* end of the next line */ + q = memchr (p, '\n', Nbyte - offset); + if (q == NULL) { + // last, incomplete line in buffer + Nextra = Nbyte - offset; + break; + } + } + if (VERBOSE) fprintf (stderr, "read %d stars (%10.6f - %10.6f, %10.6f - %10.6f)\n", Ntstars, Rmin, Rmax, Dmin, Dmax); + + + // scan through the stars, loading the containing catalogs + // skip through table for unsaved stars + for (i = 0; i < Ntstars; i++) { + if (tstars[i].flag) continue; + + // collect stars in a new output catalog + Nstars = 0; + NSTARS = 3000; + ALLOCATE (stars, Stars, NSTARS); + + // identify the relevant catalog + skylist = SkyRegionByPoint (skytable, -1, tstars[i].R, tstars[i].D); + region = skylist[0].regions[0]; + if (DEBUG) fprintf (stderr, "writing to %s\n", skylist[0].filename[0]); + + for (j = i; j < Ntstars; j++) { + if (tstars[j].flag) continue; + + // check if in skyregion + if (tstars[j].R < region[0].Rmin) continue; + if (tstars[j].R > region[0].Rmax) continue; + if (tstars[j].D < region[0].Dmin) continue; + if (tstars[j].D > region[0].Dmax) continue; + + offset = tstars[j].offset; + + stars[Nstars+0].R = tstars[j].R; + stars[Nstars+0].D = tstars[j].R; + // stars[Nstars+1].R = tstars[j].R; + // stars[Nstars+1].D = tstars[j].R; + // stars[Nstars+2].R = tstars[j].R; + // stars[Nstars+2].D = tstars[j].R; + // get2mass_3star (&stars[Nstars], &buffer[offset], Nbyte - offset); + + get2mass_star (&stars[Nstars], &buffer[offset], Nbyte - offset); + tstars[j].flag = TRUE; + + Nstars ++; + CHECK_REALLOCATE (stars, Stars, NSTARS, Nstars, 3000); + } + if (!Nstars) { + free (stars); + continue; + } + if (DEBUG) fprintf (stderr, "selected %d stars (%10.6f - %10.6f, %10.6f - %10.6f\n", Nstars, + region[0].Rmin, region[0].Rmax, region[0].Dmin, region[0].Dmax); + + if (1) { + verbose = VERBOSE; + VERBOSE = FALSE; + // now we have all of the loaded stars in this catalog + catalog.filename = skylist[0].filename[0]; + load_pt_catalog (&catalog, skylist[0].regions[0]); + + // create the subset pointers + ALLOCATE (subset, Stars *, Nstars); + for (j = 0; j < Nstars; j++) subset[j] = &stars[j]; + + find_matches_refstars (skylist[0].regions[0], subset, Nstars, &catalog, options); + free (subset); + + wcatalog (&catalog); + unlock_catalog (&catalog); + // free (catalog.filename); + // XXX don't free this! it points to an element of the skytable + } + + free (stars); + VERBOSE = verbose; + } + free (tstars); + + // at end, p points at the start of last, partial line + if (Nextra) memmove (buffer, p, Nextra); + } + + if (VERBOSE) fprintf (stderr, "\n"); + + fclose (f); + free (buffer); + return (TRUE); +} + +/* + for each 2mass file: + for each data block + generate a table of: R, D, byte, flag + for each unsaved star + find containing catalog + load catalog + find all contained stars + add to catalog + save catalog + mark all contained stars +*/ + Index: /trunk/Ohana/src/addstar/src/load_pt_catalog.c =================================================================== --- /trunk/Ohana/src/addstar/src/load_pt_catalog.c (revision 7690) +++ /trunk/Ohana/src/addstar/src/load_pt_catalog.c (revision 7691) @@ -15,7 +15,9 @@ case 1: gcatalog (catalog); /* load from disk */ + if (VERBOSE) fprintf (stderr, "loading existing file %s\n", catalog[0].filename); break; case 2: mkcatalog (region, catalog); /* fills in new header info */ + if (VERBOSE) fprintf (stderr, "creating new file %s\n", catalog[0].filename); break; }