Index: trunk/Ohana/src/addstar/doc/database.txt =================================================================== --- trunk/Ohana/src/addstar/doc/database.txt (revision 5143) +++ (revision ) @@ -1,91 +1,0 @@ - -\begin{verbatim} - -typedef struct { - float R; /* RA in decimal degrees */ - float D; /* DEC in decimal degrees */ - short int M; /* thousandths of mag (-32.000 to 32.000 valid range) */ - unsigned short int Nm; /* number of measurements */ - unsigned short int Xp, Xm; /* chisq values in tenths */ - unsigned int offset; /* offset to first Measure-ment */ -} Average; /* = 20 bytes / average */ - -typedef struct { - char dR, dD; /* tenths of arcsec (-12.7 to +12.7 valid range) */ - short int M; /* thousandths of mag (-32.000 to 32.000 valid range) */ - unsigned char dM; /* thousandths of mag (0.000 -- 0.255 valid range) */ - float t; /* time in seconds (what is reference?) */ - unsigned int average; /* reference to corresponding Average entry, - upper byte of value contains flags. - limit of 16,777,215 stars (Naverage) - in a file (=0xFFFFFF). - flags = average & 0x1000000 */ -} Measure; /* = 13 bytes / measure */ - -# define BLEND_IMAGE 0X01000000 -# define BLEND_CATALOG 0X02000000 -# define UPPER_LIMIT 0X04000000 -# define CALIBRATED 0X08000000 - -\end{verbatim} - -The above two structures define the entries in the photometry -database. The database consists of a large number of files -representing a small patch on the sky (roughly 1.5 degree$^2$ in most -places). These files are organized into directories representing -bands of Declination. A reference file determines the coordinate -boundaries for each of the files so that a given point on the sky can -unambiguously be associated with a specific file in a specific -directory. The sky coordinates for each file is the same as those -used by the HST Guide Star catalog, except for the region around the -North celestial pole, for which all stars are included in a single -file. - -Within a given file, the data are stored in a binary format, with an -ASCII FITS-like header. The header is examply in the format of a -normal FITS header, but with the exception that all files have a fixed -number of header blocks (for now 3 blocks = 8640 bytes). This is done -to speed loading the header and finding the beginning of the binary -data. The number of 3 blocks seems quite generous, as currently only -a few FITS keywords have been defined for each file, basically -keywords to define the number of stars and the total number of -measurements stored in the file, as well as values to define the RA -and DEC range of the file. - -The first section of data following the header blocks consists of -average measurements for each uniquely observed star. Each star -occupies 20 bytes, the size of the Average structure defined above. -The Average structure contains the average Ra, Dec, and Magnitude for -the star, as well as the number of measurements, and \chisq\ values -for the magnitude and position. Finally, there is a 32 bit integer -which defines the offset to the first measurement for this star. This -offset is defined as the number of Measure records from the start of -the Measure structure. - -The second section of data, following the Average data contains all -measurements for each star listed in Average. Each measurement -occupies 13 bytes, the size of the Measure structure. This structure -contains the difference of this position from the average RA and DEC, -and the instrumental magnitude of this measurement (in the units -defined by the fstat program, which give m = -2.5*log(cts) + Mo, where -Mo is currently 24.5 [10/15]). There is also the magnitude error for -this measurement, the time of the measurement (in seconds relative to -a to-be-determined zero point), and a reference to the entry in the -Average structure so we can relate a given measurement with a given star. -This last entry also includes a byte of flags, of which only 4 have -currently been defined. This means the Average offset can only be as -large as 16,777,215 (0xffffff), limiting the possible number of stars -allowed in a given file. This does not seem like a long term problem, -though: aside from the fact that this number is very large and we -only expect in the vicinity of 20,000 stars per file, the file can -easily be divided into pieces at a later date if needed. this last -step is trivial, consisting of splitting the data up into smaller -RA,DEC regions, and updating the reference catalog. With the above -definitions for the Average and Measure structures, we will typically -expect 20,000 * 15 measurements per year and 20,000 average entries in -a given file. This implies a file size of about 4.3 MB at the end of -a year. It is also possible that we will choose to split the files up -in the future if the number of measurements makes their size -unweildy. 4.3 MB is sufficiently small that this is not a problem, -but after only 5 years, the files will be over 21 MB each, getting to -be fairly significant. Index: trunk/Ohana/src/addstar/doc/description.txt =================================================================== --- trunk/Ohana/src/addstar/doc/description.txt (revision 5143) +++ (revision ) @@ -1,94 +1,0 @@ - -ADDSTAR: - -This program takes a photometry file, with astrometry, provided by -earlier routines and incorporates the stellar measurements into the -photometry database. This routine does NOT try to calibrate or check -the photometry, nor does it try to improve the coordinate -determinations or calculate chisquares for position or photometry - -those tasks are performed by other routines. - -Addstar starts by loading in the photometry file and converting the -pixel coordinates to sky coordinates using the astrometry information -in the header of the file. It also gets the detection limits and -stores the image astrometry information. - -Next, addstar determines which of all the previous images overlaps -with this image and also which of the catalog regions overlaps with -this image. - -Addstar considers each catalog region in turn, and compares the -positions of stars in the catalog with stars in the image. If a -catalog star is detected, a new measurement is added to the catalog. -If a catalog star is not detected, but is in the field of view of this -image, the detection limit for this image is added to the list of -measurements as an upper limit. If the image contains a star which is -not already included in the catalog, the star is added to the catalog, -and all previous images are checked to see if they included this -location. If so, upper limits are added to the list of measurements. - -To deal with blended images, we have taken the philosophy that all -measurements compatible with the coordinates of a given star should be -included in the list of measurements. This assumes that programs or people -downstream will figure out which is the "correct" measurement. To -this end, if a catalog star is consistent with multiple measurements, -they are all added to the catalog measurement and the measurements are -given a flag to say that there was blending in the catalog (ie, it is -probably the catalog star which is a blend). If multiple catalog -stars are consistent with the same image star, that measurement is -added to each catalog star, and given a flag to say that there was -blending on the image. - -After all catalogs have been checked, updated, and written to disk, -the image is added to the database of images. - -The format for a catalog region file is as follows: - --- -Header --- -Averages --- -Measurements --- - -The Header follows the format of a FITS header but always stored with -3 blocks to speed up read time: 2880 x 3 bytes. Two important -keywords in the header: NSTARS & NMEAS - -All average values are stored consecutively in a single block. They -are written as an array with NSTARS elements of the Average structure -defined below. The individual measurements follow the averages as a -block. To find a measurement for a given star, you need to get the -value of the offset for the star. this is the element number for the -first measurement of this star, and are followed by next measurements -for this star, until the value Nm is reached. - -Here are the structures being used for the average and measurement -values: - -typedef struct { - float R; /* RA in decimal degrees */ - float D; /* DEC in decimal degrees */ - short int M; /* thousandths of mag (-32.000 to 32.000 valid range) */ - unsigned short int Nm; /* number of measurements */ - unsigned short int Xp, Xm; /* chisq values in tenths */ - unsigned int offset; /* offset to first measurement */ -} Average; /* = 20 bytes / average */ - -typedef struct { - char dR, dD; /* tenths of arcsec (-12.7 to +12.7 valid range) */ - short int M; /* thousandths of mag (-32.000 to 32.000 valid range) */ - unsigned char dM; /* thousandths of mag (0.000 -- 0.255 valid range) */ - float t; /* time in seconds (what is reference?) */ - unsigned int average; - /* reference to corresponding average entry, upper byte stores flags: - limit of 16,777,215 stars (Naverage) in a file (=0xFFFFFF) - flags: average & 0x1000000 */ -} Measure; /* = 13 bytes / measure */ - -/* flags for measurements: */ -# define BLEND_IMAGE 0X01000000 -# define BLEND_CATALOG 0X02000000 -# define UPPER_LIMIT 0X04000000 -# define CALIBRATED 0X08000000 Index: trunk/Ohana/src/addstar/doc/merge.txt =================================================================== --- trunk/Ohana/src/addstar/doc/merge.txt (revision 5143) +++ (revision ) @@ -1,30 +1,0 @@ - -the goal of this project is to merge the functionality of the four -addstar-like programs in DVO: addstar, addrefs, addspphot, addusno. - -files which are identical or could be in all four: - -addstar/src/aregion.c -addstar/src/addstar.c -addstar/src/args.c -addstar/src/calibrate.c -addstar/src/check_permissions.c -addstar/src/ConfigInit.c -addstar/src/find_matches.c -addstar/src/gcatalog.c -addstar/src/gimages.c -addstar/src/gregions.c -addstar/src/gstars.c -addstar/src/mkcatalog.c -addstar/src/parse_time.c -addstar/src/sort_lists.c -addstar/src/wcatalog.c -addstar/src/wimage.c - -files to watch: -[eugene@kawelu src]$ ll addstar/src/ --rw-rw-r-- 1 eugene eugene 3658 Nov 24 14:13 calibrate.c - -[eugene@kawelu src]$ ll addspphot/src/ -total 116 --rw-rw-r-- 1 eugene eugene 4480 Nov 24 15:18 calibrate.c Index: trunk/Ohana/src/addstar/include/addstar.h =================================================================== --- trunk/Ohana/src/addstar/include/addstar.h (revision 5143) +++ trunk/Ohana/src/addstar/include/addstar.h (revision 5144) @@ -74,4 +74,5 @@ int XMIN, XMAX, YMIN, YMAX; int ACCEPT_ASTROM; +int TEXTMODE; PhotCode *thiscode; Index: trunk/Ohana/src/addstar/src/ConfigInit.c =================================================================== --- trunk/Ohana/src/addstar/src/ConfigInit.c (revision 5143) +++ trunk/Ohana/src/addstar/src/ConfigInit.c (revision 5144) @@ -35,6 +35,4 @@ ScanConfig (config, "IMAGE_CATALOG", "%s", 0, ImageCat); - ScanConfig (config, "IMAGE_CATALOG_TEMPLATE", "%s", 0, ImageTemplate); - ScanConfig (config, "CATALOG_TEMPLATE", "%s", 0, CatTemplate); ScanConfig (config, "GSCFILE", "%s", 0, GSCFILE); ScanConfig (config, "CATDIR", "%s", 0, CATDIR); Index: trunk/Ohana/src/addstar/src/args.c =================================================================== --- trunk/Ohana/src/addstar/src/args.c (revision 5143) +++ trunk/Ohana/src/addstar/src/args.c (revision 5144) @@ -183,4 +183,10 @@ remove_argument (N, &argc, argv); } + /* force read of image database with mismatched NSTARS & size */ + TEXTMODE = FALSE; + if ((N = get_argument (argc, argv, "-textmode"))) { + TEXTMODE = TRUE; + remove_argument (N, &argc, argv); + } /* extra error messages */ VERBOSE = FALSE; Index: trunk/Ohana/src/addstar/src/gstars.c =================================================================== --- trunk/Ohana/src/addstar/src/gstars.c (revision 5143) +++ trunk/Ohana/src/addstar/src/gstars.c (revision 5144) @@ -166,8 +166,12 @@ extend = FALSE; fits_scan (&header, "EXTEND", "%t", 1, &extend); - if (extend) { + if (extend && !TEXTMODE) { Nbytes = fits_matrix_size (&header); fseek (f, Nbytes, SEEK_CUR); rdstars = rfits (f, &image[0].nstar); + if (rdstars == NULL) { + fprintf (stderr, "ERROR: failed to read fits table\n"); + exit (1); + } } else { /* allocate space for stars */ Index: trunk/Ohana/src/imregister/src/photcode.c =================================================================== --- trunk/Ohana/src/imregister/src/photcode.c (revision 5143) +++ trunk/Ohana/src/imregister/src/photcode.c (revision 5144) @@ -1,4 +1,4 @@ # include "imregister.h" -static char *version = "photcode $Revision: 3.1 $"; +static char *version = "photcode $Revision: 3.2 $"; int main (int argc, char **argv) { @@ -61,5 +61,5 @@ } if (ccd == -1) { - fprintf (stderr, "warning: ccd %s not found in camera config file\n", ID); + fprintf (stderr, "warning: ccd %s not found in camera config file\n", ccd); ccd = 0; } Index: trunk/Ohana/src/libohana/src/LoadPhotcodes.c =================================================================== --- trunk/Ohana/src/libohana/src/LoadPhotcodes.c (revision 5143) +++ trunk/Ohana/src/libohana/src/LoadPhotcodes.c (revision 5144) @@ -485,5 +485,5 @@ int Np; - short M; + float M; Np = photcodes[0].hashcode[measure[0].source]; @@ -672,5 +672,5 @@ int i, Ns; - short Mave, Mref, Mcol, mc; + float Mave, Mref, Mcol, mc; double Mc; @@ -697,5 +697,5 @@ int Ns; - short Mave; + float Mave; Ns = photcodes[0].hashNsec[code[0].code]; @@ -708,5 +708,5 @@ int Ns; - short dM; + float dM; Ns = photcodes[0].hashNsec[code[0].code]; Index: trunk/Ohana/src/opihi/dvo/Makefile =================================================================== --- trunk/Ohana/src/opihi/dvo/Makefile (revision 5143) +++ trunk/Ohana/src/opihi/dvo/Makefile (revision 5144) @@ -28,4 +28,5 @@ $(SDIR)/ImageSelection.$(ARCH).o \ $(SDIR)/LoadImages.$(ARCH).o \ +$(SDIR)/cmpRead.$(ARCH).o \ $(SDIR)/aregion.$(ARCH).o \ $(SDIR)/compare.$(ARCH).o \ Index: trunk/Ohana/src/opihi/dvo/cmpread.c =================================================================== --- trunk/Ohana/src/opihi/dvo/cmpread.c (revision 5143) +++ trunk/Ohana/src/opihi/dvo/cmpread.c (revision 5144) @@ -1,67 +1,61 @@ # include "dvo1.h" -# define D_NSTARS 1000 -# define BYTES_STAR 66 -# define BLOCK 1000 + +/* add others as needed */ +enum {F_NONE, F_RA, F_DEC, F_X, F_Y, F_MAG, F_DMAG, F_TYPE, F_SKY, F_FX, F_FY, F_APMAG, F_GALMAG}; int cmpread (int argc, char **argv) { - int i, nstar, Nin, Nextra; - int doneread, done, Nskip, Nbytes, nbytes, Ninstar, RAnDEC; - char *c, *c2, *name, *file; - float *R, *D, *V; - double tR, tD, tmp, X, Y; + int i, field, extend, Nbytes, Nstars; + double tR, tD; + float value; FILE *f; - char *buffer; - int Nfield, Nra, Ndec; - Vector *rvec, *dvec, *vec; + Vector *vec; Header header; Coords coords; + CMPstars *stars; - if ((argc != 4) && (argc != 6)) { - fprintf (stderr, "USAGE: cmpread name Nfield \n"); - fprintf (stderr, "USAGE: cmpread ra N dec N \n"); + if (argc != 3) { + fprintf (stderr, "USAGE: cmpread field \n"); return (FALSE); } - R = D = V = NULL; - file = NULL; - name = NULL; - Nfield = Nra = Ndec = 0; - RAnDEC = FALSE; - - if (argc == 4) { - name = strcreate (argv[1]); - Nfield = atof (argv[2]); - file = strcreate (argv[3]); - RAnDEC = FALSE; - } else { - Nra = atof (argv[2]); - Ndec = atof (argv[4]); - file = strcreate (argv[5]); - RAnDEC = TRUE; - } - - if (!fits_read_header (file, &header)) { - fprintf (stderr, "ERROR: can't read header for %s\n", file); - free (file); - if (!RAnDEC) free (name); + field = F_NONE; + if (!strcasecmp (argv[1], "ra")) field = F_RA; + if (!strcasecmp (argv[1], "dec")) field = F_DEC; + if (!strcasecmp (argv[1], "mag")) field = F_MAG; + if (!strcasecmp (argv[1], "dmag")) field = F_DMAG; + if (!strcasecmp (argv[1], "x")) field = F_X; + if (!strcasecmp (argv[1], "y")) field = F_Y; + if (!strcasecmp (argv[1], "type")) field = F_TYPE; + if (!strcasecmp (argv[1], "sky")) field = F_SKY; + if (!strcasecmp (argv[1], "fx")) field = F_FX; + if (!strcasecmp (argv[1], "fy")) field = F_FY; + if (!strcasecmp (argv[1], "apmag")) field = F_APMAG; + if (!strcasecmp (argv[1], "galmag")) field = F_GALMAG; + if (field == F_NONE) { + fprintf (stderr, "invalid cmp field: %s\n", argv[1]); return (FALSE); } - /* find expected number of stars */ - fits_scan (&header, "NSTARS", "%d", 1, &nstar); - if (nstar == 0) { - fprintf (stderr, "ERROR: can't get NSTARS from header\n"); - free (file); - if (!RAnDEC) free (name); - fits_free_header (&header); + if ((vec = SelectVector (argv[1], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + /* load FITS header */ + if (!fits_read_header (argv[2], &header)) { + fprintf (stderr, "ERROR: can't read header for %s\n", argv[2]); return (FALSE); } - f = fopen (file, "r"); + if ((field == F_RA) || (field == F_DEC)) { + if (!GetCoords (&coords, &header)) { + fprintf (stderr, "can't get WCS info from header\n"); + fits_free_header (&header); + return (FALSE); + } + } + + /* re-open file to load data */ + f = fopen (argv[2], "r"); if (f == NULL) { - fprintf (stderr, "ERROR: can't read data from %s\n", file); - free (file); - if (!RAnDEC) free (name); + fprintf (stderr, "ERROR: can't read data from %s\n", argv[2]); fits_free_header (&header); return (FALSE); @@ -69,86 +63,80 @@ fseek (f, header.size, SEEK_SET); - if (RAnDEC) { - if ((rvec = SelectVector ("ra", ANYVECTOR, TRUE)) == NULL) return (FALSE); - if ((dvec = SelectVector ("dec", ANYVECTOR, TRUE)) == NULL) return (FALSE); - REALLOCATE (rvec[0].elements, float, nstar); - REALLOCATE (dvec[0].elements, float, nstar); - R = rvec[0].elements; - D = dvec[0].elements; - rvec[0].Nelements = dvec[0].Nelements = nstar; - if (!GetCoords (&coords, &header)) { - fprintf (stderr, "can't get WCS info from header\n"); - return (FALSE); - } - } else { - if ((vec = SelectVector (name, ANYVECTOR, TRUE)) == NULL) return (FALSE); - REALLOCATE (vec[0].elements, float, nstar); - V = vec[0].elements; - vec[0].Nelements = nstar; + /* find expected number of stars */ + if (!fits_scan (&header, "NSTARS", "%d", 1, &Nstars)) { + fprintf (stderr, "ERROR: can't get NSTARS from header\n"); + fits_free_header (&header); + return (FALSE); } - /* load in stars by blocks of 1000 */ - Nin = 0; - ALLOCATE (buffer, char, (BLOCK*BYTES_STAR) + 1); - buffer[BLOCK*BYTES_STAR] = 0; - Nextra = 0; - doneread = FALSE; - while (!doneread) { - Nbytes = BYTES_STAR * BLOCK - Nextra; - nbytes = fread (&buffer[Nextra], 1, Nbytes, f); - if (nbytes == 0) { - doneread = TRUE; - continue; + /* read from FITS table or from text table */ + extend = FALSE; + fits_scan (&header, "EXTEND", "%t", 1, &extend); + if (extend) { + fprintf (stderr, "reading from FITS cmp file %s\n", argv[2]); + Nbytes = fits_matrix_size (&header); + fseek (f, Nbytes, SEEK_CUR); + stars = cmpReadFits (f, &Nstars); + } else { + /* allocate space for stars */ + fprintf (stderr, "reading from TEXT cmp file %s\n", argv[2]); + if (!fits_scan (&header, "NSTARS", "%d", 1, &Nstars)) { + fprintf (stderr, "ERROR: failed to find NSTARS\n"); + exit (1); } - nbytes += Nextra; - /* check line-by-line integrity */ - c = buffer; - done = FALSE; - while ((c < buffer + nbytes) && (!done)) { - for (c2 = c; *c2 == '\n'; c2++); - if (c2 > c) { /* extra return chars */ - memmove (c, c2, (int)(buffer + nbytes - c2)); - Nskip = c2 - c; - nbytes -= Nskip; - bzero (buffer + nbytes, Nskip); - } - c2 = strchr (c, '\n'); - if (c2 == (char *) NULL) { - done = TRUE; - continue; - } - c2++; - if ((c2 - c) != BYTES_STAR) { /* bad line, delete it */ - memmove (c, c2, (int)(buffer + nbytes - c2)); - Nskip = c2 - c; - nbytes -= Nskip; - bzero (buffer + nbytes, Nskip); - } else { - c = c2; - } - } - Ninstar = nbytes / BYTES_STAR; - Nextra = nbytes % BYTES_STAR; - for (i = 0; i < Ninstar; i++, Nin++) { - if (RAnDEC) { - dparse (&X, Nra, &buffer[i*BYTES_STAR]); - dparse (&Y, Ndec, &buffer[i*BYTES_STAR]); - XY_to_RD (&tR, &tD, X, Y, &coords); - R[Nin] = tR; D[Nin] = tD; - } else { - dparse (&tmp, Nfield, &buffer[i*BYTES_STAR]); - V[Nin] = tmp; - } - } + stars = cmpReadText (f, &Nstars); } fclose (f); + REALLOCATE (vec[0].elements, float, Nstars); + vec[0].Nelements = Nstars; + bzero (vec[0].elements, Nstars*sizeof(float)); + + for (i = 0; i < Nstars; i++) { + switch (field) { + case F_RA: + XY_to_RD (&tR, &tD, stars[i].X, stars[i].Y, &coords); + value = tR; + break; + case F_DEC: + XY_to_RD (&tR, &tD, stars[i].X, stars[i].Y, &coords); + value = tD; + break; + case F_X: + value = stars[i].X; + break; + case F_Y: + value = stars[i].Y; + break; + case F_MAG: + value = stars[i].M; + break; + case F_APMAG: + value = stars[i].Map; + break; + case F_GALMAG: + value = stars[i].Mgal; + break; + case F_DMAG: + value = stars[i].dM; + break; + case F_TYPE: + value = stars[i].dophot; + break; + case F_SKY: + value = stars[i].sky; + break; + case F_FX: + value = stars[i].fx; + break; + case F_FY: + value = stars[i].fy; + break; + } + vec[0].elements[i] = value; + } + free (stars); fits_free_header (&header); - free (buffer); - free (file); - if (!RAnDEC) free (name); - - fprintf (stderr, "loaded %d objects\n", Nin); + fprintf (stderr, "loaded %d objects\n", Nstars); return (TRUE); } - Index: trunk/Ohana/src/opihi/include/dvo1.h =================================================================== --- trunk/Ohana/src/opihi/include/dvo1.h (revision 5143) +++ trunk/Ohana/src/opihi/include/dvo1.h (revision 5144) @@ -22,4 +22,19 @@ double RA0, RA1, DEC0, DEC1; } RegionFile; + +typedef struct { + double X; + double Y; + double R; + double D; + double M, dM; + char dophot; + double sky; + double fx, fy, df; + double Mgal, Map; + int found; + short int code; + e_time t; +} CMPstars; /*** dvo prototypes ***/ @@ -73,4 +88,6 @@ void sort_images PROTO((Image *image, int N)); void sortave PROTO((Average *ave, int N)); +CMPstars *cmpReadFits (FILE *f, int *nstars); +CMPstars *cmpReadText (FILE *f, int *nstars); # endif