Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile (revision 41170) @@ -27,4 +27,5 @@ loadgaia : $(BIN)/loadgaia.$(ARCH) loadgaia_dr2 : $(BIN)/loadgaia_dr2.$(ARCH) +loadukirt_uhs : $(BIN)/loadukirt_uhs.$(ARCH) loadstarpar : $(BIN)/loadstarpar.$(ARCH) loadstarpar_client : $(BIN)/loadstarpar_client.$(ARCH) @@ -44,5 +45,5 @@ # programs in 'SERVER' use the client-server concept and are out of date -INSTALL = addstar addstar_client sedstar loadgalphot loadgaia loadgaia_dr2 loadstarpar loadstarpar_client setobjflags setobjflags_client loadICRF loadICRF_client skycells mkcmf dumpskycells findskycell load2mass loadwise loadtycho loadbsc loadsupercos +INSTALL = addstar addstar_client sedstar loadgalphot loadgaia loadgaia_dr2 loadukirt_uhs loadstarpar loadstarpar_client setobjflags setobjflags_client loadICRF loadICRF_client skycells mkcmf dumpskycells findskycell load2mass loadwise loadtycho loadbsc loadsupercos SERVER = addstarc addstard addstart @@ -305,4 +306,26 @@ $(SRC)/psps_ids.$(ARCH).o +LOAD-UKIRT_UHS = \ +$(SRC)/loadukirt_uhs.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/args_loadukirt_uhs.$(ARCH).o \ +$(SRC)/find_matches_ukirt_uhs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_catalog.$(ARCH).o \ +$(SRC)/loadukirt_uhs_make_subset.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_uhs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ugcs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ugps.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ulas.$(ARCH).o \ +$(SRC)/loadukirt_uhs_table.$(ARCH).o \ +$(SRC)/parse_csv.$(ARCH).o \ +$(SRC)/resort_catalog.$(ARCH).o \ +$(SRC)/build_links.$(ARCH).o \ +$(SRC)/strhash.$(ARCH).o \ +$(SRC)/sortIDs.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o + LOAD-STARPAR = \ $(SRC)/loadstarpar.$(ARCH).o \ @@ -528,4 +551,5 @@ $(LOAD-GAIA) : $(INC)/addstar.h $(INC)/gaia.h $(LOAD-GAIA_DR2) : $(INC)/addstar.h $(INC)/gaia_dr2.h +$(LOAD-UKIRT_UHS) : $(INC)/addstar.h $(INC)/ukirt_uhs.h $(LOAD-STARPAR) : $(INC)/addstar.h $(INC)/loadstarpar.h $(LOAD-STARPAR-CLIENT) : $(INC)/addstar.h $(INC)/loadstarpar.h @@ -551,4 +575,5 @@ $(BIN)/loadgaia.$(ARCH) : $(LOAD-GAIA) $(BIN)/loadgaia_dr2.$(ARCH) : $(LOAD-GAIA_DR2) +$(BIN)/loadukirt_uhs.$(ARCH) : $(LOAD-UKIRT_UHS) $(BIN)/loadstarpar.$(ARCH) : $(LOAD-STARPAR) $(BIN)/loadstarpar_client.$(ARCH) : $(LOAD-STARPAR-CLIENT) Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h (revision 41170) @@ -0,0 +1,65 @@ + +typedef enum {UKIRT_MODE_NONE, UKIRT_MODE_UHS, UKIRT_MODE_UGCS, UKIRT_MODE_UGPS, UKIRT_MODE_ULAS} UkirtMode; + +# define NSTARS_MAX 10000000 +// # define NSTARS_MAX 10 + +# define BUFFER_SIZE 0x100000 +// # define BUFFER_SIZE 900 + +typedef struct { + Average average; + Measure *measure; + int flag; // in a subset? + int found; // assigned to an object? +} UKIRT_Stars; + +int HOST_ID; +char *HOSTDIR; +char *CPT_FILE; +char *INPUT; +int UKIRT_NFILTER; + +UkirtMode UKIRT_MODE; + +AddstarClientOptions args_loadukirt_uhs (int *argc, char **argv, AddstarClientOptions options); +// AddstarClientOptions args_loadukirt_uhs_client (int *argc, char **argv, AddstarClientOptions options); + +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options); + +UKIRT_Stars *loadukirt_uhs_make_subset (UKIRT_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset); + +// int loadukirt_uhs_save_remote (UKIRT_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region, char *fullname, AddstarClientOptions *options); +// int save_remote_host (HostInfo *host); + +// int init_remote_hosts (void); +// void free_remote_hosts (void); +// int find_empty_slot (void); +// int harvest_all (void); +// int harvest_host (void); + +int loadukirt_uhs_catalog (UKIRT_Stars *stars, int Nstars, SkyRegion *region, char *filename, AddstarClientOptions *options); + +// int galactic_to_celestial (double *R, double *D, double l, double b); + +int find_matches_ukirt_uhs (SkyRegion *region, UKIRT_Stars *stars, int Nstars, Catalog *catalog, AddstarClientOptions *options); + +int loadukirt_uhs_save_stars (char *filename, UKIRT_Stars *stars, int Nstars); +// UKIRT_Stars *loadukirt_uhs_load_stars (char *filename, int *nstars); + +UKIRT_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); + +UKIRT_Stars *loadukirt_uhs_readstars_ugps (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_ugcs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_ulas (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_uhs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); + + +int loadukirt_uhs_sortStars (UKIRT_Stars *stars, int Nstars); + +// int loadukirt_uhs_tmpdir (void); + +// roll these into a function with a structure to carry the Nlast, etc? +char *dparse_csv_rpt (double *X, int Nwant, int Nlast, char *line, int *status); +char *iparse_csv_rpt (int *X, int Nwant, int Nlast, char *line, int *status); +char *jparse_csv_rpt (uint64_t *X, int Nwant, int Nlast, char *line, int *status); Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c (revision 41170) @@ -118,5 +118,5 @@ if (photcode == 0) { if (!gfits_scan (header, "PHOTCODE", "%s", 1, photname)) { - fprintf (stderr, "photcode not supplied in header\n"); + fprintf (stderr, "failure: photcode not supplied in header\n"); return (FALSE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c (revision 41170) @@ -0,0 +1,165 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +static void help (void); + +AddstarClientOptions args_loadukirt_uhs (int *argc, char **argv, AddstarClientOptions options) { + + int N; + + /* check for help request */ + if (get_argument (*argc, argv, "-help") || + get_argument (*argc, argv, "-h")) { + help (); + } + + // UKIRT_MODE is global + UKIRT_MODE = UKIRT_MODE_NONE; + if ((N = get_argument (*argc, argv, "-uhs"))) { + UKIRT_MODE = UKIRT_MODE_UHS; + UKIRT_NFILTER = 1; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ugcs"))) { + UKIRT_MODE = UKIRT_MODE_UGCS; + UKIRT_NFILTER = 6; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ugps"))) { + UKIRT_MODE = UKIRT_MODE_UGPS; + UKIRT_NFILTER = 5; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ulas"))) { + UKIRT_MODE = UKIRT_MODE_ULAS; + UKIRT_NFILTER = 5; + remove_argument (N, argc, argv); + } + if (UKIRT_MODE == UKIRT_MODE_NONE) { + fprintf (stderr, "missing or invalid ukirt mode\n"); + help (); + } + + /* basic mode: image, list, refcat */ + options.mode = ADDSTAR_MODE_REFCAT; + + /* we do not allow a subset to be extracted -- all or nothing, babe */ + UserPatch.Rmin = 0; + UserPatch.Rmax= 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + + // XXX for the moment, make this selection manual. it needs to be automatic + // based on the state of the SkyTable + HOST_ID = 0; + PARALLEL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, argc, argv); + } + + /* only add to existing regions */ + options.existing_regions = FALSE; + if ((N = get_argument (*argc, argv, "-existing-regions"))) { + options.existing_regions = TRUE; + remove_argument (N, argc, argv); + } + /* only add to existing objects */ + options.only_match = FALSE; + if ((N = get_argument (*argc, argv, "-only-match"))) { + options.only_match = TRUE; + remove_argument (N, argc, argv); + } + /* replace measurement, don't duplicate (ref/cat only) */ + options.replace = FALSE; + if ((N = get_argument (*argc, argv, "-replace"))) { + options.replace = TRUE; + remove_argument (N, argc, argv); + } + /* override any header PHOTCODE values */ + options.photcode = 0; + + /* accept proper-motion (& parallax) data from reference */ + ACCEPT_MOTION = FALSE; + if ((N = get_argument (*argc, argv, "-accept-motion"))) { + ACCEPT_MOTION = TRUE; + remove_argument (N, argc, argv); + } + + /* provide a time for dataset */ + options.timeref = 0; + if ((N = get_argument (*argc, argv, "-time"))) { + time_t tmp; + remove_argument (N, argc, argv); + if (!ohana_str_to_time (argv[N], &tmp)) { + fprintf (stderr, "syntax error in time\n"); + exit (1); + } + options.timeref = tmp; + remove_argument (N, argc, argv); + } + + /* extra error messages */ + VERBOSE = FALSE; + if ((N = get_argument (*argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, argc, argv); + } + + /* other addstar options which cannot be used in loadukirt_uhs */ + // options.timeref = 0; + options.mosaic = FALSE; + options.skip_missed = FALSE; + options.closest = FALSE; + options.nosort = FALSE; + options.update = FALSE; + options.only_images = FALSE; + options.calibrate = FALSE; + options.quality_airmass = FALSE; + ACCEPT_ASTROM = FALSE; + FORCE_READ = FALSE; + TEXTMODE = FALSE; + SUBPIX = FALSE; + DUMP = NULL; + + if (*argc != 2) { + fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile)\n"); + exit (2); + } + return (options); +} + +static void help () { + + fprintf (stderr, "USAGE: loadukirt_uhs [options] (file) [-uhs | -ugcs | -ulas | -ugps] [..more files]\n"); + fprintf (stderr, " add data from UKIRT CSV file to DVO\n"); + fprintf (stderr, " mode is required : -uhs | -ugcs | -ulas | -ugps\n\n"); + + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -only-match : only add measurements to existing objects\n"); + fprintf (stderr, " -replace : replace time/photcode measurements (no duplication)\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c (revision 41170) @@ -0,0 +1,346 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +int find_matches_ukirt_uhs (SkyRegion *region, UKIRT_Stars *stars, int NstarsIn, Catalog *tgtcat, AddstarClientOptions *options) { + + int Nsecfilt = GetPhotcodeNsecfilt (); + int Nsec = GetPhotcodeNsec (options->photcode); + + /** allocate local arrays (stars) **/ + ALLOCATE_PTR (X1, double, NstarsIn); + ALLOCATE_PTR (Y1, double, NstarsIn); + ALLOCATE_PTR (N1, off_t, NstarsIn); + + /** allocate local arrays (tgtcat) **/ + off_t NAVE = tgtcat[0].Naverage; + off_t Nave = tgtcat[0].Naverage; + ALLOCATE_PTR (X2, double, NAVE); + ALLOCATE_PTR (Y2, double, NAVE); + ALLOCATE_PTR (N2, off_t, NAVE); + + /* for secfilt j and star i, secfilt[i*Nsecfilt+j] */ + + /* internal counters */ + off_t Nmatch = 0; + off_t NMEAS = tgtcat[0].Nmeasure; + off_t Nmeas = tgtcat[0].Nmeasure; + + off_t *next_meas = NULL; + + // current max obj ID for this tgtcat + unsigned int objID = tgtcat[0].objID; + unsigned int catID = tgtcat[0].catID; + + /* project onto rectilinear grid with 1 arcsec pixels. the choice of ARC projection has + * the advantage that every point in R,D has a mapping to a unique X,Y. However, note + * that not all possible X,Y points map back to R,D and the local plate scale changes + * substantially far from the projection pole. We use the center of the region (tgtcat) + * for crval1,2. + */ + + Coords tcoords; + InitCoords (&tcoords, "DEC--ARC"); + tcoords.crval1 = 0.5*(region[0].Rmin + region[0].Rmax); + if ((region[0].Dmax < 90) && (region[0].Dmin > -90)) { + tcoords.crval2 = 0.5*(region[0].Dmin + region[0].Dmax); + } else { + tcoords.crval2 = (region[0].Dmax >= 90) ? 90.0 : -90.0; + } + tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; + + /* build spatial index (RA sort) referencing input array sequence */ + off_t Nstars = 0; + for (off_t i = 0; i < NstarsIn; i++) { + int status = RD_to_XY (&X1[Nstars], &Y1[Nstars], stars[i].average.R, stars[i].average.D, &tcoords); + if (!status) continue; + N1[Nstars] = i; + Nstars ++; + } + if (Nstars < 1) { + if (VERBOSE) fprintf (stderr, "skipping %s, no overlapping stars\n", tgtcat[0].filename); + free (X1); + free (Y1); + free (N1); + free (X2); + free (Y2); + free (N2); + return 0; + } + if (Nstars > 1) sort_coords_index (X1, Y1, N1, Nstars); + + /* build spatial index (RA sort) */ + for (off_t i = 0; i < Nave; i++) { + RD_to_XY (&X2[i], &Y2[i], tgtcat[0].average[i].R, tgtcat[0].average[i].D, &tcoords); + N2[i] = i; + } + if (Nave > 1) sort_coords_index (X2, Y2, N2, Nave); + + /* set up pointers for linked list of measure */ + if (tgtcat[0].sorted && (tgtcat[0].Nmeasure == tgtcat[0].Nmeasure_disk)) { + // this version is only valid if we have done a full tgtcat load, and if the tgtcat + // is sorted while processed + next_meas = init_measure_links (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas); + } else { + next_meas = build_measure_links (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas); + } + + /* choose a radius for matches */ + double RADIUS = (options->radius == 0) ? 2.0 : options->radius; /* provided by config */ + double RADIUS2 = RADIUS*RADIUS; + + /****************** find matched stars ********************/ + + for (off_t i = 0, j = 0; (i < Nstars) && (j < Nave); ) { + if (!finite(X1[i]) || !finite(Y1[i])) { + i++; + continue; + } + if (!finite(X2[j]) || !finite(Y2[j])) { + j++; + continue; + } + + /* negative dX: j is too large */ + double dX = X1[i] - X2[j]; + if (dX <= -1.02*RADIUS) { + i++; + continue; + } + /* positive dX, i is too large */ + if (dX >= 1.02*RADIUS) { + j++; + continue; + } + + // skip this star if already assigned to an object + if (stars[N1[i]].found) { + i++; + continue; + } + + /* this block will match a given detection to the closest object within range of that detection. + XXX note that this matches ALL detections within range of the single object to that same object + this is bad, but I cannot just go in linear order (ie, mark off each object as they are + used). I should make a list of all Nobj * Ndet pairs in range and choose the matches + based on their separations. UGH + */ + + /* within match range; look for matches */ + off_t Jmin = -1; + double Rmin = RADIUS2; + for (off_t J = j; (dX > -1.02*RADIUS) && (J < Nave); J++) { + /* find closest match for this detection */ + dX = X1[i] - X2[J]; + double dY = Y1[i] - Y2[J]; + double dR = dX*dX + dY*dY; + if (dR > RADIUS2) continue; + if (dR < Rmin) { + Rmin = dR; + Jmin = J; + } + } + + /* no match, try next detection */ + if (Jmin == -1) { + i++; + continue; + } + + /*** a match is found, add to average, measure ***/ + Nmatch ++; + off_t n = N2[Jmin]; + off_t N = N1[i]; + + /* make sure there is space for next entry */ + if (Nmeas >= NMEAS - stars[N].average.Nmeasure) { + NMEAS = Nmeas + 1000 + stars[N].average.Nmeasure; + REALLOCATE (next_meas, off_t, NMEAS); + REALLOCATE (tgtcat[0].measure, Measure, NMEAS); + } + + /** add measurements for this star **/ + for (int mSrc = 0; mSrc < stars[N].average.Nmeasure; mSrc++) { + + /* add to end of measurement list */ + add_meas_link (&tgtcat[0].average[n], next_meas, Nmeas, NMEAS); + + /** add measurements for this star **/ + + // set the new measurements + tgtcat[0].measure[Nmeas] = stars[N].measure[mSrc]; + + // measure now carries R,D (not dR,dD) + tgtcat[0].measure[Nmeas].dbFlags = 0; + tgtcat[0].measure[Nmeas].averef = n; + tgtcat[0].measure[Nmeas].objID = tgtcat[0].average[n].objID; + tgtcat[0].measure[Nmeas].catID = tgtcat[0].catID; + + float dRoff = dvoOffsetR(&tgtcat[0].measure[Nmeas], &tgtcat[0].average[n]); + + // rationalize dR + if (dRoff > +180.0*3600.0) { + // average on high end of boundary, move star up + tgtcat[0].measure[Nmeas].R += 360.0; + dRoff -= 360.0*3600.0; + } + if (dRoff < -180.0*3600.0) { + // average on low end of boundary, move star down + tgtcat[0].measure[Nmeas].R -= 360.0; + dRoff += 360.0*3600.0; + } + if (fabs(dRoff) > 10*RADIUS) { + // take declination into account and check again. + double cosD = cos(RAD_DEG*tgtcat[0].average[n].D); + if (fabs(dRoff*cosD) > 10*RADIUS) { + fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", + tgtcat[0].average[n].R, tgtcat[0].average[n].D, + stars[N].average.R, stars[N].average.D, + X1[i], X2[Jmin], + Y1[i], Y2[Jmin]); + // XXX abort on this? -- this is a bad failure... + } + } + + /* Nm is updated, but not written out in -update mode (for existing entries) + Nm is recalculated in build_meas_links if loaded table is not sorted */ + tgtcat[0].average[n].Nmeasure ++; + Nmeas ++; + + /* if we choose to flag close encounters, see find_matches.c */ + /* if we choose to calculate RA,DEC averages, see update_coords.c */ + } + + if (ACCEPT_MOTION) { + tgtcat[0].average[n].uR = stars[N].average.uR; + tgtcat[0].average[n].uD = stars[N].average.uD; + tgtcat[0].average[n].duR = stars[N].average.duR; + tgtcat[0].average[n].duD = stars[N].average.duD; + tgtcat[0].average[n].P = stars[N].average.P; + tgtcat[0].average[n].dP = stars[N].average.dP; + tgtcat[0].average[n].Tmean = stars[N].average.Tmean; + } + + stars[N].found = TRUE; + i++; + } + + /*************** add unmatched stars *************************/ + + /** incorporate unmatched image stars, if this star is in field of this tgtcat **/ + /* these new entries are all written out in UPDATE mode */ + for (off_t i = 0; (i < Nstars) && !options->only_match; i++) { + /* make sure there is space for next entry */ + if (Nmeas >= NMEAS - stars[i].average.Nmeasure) { + NMEAS = Nmeas + 1000 + stars[i].average.Nmeasure; + REALLOCATE (next_meas, off_t, NMEAS); + REALLOCATE (tgtcat[0].measure, Measure, NMEAS); + } + if (Nave >= NAVE) { + NAVE = Nave + 1000; + REALLOCATE (tgtcat[0].average, Average, NAVE); + if (tgtcat[0].secfilt) { + // we only update the secfilt table if it has been allocated for output + REALLOCATE (tgtcat[0].secfilt, SecFilt, NAVE*tgtcat[0].Nsecfilt); + } + } + + if (stars[i].found) continue; + if (!IN_REGION (stars[i].average.R, stars[i].average.D)) continue; + + dvo_average_init (&tgtcat[0].average[Nave]); + tgtcat[0].average[Nave].R = stars[i].average.R; + tgtcat[0].average[Nave].D = stars[i].average.D; + tgtcat[0].average[Nave].dR = stars[i].average.dR; + tgtcat[0].average[Nave].dD = stars[i].average.dD; + + tgtcat[0].average[Nave].measureOffset = Nmeas; + tgtcat[0].average[Nave].objID = objID; + tgtcat[0].average[Nave].catID = catID; + + if (PSPS_ID) { + tgtcat[0].average[Nave].extID = CreatePSPSObjectID(tgtcat[0].average[Nave].R, tgtcat[0].average[Nave].D); + } + + if (ACCEPT_MOTION) { + tgtcat[0].average[Nave].uR = stars[i].average.uR; + tgtcat[0].average[Nave].uD = stars[i].average.uD; + tgtcat[0].average[Nave].duR = stars[i].average.duR; + tgtcat[0].average[Nave].duD = stars[i].average.duD; + tgtcat[0].average[Nave].P = stars[i].average.P; + tgtcat[0].average[Nave].dP = stars[i].average.dP; + tgtcat[0].average[Nave].Tmean = stars[i].average.Tmean; + } + + objID ++; + + // we only update the secfilt table if it has been allocated for output + for (int j = 0; tgtcat[0].secfilt && (j < Nsecfilt); j++) { + dvo_secfilt_init (&tgtcat[0].secfilt[Nave*Nsecfilt+j], SECFILT_RESET_ALL); + } + + // supply the measurements from this detection + for (int mSrc = 0; mSrc < stars[i].average.Nmeasure; mSrc++) { + dvo_measure_init (&tgtcat[0].measure[Nmeas]); + tgtcat[0].measure[Nmeas] = stars[i].measure[mSrc]; + + // the following measure elements cannot be set until here: + tgtcat[0].measure[Nmeas].dbFlags = 0; + tgtcat[0].measure[Nmeas].averef = Nave; + tgtcat[0].measure[Nmeas].objID = tgtcat[0].average[Nave].objID; + tgtcat[0].measure[Nmeas].catID = tgtcat[0].catID; + + /* set the average magnitude if not already set and the photcode.equiv is not 0 */ + /* in UPDATE mode, this value is not saved; use relphot to recalculate */ + if (Nsec > -1) { + tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + } + + tgtcat[0].average[Nave].Nmeasure ++; + Nmeas ++; + + // update the next_meas pointer for this entry (last one for this star is -1) + next_meas[Nmeas-1] = (mSrc < stars[i].average.Nmeasure - 1) ? Nmeas : -1; + } + stars[i].found = TRUE; + Nave ++; + } + + REALLOCATE (tgtcat[0].average, Average, Nave); + REALLOCATE (tgtcat[0].measure, Measure, Nmeas); + + if (options->nosort) { + tgtcat[0].sorted = FALSE; + } else { + tgtcat[0].sorted = TRUE; + tgtcat[0].measure = sort_measure (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas, next_meas); + } + + /* check if the tgtcat has changed? if no change, no need to write */ + tgtcat[0].objID = objID; // new max value, save on tgtcat close + tgtcat[0].Naverage = Nave; + tgtcat[0].Nmeasure = Nmeas; + tgtcat[0].Nsecfilt_mem = tgtcat[0].secfilt ? Nave*Nsecfilt : 0; + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); + + free (X1); + free (Y1); + free (N1); + free (X2); + free (Y2); + free (N2); + free (next_meas); + + return (Nmatch); +} + +/* + notes: + + for finding if a tgtcat star is in an image or an image star is in the tgtcat: + + tgtcats have boundaries defined by RA and DEC, but they may curve in projection + images have boundaries which are lines in pixels coords, but curve in RA and DEC + + tgtcat[0].found[Ncat] but stars[Nstar].found + +*/ Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c (revision 41170) @@ -0,0 +1,40 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* This is the DVO program to upload UKIRT_UHS detections from cvs files supplied by Mike + Read into a DVO database. It is modeled on the loadgaia_dr2 program with column modifications + determined by Zhoujian Zhang + + USAGE: loadukirt_uhs -D CATDIR (catdir) (ukirt_uhsfile) [...more files] +*/ + +int main (int argc, char **argv) { + + SkyTable *sky; + SkyList *skylist = NULL; + AddstarClientOptions options; + + // need to construct these options with args_loadtycho... + SetSignals (); + options = ConfigInit (&argc, argv); + options = args_loadukirt_uhs (&argc, argv, options); + + // load the full sky description table: + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // generate the subset matching the user-selected region + skylist = SkyListByPatch (sky, -1, &UserPatch); + + loadukirt_uhs_table (skylist, argv[1], &options); + + SkyTableFree (sky); + SkyListFree(skylist); + FreePhotcodeTable(); + free (CATDIR); + + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); + exit (0); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c (revision 41170) @@ -0,0 +1,32 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +int loadukirt_uhs_catalog (UKIRT_Stars *stars, int Nstars, SkyRegion *region, char *filename, AddstarClientOptions *options) { + + Catalog catalog; + + // now we have all of the loaded stars in this catalog + dvo_catalog_init (&catalog, TRUE); + catalog.filename = filename; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, region, VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename); + exit (2); + } + + find_matches_ukirt_uhs (region, stars, Nstars, &catalog, options); + + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + + dvo_catalog_free (&catalog); + + return (TRUE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c (revision 41170) @@ -0,0 +1,58 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +// assign stars in the given region to the subset (NOTE: stars are sorted by RA, start is +// first entry in stars array in this region) + +UKIRT_Stars *loadukirt_uhs_make_subset (UKIRT_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset) { + + int i; + + // collect array of (Stars *) stars in a new output catalog + int Nsubset = 0; + int NSUBSET = 3000; + + ALLOCATE_PTR (subset, UKIRT_Stars, NSUBSET); + // for (int i = 0; i < NSUBSET; i++) { + // ALLOCATE (subset[i].measure, Measure, UKIRT_NFILTER); + // } + + // find the rest of the stars in this output region + for (i = start; i < Nstars; i++) { + if (stars[i].flag) continue; + + // check if in skyregion + if (stars[i].average.R < region[0].Rmin) continue; + if (stars[i].average.R >= region[0].Rmax) break; + if (stars[i].average.D < region[0].Dmin) continue; + if (stars[i].average.D >= region[0].Dmax) continue; + + // check if in UserPatch (a GLOBAL) + if (stars[i].average.R < UserPatch.Rmin) continue; + if (stars[i].average.R > UserPatch.Rmax) break; + if (stars[i].average.D < UserPatch.Dmin) continue; + if (stars[i].average.D > UserPatch.Dmax) continue; + + // the *measure value is a copy of the version allocated to stars[i] + // do NOT free subset[i].measure + subset[Nsubset] = stars[i]; + Nsubset ++; + + stars[i].flag = TRUE; + + CHECK_REALLOCATE (subset, UKIRT_Stars, NSUBSET, Nsubset, 10000); + + /* + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + for (int j = Nstars; j < NSTARS; j++) { + ALLOCATE (stars[j].measure, Measure, NFILTER); + } + } + */ + } + + *nsubset = Nsubset; + return subset; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c (revision 41170) @@ -0,0 +1,46 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +UKIRT_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + switch (UKIRT_MODE) { + case UKIRT_MODE_UHS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_uhs (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_UGCS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ugcs (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_UGPS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ugps (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_ULAS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ulas (f, buffer, nstart, options, nstars); + return stars; + } + + default: + fprintf (stderr, "programming error: invalid mode\n"); + exit (1); + } + return NULL; +} + +int loadukirt_uhs_sortStars (UKIRT_Stars *stars, int Nstars) { + +# define SWAPFUNC(A,B){ UKIRT_Stars temp = stars[A]; stars[A] = stars[B]; stars[B] = temp; } +# define COMPARE(A,B)(stars[A].average.R < stars[B].average.R) + + OHANA_SORT (Nstars, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + + return TRUE; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c (revision 41170) @@ -0,0 +1,802 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + + 6 sigRa : + 7 sigDec : + 8 epoch : t ???? -- how is there a single epoch? + 9 muRa : + 10 muDec : + 11 sigMuRa : + 12 sigMuDec : + 13 chi2 : + 14 nFrames : + + 15 cx : + 16 cy : + 17 cz : + 18 htmID : + 19 l : + 20 b : + 21 lambda : + 22 eta : + 23 priOrSec : primary bit in dbFlags + 24 zmyPnt : + 25 zmyPntErr : + 26 ymjPnt : + 27 ymjPntErr : + 28 jmhPnt : + 29 jmhPntErr : + 30 hmk_1Pnt : + 31 hmk_1PntErr : + 32 zmyExt : + 33 zmyExtErr : + 34 ymjExt : + 35 ymjExtErr : + 36 jmhExt : + 37 jmhExtErr : + 38 hmk_1Ext : + 39 hmk_1ExtErr : + + 40 mergedClassStat : + 41 mergedClass : psfQFperf or photFlags2 + 42 pStar : psfChisq + 43 pGalaxy : extNsigma + 44 pNoise : + 45 pSaturated : + + 46 zHallMag : M + 47 zHallMagErr : dM + 48 zPetroMag : + 49 zPetroMagErr : + 50 zAperMag3 : Map + 51 zAperMag3Err : dMap + 52 zAperMag4 : Mkron + 53 zAperMag4Err : dMron + 54 zAperMag6 : + 55 zAperMag6Err : + 56 zGausig : + 57 zEll : + 58 zPA : posangle + 59 zErrBits : + 60 zDeblend : + 61 zClass : psfQF + 62 zClassStat : + 63 zppErrBits : photFlags + 64 zSeqNum : detID + 65 zObjID : + 66 zXi : + 67 zEta : + + 68 yHallMag : M + 69 yHallMagErr : dM + 70 yPetroMag : + 71 yPetroMagErr : + 72 yAperMag3 : Map + 73 yAperMag3Err : dMap + 74 yAperMag4 : Mkron + 75 yAperMag4Err : dMron + 76 yAperMag6 : + 77 yAperMag6Err : + 78 yGausig : + 79 yEll : + 80 yPA : posangle + 81 yErrBits : + 82 yDeblend : + 83 yClass : psfQF + 84 yClassStat : + 85 yppErrBits : photFlags + 86 ySeqNum : detID + 87 yObjID : + 88 yXi : + 89 yEta : + + 90 jHallMag : M + 91 jHallMagErr : dM + 92 jPetroMag : + 93 jPetroMagErr : + 94 jAperMag3 : Map + 95 jAperMag3Err : dMap + 96 jAperMag4 : Mkron + 97 jAperMag4Err : dMron + 98 jAperMag6 : + 99 jAperMag6Err : +100 jGausig : +101 jEll : +102 jPA : posangle +103 jErrBits : +104 jDeblend : +105 jClass : psfQF +106 jClassStat : +107 jppErrBits : photFlags +108 jSeqNum : detID +109 jObjID : +110 jXi : +111 jEta : + +112 hHallMag : M +113 hHallMagErr : dM +114 hPetroMag : +115 hPetroMagErr : +116 hAperMag3 : Map +117 hAperMag3Err : dMap +118 hAperMag4 : Mkron +119 hAperMag4Err : dMron +120 hAperMag6 : +121 hAperMag6Err : +122 hGausig : +123 hEll : +124 hPA : posangle +125 hErrBits : +126 hDeblend : +127 hClass : psfQF +128 hClassStat : +129 hppErrBits : photFlags +130 hSeqNum : detID +131 hObjID : +132 hXi : +133 hEta : + +134 k_1HallMag : M +135 k_1HallMagErr : dM +136 k_1PetroMag : +137 k_1PetroMagErr : +138 k_1AperMag3 : Map +139 k_1AperMag3Err : dMap +140 k_1AperMag4 : Mkron +141 k_1AperMag4Err : dMron +142 k_1AperMag6 : +143 k_1AperMag6Err : +144 k_1Gausig : +145 k_1Ell : +146 k_1PA : posangle +147 k_1ErrBits : +148 k_1Deblend : +149 k_1Class : psfQF +150 k_1ClassStat : +151 k_1ppErrBits : photFlags +152 k_1SeqNum : detID +153 k_1ObjID : +154 k_1Xi : +155 k_1Eta : + +156 k_2HallMag : M +157 k_2HallMagErr : dM +158 k_2PetroMag : +159 k_2PetroMagErr : +160 k_2AperMag3 : Map +161 k_2AperMag3Err : dMap +162 k_2AperMag4 : Mkron +163 k_2AperMag4Err : dMron +164 k_2AperMag6 : +165 k_2AperMag6Err : +166 k_2Gausig : +167 k_2Ell : +168 k_2PA : posangle +169 k_2ErrBits : +170 k_2Deblend : +171 k_2Class : psfQF +172 k_2ClassStat : +173 k_2ppErrBits : photFlags +174 k_2SeqNum : detID +175 k_2ObjID : +176 k_2Xi : +177 k_2Eta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ugcs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeZ = GetPhotcodeCodebyName ("UKIRT_Z"); if (!codeZ) Shutdown ("missing photcode UKIRT_Z"); + int codeY = GetPhotcodeCodebyName ("UKIRT_Y"); if (!codeY) Shutdown ("missing photcode UKIRT_Y"); + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (zHallMag, double, NELEM); + ALLOCATE_PTR (zHallMagErr, double, NELEM); + ALLOCATE_PTR (zAperMag3, double, NELEM); + ALLOCATE_PTR (zAperMag3Err, double, NELEM); + ALLOCATE_PTR (zAperMag4, double, NELEM); + ALLOCATE_PTR (zAperMag4Err, double, NELEM); + ALLOCATE_PTR (zPA, double, NELEM); + ALLOCATE_PTR (zClass, int, NELEM); + ALLOCATE_PTR (zppErrBits, int, NELEM); + ALLOCATE_PTR (zSeqNum, int, NELEM); + + ALLOCATE_PTR (yHallMag, double, NELEM); + ALLOCATE_PTR (yHallMagErr, double, NELEM); + ALLOCATE_PTR (yAperMag3, double, NELEM); + ALLOCATE_PTR (yAperMag3Err, double, NELEM); + ALLOCATE_PTR (yAperMag4, double, NELEM); + ALLOCATE_PTR (yAperMag4Err, double, NELEM); + ALLOCATE_PTR (yPA, double, NELEM); + ALLOCATE_PTR (yClass, int, NELEM); + ALLOCATE_PTR (yppErrBits, int, NELEM); + ALLOCATE_PTR (ySeqNum, int, NELEM); + + ALLOCATE_PTR (jHallMag, double, NELEM); + ALLOCATE_PTR (jHallMagErr, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + ALLOCATE_PTR (hHallMag, double, NELEM); + ALLOCATE_PTR (hHallMagErr, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (k1HallMag, double, NELEM); + ALLOCATE_PTR (k1HallMagErr, double, NELEM); + ALLOCATE_PTR (k1AperMag3, double, NELEM); + ALLOCATE_PTR (k1AperMag3Err, double, NELEM); + ALLOCATE_PTR (k1AperMag4, double, NELEM); + ALLOCATE_PTR (k1AperMag4Err, double, NELEM); + ALLOCATE_PTR (k1PA, double, NELEM); + ALLOCATE_PTR (k1Class, int, NELEM); + ALLOCATE_PTR (k1ppErrBits, int, NELEM); + ALLOCATE_PTR (k1SeqNum, int, NELEM); + + ALLOCATE_PTR (k2HallMag, double, NELEM); + ALLOCATE_PTR (k2HallMagErr, double, NELEM); + ALLOCATE_PTR (k2AperMag3, double, NELEM); + ALLOCATE_PTR (k2AperMag3Err, double, NELEM); + ALLOCATE_PTR (k2AperMag4, double, NELEM); + ALLOCATE_PTR (k2AperMag4Err, double, NELEM); + ALLOCATE_PTR (k2PA, double, NELEM); + ALLOCATE_PTR (k2Class, int, NELEM); + ALLOCATE_PTR (k2ppErrBits, int, NELEM); + ALLOCATE_PTR (k2SeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + dPARSE ( 8, 5, epoch, "epoch"); + jPARSE ( 23, 8, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 41, 23, mergedClass, "mergedClass"); + dPARSE ( 42, 41, pStar, "pStar"); + dPARSE ( 43, 42, pGalaxy, "pGalaxy"); + dPARSE ( 46, 43, zHallMag, "zHallMag"); + dPARSE ( 47, 46, zHallMagErr, "zHallMagErr"); + dPARSE ( 50, 47, zAperMag3, "zAperMag3"); + dPARSE ( 51, 50, zAperMag3Err, "zAperMag3Err"); + dPARSE ( 52, 51, zAperMag4, "zAperMag4"); + dPARSE ( 53, 52, zAperMag4Err, "zAperMag4Err"); + dPARSE ( 58, 53, zPA, "zPA"); + iPARSE ( 61, 58, zClass, "zClass"); + iPARSE ( 63, 61, zppErrBits, "zppErrBits"); + iPARSE ( 64, 63, zSeqNum, "zSeqNum"); + dPARSE ( 68, 64, yHallMag, "yHallMag"); + dPARSE ( 69, 68, yHallMagErr, "yHallMagErr"); + dPARSE ( 72, 69, yAperMag3, "yAperMag3"); + dPARSE ( 73, 72, yAperMag3Err, "yAperMag3Err"); + dPARSE ( 74, 73, yAperMag4, "yAperMag4"); + dPARSE ( 75, 74, yAperMag4Err, "yAperMag4Err"); + dPARSE ( 80, 75, yPA, "yPA"); + iPARSE ( 83, 80, yClass, "yClass"); + iPARSE ( 85, 83, yppErrBits, "yppErrBits"); + iPARSE ( 86, 85, ySeqNum, "ySeqNum"); + dPARSE ( 90, 86, jHallMag, "jHallMag"); + dPARSE ( 91, 90, jHallMagErr, "jHallMagErr"); + dPARSE ( 94, 91, jAperMag3, "jAperMag3"); + dPARSE ( 95, 94, jAperMag3Err, "jAperMag3Err"); + dPARSE ( 96, 95, jAperMag4, "jAperMag4"); + dPARSE ( 97, 96, jAperMag4Err, "jAperMag4Err"); + dPARSE (102, 97, jPA, "jPA"); + iPARSE (105, 102, jClass, "jClass"); + iPARSE (107, 105, jppErrBits, "jppErrBits"); + iPARSE (108, 107, jSeqNum, "jSeqNum"); + dPARSE (112, 108, hHallMag, "hHallMag"); + dPARSE (113, 112, hHallMagErr, "hHallMagErr"); + dPARSE (116, 113, hAperMag3, "hAperMag3"); + dPARSE (117, 116, hAperMag3Err, "hAperMag3Err"); + dPARSE (118, 117, hAperMag4, "hAperMag4"); + dPARSE (119, 118, hAperMag4Err, "hAperMag4Err"); + dPARSE (124, 119, hPA, "hPA"); + iPARSE (127, 124, hClass, "hClass"); + iPARSE (129, 127, hppErrBits, "hppErrBits"); + iPARSE (130, 129, hSeqNum, "hSeqNum"); + dPARSE (134, 130, k1HallMag, "k1HallMag"); + dPARSE (135, 134, k1HallMagErr, "k1HallMagErr"); + dPARSE (138, 135, k1AperMag3, "k1AperMag3"); + dPARSE (139, 138, k1AperMag3Err, "k1AperMag3Err"); + dPARSE (140, 139, k1AperMag4, "k1AperMag4"); + dPARSE (141, 140, k1AperMag4Err, "k1AperMag4Err"); + dPARSE (146, 141, k1PA, "k1PA"); + iPARSE (149, 146, k1Class, "k1Class"); + iPARSE (151, 149, k1ppErrBits, "k1ppErrBits"); + iPARSE (152, 151, k1SeqNum, "k1SeqNum"); + dPARSE (156, 152, k2HallMag, "k2HallMag"); + dPARSE (157, 156, k2HallMagErr, "k2HallMagErr"); + dPARSE (160, 157, k2AperMag3, "k2AperMag3"); + dPARSE (161, 160, k2AperMag3Err, "k2AperMag3Err"); + dPARSE (162, 161, k2AperMag4, "k2AperMag4"); + dPARSE (163, 162, k2AperMag4Err, "k2AperMag4Err"); + dPARSE (168, 163, k2PA, "k2PA"); + iPARSE (171, 168, k2Class, "k2Class"); + iPARSE (173, 171, k2ppErrBits, "k2ppErrBits"); + iPARSE (174, 173, k2SeqNum, "k2SeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (zHallMag, double, NELEM); + REALLOCATE (zHallMagErr, double, NELEM); + REALLOCATE (zAperMag3, double, NELEM); + REALLOCATE (zAperMag3Err, double, NELEM); + REALLOCATE (zAperMag4, double, NELEM); + REALLOCATE (zAperMag4Err, double, NELEM); + REALLOCATE (zPA, double, NELEM); + REALLOCATE (zClass, int, NELEM); + REALLOCATE (zppErrBits, int, NELEM); + REALLOCATE (zSeqNum, int, NELEM); + REALLOCATE (yHallMag, double, NELEM); + REALLOCATE (yHallMagErr, double, NELEM); + REALLOCATE (yAperMag3, double, NELEM); + REALLOCATE (yAperMag3Err, double, NELEM); + REALLOCATE (yAperMag4, double, NELEM); + REALLOCATE (yAperMag4Err, double, NELEM); + REALLOCATE (yPA, double, NELEM); + REALLOCATE (yClass, int, NELEM); + REALLOCATE (yppErrBits, int, NELEM); + REALLOCATE (ySeqNum, int, NELEM); + REALLOCATE (jHallMag, double, NELEM); + REALLOCATE (jHallMagErr, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + REALLOCATE (hHallMag, double, NELEM); + REALLOCATE (hHallMagErr, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (k1HallMag, double, NELEM); + REALLOCATE (k1HallMagErr, double, NELEM); + REALLOCATE (k1AperMag3, double, NELEM); + REALLOCATE (k1AperMag3Err, double, NELEM); + REALLOCATE (k1AperMag4, double, NELEM); + REALLOCATE (k1AperMag4Err, double, NELEM); + REALLOCATE (k1PA, double, NELEM); + REALLOCATE (k1Class, int, NELEM); + REALLOCATE (k1ppErrBits, int, NELEM); + REALLOCATE (k1SeqNum, int, NELEM); + REALLOCATE (k2HallMag, double, NELEM); + REALLOCATE (k2HallMagErr, double, NELEM); + REALLOCATE (k2AperMag3, double, NELEM); + REALLOCATE (k2AperMag3Err, double, NELEM); + REALLOCATE (k2AperMag4, double, NELEM); + REALLOCATE (k2AperMag4Err, double, NELEM); + REALLOCATE (k2PA, double, NELEM); + REALLOCATE (k2Class, int, NELEM); + REALLOCATE (k2ppErrBits, int, NELEM); + REALLOCATE (k2SeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // y measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = yHallMag[i]; + stars[Nstars].measure[0].dM = yHallMagErr[i]; + stars[Nstars].measure[0].Map = yAperMag3[i]; + stars[Nstars].measure[0].dMap = yAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = yAperMag4[i]; + stars[Nstars].measure[0].dMkron = yAperMag4Err[i]; + stars[Nstars].measure[0].posangle = yPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = yClass[i]; + stars[Nstars].measure[0].photFlags = yppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = ySeqNum[i]; + stars[Nstars].measure[0].photcode = codeY; + + // z measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = zHallMag[i]; + stars[Nstars].measure[1].dM = zHallMagErr[i]; + stars[Nstars].measure[1].Map = zAperMag3[i]; + stars[Nstars].measure[1].dMap = zAperMag3Err[i]; + stars[Nstars].measure[1].Mkron = zAperMag4[i]; + stars[Nstars].measure[1].dMkron = zAperMag4Err[i]; + stars[Nstars].measure[1].posangle = zPA[i]; // XXX units + stars[Nstars].measure[1].psfQF = zClass[i]; + stars[Nstars].measure[1].photFlags = zppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = zSeqNum[i]; + stars[Nstars].measure[1].photcode = codeZ; + + // J measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = jHallMag[i]; + stars[Nstars].measure[2].dM = jHallMagErr[i]; + stars[Nstars].measure[2].Map = jAperMag3[i]; + stars[Nstars].measure[2].dMap = jAperMag3Err[i]; + stars[Nstars].measure[2].Mkron = jAperMag4[i]; + stars[Nstars].measure[2].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[2].posangle = jPA[i]; // XXX units + stars[Nstars].measure[2].psfQF = jClass[i]; + stars[Nstars].measure[2].photFlags = jppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = jSeqNum[i]; + stars[Nstars].measure[2].photcode = codeJ; + + // H measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = hHallMag[i]; + stars[Nstars].measure[3].dM = hHallMagErr[i]; + stars[Nstars].measure[3].Map = hAperMag3[i]; + stars[Nstars].measure[3].dMap = hAperMag3Err[i]; + stars[Nstars].measure[3].Mkron = hAperMag4[i]; + stars[Nstars].measure[3].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[3].posangle = hPA[i]; // XXX units + stars[Nstars].measure[3].psfQF = hClass[i]; + stars[Nstars].measure[3].photFlags = hppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = hSeqNum[i]; + stars[Nstars].measure[3].photcode = codeH; + + // K1 measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = k1HallMag[i]; + stars[Nstars].measure[4].dM = k1HallMagErr[i]; + stars[Nstars].measure[4].Map = k1AperMag3[i]; + stars[Nstars].measure[4].dMap = k1AperMag3Err[i]; + stars[Nstars].measure[4].Mkron = k1AperMag4[i]; + stars[Nstars].measure[4].dMkron = k1AperMag4Err[i]; + stars[Nstars].measure[4].posangle = k1PA[i]; // XXX units + stars[Nstars].measure[4].psfQF = k1Class[i]; + stars[Nstars].measure[4].photFlags = k1ppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = k1SeqNum[i]; + stars[Nstars].measure[4].photcode = codeK; + + // K1 measure + stars[Nstars].measure[5].extID = sourceID[i]; + stars[Nstars].measure[5].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[5].R = ra[i]; + stars[Nstars].measure[5].D = dec[i]; + stars[Nstars].measure[5].psfChisq = pStar[i]; + stars[Nstars].measure[5].extNsigma = pGalaxy[i]; + stars[Nstars].measure[5].M = k2HallMag[i]; + stars[Nstars].measure[5].dM = k2HallMagErr[i]; + stars[Nstars].measure[5].Map = k2AperMag3[i]; + stars[Nstars].measure[5].dMap = k2AperMag3Err[i]; + stars[Nstars].measure[5].Mkron = k2AperMag4[i]; + stars[Nstars].measure[5].dMkron = k2AperMag4Err[i]; + stars[Nstars].measure[5].posangle = k2PA[i]; // XXX units + stars[Nstars].measure[5].psfQF = k2Class[i]; + stars[Nstars].measure[5].photFlags = k2ppErrBits[i]; + stars[Nstars].measure[5].photFlags2 = photFlags2; + stars[Nstars].measure[5].detID = k2SeqNum[i]; + stars[Nstars].measure[5].photcode = codeK; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (zHallMag); + FREE (zHallMagErr); + FREE (zAperMag3); + FREE (zAperMag3Err); + FREE (zAperMag4); + FREE (zAperMag4Err); + FREE (zPA); + FREE (zClass); + FREE (zppErrBits); + FREE (zSeqNum); + + FREE (yHallMag); + FREE (yHallMagErr); + FREE (yAperMag3); + FREE (yAperMag3Err); + FREE (yAperMag4); + FREE (yAperMag4Err); + FREE (yPA); + FREE (yClass); + FREE (yppErrBits); + FREE (ySeqNum); + + FREE (jHallMag); + FREE (jHallMagErr); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + FREE (hHallMag); + FREE (hHallMagErr); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (k1HallMag); + FREE (k1HallMagErr); + FREE (k1AperMag3); + FREE (k1AperMag3Err); + FREE (k1AperMag4); + FREE (k1AperMag4Err); + FREE (k1PA); + FREE (k1Class); + FREE (k1ppErrBits); + FREE (k1SeqNum); + + FREE (k2HallMag); + FREE (k2HallMagErr); + FREE (k2AperMag3); + FREE (k2AperMag3Err); + FREE (k2AperMag4); + FREE (k2AperMag4Err); + FREE (k2PA); + FREE (k2Class); + FREE (k2ppErrBits); + FREE (k2SeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c (revision 41170) @@ -0,0 +1,682 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + + 6 sigRa : + 7 sigDec : + 8 epoch : t ???? -- how is there a single epoch? + 9 muRa : + 10 muDec : + 11 sigMuRa : + 12 sigMuDec : + 13 chi2 : + 14 nFrames : + + 15 cx : + 16 cy : + 17 cz : + 18 htmID : + 19 l : + 20 b : + 21 lambda : + 22 eta : + 23 priOrSec : primary bit in dbFlags + + 24 jmhPnt : + 25 jmhPntErr : + 26 hmk1Pnt : + 27 hmk1PntErr : + 28 h2mk1Pnt : + 29 h2mk1PntErr : + + 30 mergedClassStat : + 31 mergedClass : psfQFperf or photFlags2 + 32 pStar : psfChisq + 33 pGalaxy : extNsigma + 34 pNoise : + 35 pSaturated : + + 36 jAperMag1 : M + 37 jAperMag1Err : dM + 38 jAperMag3 : Map + 39 jAperMag3Err : dMap + 40 jAperMag4 : Mkron + 41 jAperMag4Err : dMron + 42 jGausig : + 43 jEll : + 44 jPA : posangle + 45 jErrBits : + 46 jDeblend : + 47 jClass : psfQF + 48 jClassStat : + 49 jppErrBits : photFlags + 50 jSeqNum : detID + 51 jObjID : + 52 jXi : + 53 jEta : + 54 hAperMag1 : M + 55 hAperMag1Err : dM + 56 hAperMag3 : Map + 57 hAperMag3Err : dMap + 58 hAperMag4 : Mkron + 59 hAperMag4Err : dMron + 60 hGausig : + 61 hEll : + 62 hPA : posangle + 63 hErrBits : + 64 hDeblend : + 65 hClass : psfQF + 66 hClassStat : + 67 hppErrBits : photFlags + 68 hSeqNum : detID + 69 hObjID : + 70 hXi : + 71 hEta : + 72 k_1AperMag1 : M + 73 k_1AperMag1Err : dM + 74 k_1AperMag3 : Map + 75 k_1AperMag3Err : dMap + 76 k_1AperMag4 : Mkron + 77 k_1AperMag4Err : dMron + 78 k_1Gausig : + 79 k_1Ell : + 80 k_1PA : posangle + 81 k_1ErrBits : + 82 k_1Deblend : + 83 k_1Class : psfQF + 84 k_1ClassStat : + 85 k_1ppErrBits : photFlags + 86 k_1SeqNum : detID + 87 k_1ObjID : + 88 k_1Xi : + 89 k_1Eta : + 90 k_2AperMag1 : M + 91 k_2AperMag1Err : dM + 92 k_2AperMag3 : Map + 93 k_2AperMag3Err : dMap + 94 k_2AperMag4 : Mkron + 95 k_2AperMag4Err : dMron + 96 k_2Gausig : + 97 k_2Ell : + 98 k_2PA : posangle + 99 k_2ErrBits : +100 k_2Deblend : +101 k_2Class : psfQF +102 k_2ClassStat : +103 k_2ppErrBits : photFlags +104 k_2SeqNum : detID +105 k_2ObjID : +106 k_2Xi : +107 k_2Eta : +108 h_2AperMag1 : M +109 h_2AperMag1Err : dM +110 h_2AperMag3 : Map +111 h_2AperMag3Err : dMap +112 h_2AperMag4 : Mkron +113 h_2AperMag4Err : dMron +114 h_2Gausig : +115 h_2Ell : +116 h_2PA : posangle +117 h_2ErrBits : +118 h_2Deblend : +119 h_2Class : psfQF +120 h_2ClassStat : +121 h_2ppErrBits : photFlags +122 h_2SeqNum : detID +123 h_2ObjID : +124 h_2Xi : +125 h_2Eta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ugps (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (jAperMag1, double, NELEM); + ALLOCATE_PTR (jAperMag1Err, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + ALLOCATE_PTR (hAperMag1, double, NELEM); + ALLOCATE_PTR (hAperMag1Err, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (k1AperMag1, double, NELEM); + ALLOCATE_PTR (k1AperMag1Err, double, NELEM); + ALLOCATE_PTR (k1AperMag3, double, NELEM); + ALLOCATE_PTR (k1AperMag3Err, double, NELEM); + ALLOCATE_PTR (k1AperMag4, double, NELEM); + ALLOCATE_PTR (k1AperMag4Err, double, NELEM); + ALLOCATE_PTR (k1PA, double, NELEM); + ALLOCATE_PTR (k1Class, int, NELEM); + ALLOCATE_PTR (k1ppErrBits, int, NELEM); + ALLOCATE_PTR (k1SeqNum, int, NELEM); + + ALLOCATE_PTR (k2AperMag1, double, NELEM); + ALLOCATE_PTR (k2AperMag1Err, double, NELEM); + ALLOCATE_PTR (k2AperMag3, double, NELEM); + ALLOCATE_PTR (k2AperMag3Err, double, NELEM); + ALLOCATE_PTR (k2AperMag4, double, NELEM); + ALLOCATE_PTR (k2AperMag4Err, double, NELEM); + ALLOCATE_PTR (k2PA, double, NELEM); + ALLOCATE_PTR (k2Class, int, NELEM); + ALLOCATE_PTR (k2ppErrBits, int, NELEM); + ALLOCATE_PTR (k2SeqNum, int, NELEM); + + ALLOCATE_PTR (h2AperMag1, double, NELEM); + ALLOCATE_PTR (h2AperMag1Err, double, NELEM); + ALLOCATE_PTR (h2AperMag3, double, NELEM); + ALLOCATE_PTR (h2AperMag3Err, double, NELEM); + ALLOCATE_PTR (h2AperMag4, double, NELEM); + ALLOCATE_PTR (h2AperMag4Err, double, NELEM); + ALLOCATE_PTR (h2PA, double, NELEM); + ALLOCATE_PTR (h2Class, int, NELEM); + ALLOCATE_PTR (h2ppErrBits, int, NELEM); + ALLOCATE_PTR (h2SeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + dPARSE ( 8, 5, epoch, "epoch"); + jPARSE ( 23, 8, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 31, 23, mergedClass, "mergedClass"); + dPARSE ( 32, 31, pStar, "pStar"); + dPARSE ( 33, 32, pGalaxy, "pGalaxy"); + dPARSE ( 36, 33, jAperMag1, "jAperMag1"); + dPARSE ( 37, 36, jAperMag1Err, "jAperMag1Err"); + dPARSE ( 38, 37, jAperMag3, "jAperMag3"); + dPARSE ( 39, 38, jAperMag3Err, "jAperMag3Err"); + dPARSE ( 40, 39, jAperMag4, "jAperMag4"); + dPARSE ( 41, 40, jAperMag4Err, "jAperMag4Err"); + dPARSE ( 44, 41, jPA, "jPA"); + iPARSE ( 47, 44, jClass, "jClass"); + iPARSE ( 49, 47, jppErrBits, "jppErrBits"); + iPARSE ( 50, 49, jSeqNum, "jSeqNum"); + dPARSE ( 54, 50, hAperMag1, "hAperMag1"); + dPARSE ( 55, 54, hAperMag1Err, "hAperMag1Err"); + dPARSE ( 56, 55, hAperMag3, "hAperMag3"); + dPARSE ( 57, 56, hAperMag3Err, "hAperMag3Err"); + dPARSE ( 58, 57, hAperMag4, "hAperMag4"); + dPARSE ( 59, 58, hAperMag4Err, "hAperMag4Err"); + dPARSE ( 62, 59, hPA, "hPA"); + iPARSE ( 65, 62, hClass, "hClass"); + iPARSE ( 67, 65, hppErrBits, "hppErrBits"); + iPARSE ( 68, 67, hSeqNum, "hSeqNum"); + dPARSE ( 72, 68, k1AperMag1, "k1AperMag1"); + dPARSE ( 73, 72, k1AperMag1Err, "k1AperMag1Err"); + dPARSE ( 74, 73, k1AperMag3, "k1AperMag3"); + dPARSE ( 75, 74, k1AperMag3Err, "k1AperMag3Err"); + dPARSE ( 76, 75, k1AperMag4, "k1AperMag4"); + dPARSE ( 77, 76, k1AperMag4Err, "k1AperMag4Err"); + dPARSE ( 80, 77, k1PA, "k1PA"); + iPARSE ( 83, 80, k1Class, "k1Class"); + iPARSE ( 85, 83, k1ppErrBits, "k1ppErrBits"); + iPARSE ( 86, 85, k1SeqNum, "k1SeqNum"); + dPARSE ( 90, 86, k2AperMag1, "k2AperMag1"); + dPARSE ( 91, 90, k2AperMag1Err, "k2AperMag1Err"); + dPARSE ( 92, 91, k2AperMag3, "k2AperMag3"); + dPARSE ( 93, 92, k2AperMag3Err, "k2AperMag3Err"); + dPARSE ( 94, 93, k2AperMag4, "k2AperMag4"); + dPARSE ( 95, 94, k2AperMag4Err, "k2AperMag4Err"); + dPARSE ( 98, 95, k2PA, "k2PA"); + iPARSE (101, 98, k2Class, "k2Class"); + iPARSE (103, 101, k2ppErrBits, "k2ppErrBits"); + iPARSE (104, 103, k2SeqNum, "k2SeqNum"); + dPARSE (108, 104, h2AperMag1, "hAperMag1"); + dPARSE (109, 108, h2AperMag1Err, "hAperMag1Err"); + dPARSE (110, 109, h2AperMag3, "hAperMag3"); + dPARSE (111, 110, h2AperMag3Err, "hAperMag3Err"); + dPARSE (112, 111, h2AperMag4, "hAperMag4"); + dPARSE (113, 112, h2AperMag4Err, "hAperMag4Err"); + dPARSE (116, 113, h2PA, "hPA"); + iPARSE (119, 116, h2Class, "hClass"); + iPARSE (121, 119, h2ppErrBits, "hppErrBits"); + iPARSE (122, 121, h2SeqNum, "hSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (jAperMag1, double, NELEM); + REALLOCATE (jAperMag1Err, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + REALLOCATE (hAperMag1, double, NELEM); + REALLOCATE (hAperMag1Err, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (k1AperMag1, double, NELEM); + REALLOCATE (k1AperMag1Err, double, NELEM); + REALLOCATE (k1AperMag3, double, NELEM); + REALLOCATE (k1AperMag3Err, double, NELEM); + REALLOCATE (k1AperMag4, double, NELEM); + REALLOCATE (k1AperMag4Err, double, NELEM); + REALLOCATE (k1PA, double, NELEM); + REALLOCATE (k1Class, int, NELEM); + REALLOCATE (k1ppErrBits, int, NELEM); + REALLOCATE (k1SeqNum, int, NELEM); + REALLOCATE (k2AperMag1, double, NELEM); + REALLOCATE (k2AperMag1Err, double, NELEM); + REALLOCATE (k2AperMag3, double, NELEM); + REALLOCATE (k2AperMag3Err, double, NELEM); + REALLOCATE (k2AperMag4, double, NELEM); + REALLOCATE (k2AperMag4Err, double, NELEM); + REALLOCATE (k2PA, double, NELEM); + REALLOCATE (k2Class, int, NELEM); + REALLOCATE (k2ppErrBits, int, NELEM); + REALLOCATE (k2SeqNum, int, NELEM); + REALLOCATE (h2AperMag1, double, NELEM); + REALLOCATE (h2AperMag1Err, double, NELEM); + REALLOCATE (h2AperMag3, double, NELEM); + REALLOCATE (h2AperMag3Err, double, NELEM); + REALLOCATE (h2AperMag4, double, NELEM); + REALLOCATE (h2AperMag4Err, double, NELEM); + REALLOCATE (h2PA, double, NELEM); + REALLOCATE (h2Class, int, NELEM); + REALLOCATE (h2ppErrBits, int, NELEM); + REALLOCATE (h2SeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // J measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = jAperMag1[i]; + stars[Nstars].measure[0].dM = jAperMag1Err[i]; + stars[Nstars].measure[0].Map = jAperMag3[i]; + stars[Nstars].measure[0].dMap = jAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = jAperMag4[i]; + stars[Nstars].measure[0].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[0].posangle = jPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = jClass[i]; + stars[Nstars].measure[0].photFlags = jppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = jSeqNum[i]; + stars[Nstars].measure[0].photcode = codeJ; + + // H measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = hAperMag1[i]; + stars[Nstars].measure[1].dM = hAperMag1Err[i]; + stars[Nstars].measure[1].Map = hAperMag3[i]; + stars[Nstars].measure[1].dMap = hAperMag3Err[i]; + stars[Nstars].measure[1].Mkron = hAperMag4[i]; + stars[Nstars].measure[1].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[1].posangle = hPA[i]; // XXX units + stars[Nstars].measure[1].psfQF = hClass[i]; + stars[Nstars].measure[1].photFlags = hppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = hSeqNum[i]; + stars[Nstars].measure[1].photcode = codeH; + + // K1 measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = k1AperMag1[i]; + stars[Nstars].measure[2].dM = k1AperMag1Err[i]; + stars[Nstars].measure[2].Map = k1AperMag3[i]; + stars[Nstars].measure[2].dMap = k1AperMag3Err[i]; + stars[Nstars].measure[2].Mkron = k1AperMag4[i]; + stars[Nstars].measure[2].dMkron = k1AperMag4Err[i]; + stars[Nstars].measure[2].posangle = k1PA[i]; // XXX units + stars[Nstars].measure[2].psfQF = k1Class[i]; + stars[Nstars].measure[2].photFlags = k1ppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = k1SeqNum[i]; + stars[Nstars].measure[2].photcode = codeK; + + // K1 measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = k2AperMag1[i]; + stars[Nstars].measure[3].dM = k2AperMag1Err[i]; + stars[Nstars].measure[3].Map = k2AperMag3[i]; + stars[Nstars].measure[3].dMap = k2AperMag3Err[i]; + stars[Nstars].measure[3].Mkron = k2AperMag4[i]; + stars[Nstars].measure[3].dMkron = k2AperMag4Err[i]; + stars[Nstars].measure[3].posangle = k2PA[i]; // XXX units + stars[Nstars].measure[3].psfQF = k2Class[i]; + stars[Nstars].measure[3].photFlags = k2ppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = k2SeqNum[i]; + stars[Nstars].measure[3].photcode = codeK; + + // H2 measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = h2AperMag1[i]; + stars[Nstars].measure[4].dM = h2AperMag1Err[i]; + stars[Nstars].measure[4].Map = h2AperMag3[i]; + stars[Nstars].measure[4].dMap = h2AperMag3Err[i]; + stars[Nstars].measure[4].Mkron = h2AperMag4[i]; + stars[Nstars].measure[4].dMkron = h2AperMag4Err[i]; + stars[Nstars].measure[4].posangle = h2PA[i]; // XXX units + stars[Nstars].measure[4].psfQF = h2Class[i]; + stars[Nstars].measure[4].photFlags = h2ppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = h2SeqNum[i]; + stars[Nstars].measure[4].photcode = codeH; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (jAperMag1); + FREE (jAperMag1Err); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + FREE (hAperMag1); + FREE (hAperMag1Err); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (k1AperMag1); + FREE (k1AperMag1Err); + FREE (k1AperMag3); + FREE (k1AperMag3Err); + FREE (k1AperMag4); + FREE (k1AperMag4Err); + FREE (k1PA); + FREE (k1Class); + FREE (k1ppErrBits); + FREE (k1SeqNum); + + FREE (k2AperMag1); + FREE (k2AperMag1Err); + FREE (k2AperMag3); + FREE (k2AperMag3Err); + FREE (k2AperMag4); + FREE (k2AperMag4Err); + FREE (k2PA); + FREE (k2Class); + FREE (k2ppErrBits); + FREE (k2SeqNum); + + FREE (h2AperMag1); + FREE (h2AperMag1Err); + FREE (h2AperMag3); + FREE (h2AperMag3Err); + FREE (h2AperMag4); + FREE (h2AperMag4Err); + FREE (h2PA); + FREE (h2Class); + FREE (h2ppErrBits); + FREE (h2SeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c (revision 41170) @@ -0,0 +1,338 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UHS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + 6 cx : + 7 cy : + 8 cz : + 9 htmID : + 10 l : + 11 b : + 12 lambda : + 13 eta : + 14 priOrSec : primary bit in dbFlags + 15 mergedClassStat : + 16 mergedClass : psfQFperf or photFlags2 + 17 pStar : psfChisq + 18 pGalaxy : extNsigma + 19 pNoise : + 20 pSaturated : + 21 eBV : + 22 aJ : + 23 jHallMag : M + 24 jHallMagErr : dM + 25 jPetroMag : + 26 jPetroMagErr : + 27 jAperMag3 : Map + 28 jAperMag3Err : dMap + 29 jAperMag4 : Mkron + 30 jAperMag4Err : dMron + 31 jAperMag6 : + 32 jAperMag6Err : + 33 jGausig : + 34 jEll : + 35 jPA : posangle + 36 jErrBits : + 37 jDeblend : + 38 jClass : psfQF + 39 jClassStat : + 40 jppErrBits : photFlags + 41 jSeqNum : detID + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_uhs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_UHS_J"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + ALLOCATE_PTR (jHallMag, double, NELEM); + ALLOCATE_PTR (jHallMagErr, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + jPARSE (14, 5, priOrSec, "priOrSec"); + iPARSE (16, 14, mergedClass, "mergedClass"); + dPARSE (17, 16, pStar, "pStar"); + dPARSE (18, 17, pGalaxy, "pGalaxy"); + dPARSE (23, 18, jHallMag, "jHallMag"); + dPARSE (24, 23, jHallMagErr, "jHallMagErr"); + dPARSE (27, 24, jAperMag3, "jAperMag3"); + dPARSE (28, 27, jAperMag3Err, "jAperMag3Err"); + dPARSE (29, 28, jAperMag4, "jAperMag4"); + dPARSE (30, 29, jAperMag4Err, "jAperMag4Err"); + dPARSE (35, 30, jPA, "jPA"); + iPARSE (38, 35, jClass, "jClass"); + iPARSE (40, 38, jppErrBits, "jppErrBits"); + iPARSE (41, 40, jSeqNum, "jSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (jHallMag, double, NELEM); + REALLOCATE (jHallMagErr, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have one UKIRT UHS (J-band) measurement per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = jHallMag[i]; + stars[Nstars].measure[0].dM = jHallMagErr[i]; + stars[Nstars].measure[0].Map = jAperMag3[i]; + stars[Nstars].measure[0].dMap = jAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = jAperMag4[i]; + stars[Nstars].measure[0].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[0].posangle = jPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = jClass[i]; + stars[Nstars].measure[0].photFlags = jppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = jSeqNum[i]; + stars[Nstars].measure[0].photcode = codeJ; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + FREE (jHallMag); + FREE (jHallMagErr); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c (revision 41170) @@ -0,0 +1,822 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + + 1 sourceID : extID + 2 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 3 ra : R + 4 dec : D + 5 sigRa : + 6 sigDec : + 7 epoch : t ???? -- how is there a single epoch? + 8 muRa : + 9 muDec : + 10 sigMuRa : + 11 sigMuDec : + 12 chi2 : + 13 nFrames : + 14 cx : + 15 cy : + 16 cz : + 17 htmID : + 18 l : + 19 b : + 20 lambda : + 21 eta : + 22 priOrSec : primary bit in dbFlags + 23 ymj_1Pnt : + 24 ymj_1PntErr : + 25 j_1mhPnt : + 26 j_1mhPntErr : + 27 hmkPnt : + 28 hmkPntErr : + 29 ymj_1Ext : + 30 ymj_1ExtErr : + 31 j_1mhExt : + 32 j_1mhExtErr : + 33 hmkExt : + 34 hmkExtErr : + 35 mergedClassStat : + 36 mergedClass : psfQFperf or photFlags2 + 37 pStar : psfChisq + 38 pGalaxy : extNsigma + 39 pNoise : + 40 pSaturated : + 41 eBV : + 42 aY : + 43 aJ + 44 aH + 45 aK + 46 yHallMag : M + 47 yHallMagErr : dM + 48 yPetroMag : + 49 yPetroMagErr : + 50 yAperMag3 : Map + 51 yAperMag3Err : dMap + 52 yAperMag4 : Mkron + 53 yAperMag4Err : dMron + 54 yAperMag6 : + 55 yAperMag6Err : + 56 yGausig : + 57 yEll : + 58 yPA : posangle + 59 yErrBits : + 60 yDeblend : + 61 yClass : psfQF + 62 yClassStat : + 63 yppErrBits : photFlags + 64 ySeqNum : detID + 65 yObjID : + 66 yXi : + 67 yEta : + 68 j_1HallMag : M + 69 j_1HallMagErr : dM + 70 j_1PetroMag : + 71 j_1PetroMagErr : + 72 j_1AperMag3 : Map + 73 j_1AperMag3Err : dMap + 74 j_1AperMag4 : Mkron + 75 j_1AperMag4Err : dMron + 76 j_1AperMag6 : + 77 j_1AperMag6Err : + 78 j_1Gausig : + 79 j_1Ell : + 80 j_1PA : posangle + 81 j_1ErrBits : + 82 j_1Deblend : + 83 j_1Class : psfQF + 84 j_1ClassStat : + 85 j_1ppErrBits : photFlags + 86 j_1SeqNum : detID + 87 j_1ObjID : + 88 j_1Xi : + 89 j_1Eta : + 90 j_2HallMag : M + 91 j_2HallMagErr : dM + 92 j_2PetroMag : + 93 j_2PetroMagErr : + 94 j_2AperMag3 : Map + 95 j_2AperMag3Err : dMap + 96 j_2AperMag4 : Mkron + 97 j_2AperMag4Err : dMron + 98 j_2AperMag6 : + 99 j_2AperMag6Err : +100 j_2Gausig : +101 j_2Ell : +102 j_2PA : posangle +103 j_2ErrBits : +104 j_2Deblend : +105 j_2Class : psfQF +106 j_2ClassStat : +107 j_2ppErrBits : photFlags +108 j_2SeqNum : detID +109 j_2ObjID : +110 j_2Xi : +111 j_2Eta : +112 hHallMag : M +113 hHallMagErr : dM +114 hPetroMag : +115 hPetroMagErr : +116 hAperMag3 : Map +117 hAperMag3Err : dMap +118 hAperMag4 : Mkron +119 hAperMag4Err : dMron +120 hAperMag6 : +121 hAperMag6Err : +122 hGausig : +123 hEll : +124 hPA : posangle +125 hErrBits : +126 hDeblend : +127 hClass : psfQF +128 hClassStat : +129 hppErrBits : photFlags +130 hSeqNum : detID +131 hObjID : +132 hXi : +133 hEta : +134 kHallMag : M +135 kHallMagErr : dM +136 kPetroMag : +137 kPetroMagErr : +138 kAperMag3 : Map +139 kAperMag3Err : dMap +140 kAperMag4 : Mkron +141 kAperMag4Err : dMron +142 kAperMag6 : +143 kAperMag6Err : +144 kGausig : +145 kEll : +146 kPA : posangle +147 kErrBits : +148 kDeblend : +149 kClass : psfQF +150 kClassStat : +151 kppErrBits : photFlags +152 kSeqNum : detID +153 kObjID : +154 kXi : +155 kEta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ulas (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeY = GetPhotcodeCodebyName ("UKIRT_Y"); if (!codeY) Shutdown ("missing photcode UKIRT_Y"); + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (yHallMag, double, NELEM); + ALLOCATE_PTR (yHallMagErr, double, NELEM); + ALLOCATE_PTR (yAperMag3, double, NELEM); + ALLOCATE_PTR (yAperMag3Err, double, NELEM); + ALLOCATE_PTR (yAperMag4, double, NELEM); + ALLOCATE_PTR (yAperMag4Err, double, NELEM); + ALLOCATE_PTR (yPA, double, NELEM); + ALLOCATE_PTR (yClass, int, NELEM); + ALLOCATE_PTR (yppErrBits, int, NELEM); + ALLOCATE_PTR (ySeqNum, int, NELEM); + + ALLOCATE_PTR (j1HallMag, double, NELEM); + ALLOCATE_PTR (j1HallMagErr, double, NELEM); + ALLOCATE_PTR (j1AperMag3, double, NELEM); + ALLOCATE_PTR (j1AperMag3Err, double, NELEM); + ALLOCATE_PTR (j1AperMag4, double, NELEM); + ALLOCATE_PTR (j1AperMag4Err, double, NELEM); + ALLOCATE_PTR (j1PA, double, NELEM); + ALLOCATE_PTR (j1Class, int, NELEM); + ALLOCATE_PTR (j1ppErrBits, int, NELEM); + ALLOCATE_PTR (j1SeqNum, int, NELEM); + + ALLOCATE_PTR (j2HallMag, double, NELEM); + ALLOCATE_PTR (j2HallMagErr, double, NELEM); + ALLOCATE_PTR (j2AperMag3, double, NELEM); + ALLOCATE_PTR (j2AperMag3Err, double, NELEM); + ALLOCATE_PTR (j2AperMag4, double, NELEM); + ALLOCATE_PTR (j2AperMag4Err, double, NELEM); + ALLOCATE_PTR (j2PA, double, NELEM); + ALLOCATE_PTR (j2Class, int, NELEM); + ALLOCATE_PTR (j2ppErrBits, int, NELEM); + ALLOCATE_PTR (j2SeqNum, int, NELEM); + + ALLOCATE_PTR (hHallMag, double, NELEM); + ALLOCATE_PTR (hHallMagErr, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (kHallMag, double, NELEM); + ALLOCATE_PTR (kHallMagErr, double, NELEM); + ALLOCATE_PTR (kAperMag3, double, NELEM); + ALLOCATE_PTR (kAperMag3Err, double, NELEM); + ALLOCATE_PTR (kAperMag4, double, NELEM); + ALLOCATE_PTR (kAperMag4Err, double, NELEM); + ALLOCATE_PTR (kPA, double, NELEM); + ALLOCATE_PTR (kClass, int, NELEM); + ALLOCATE_PTR (kppErrBits, int, NELEM); + ALLOCATE_PTR (kSeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 2, 1, frameSetID, "frameSetID"); + dPARSE ( 3, 2, ra, "ra"); + dPARSE ( 4, 3, dec, "dec"); + dPARSE ( 7, 4, epoch, "epoch"); + jPARSE ( 22, 7, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 36, 22, mergedClass, "mergedClass"); + dPARSE ( 37, 36, pStar, "pStar"); + dPARSE ( 38, 37, pGalaxy, "pGalaxy"); + dPARSE ( 46, 38, yHallMag, "yHallMag"); + dPARSE ( 47, 46, yHallMagErr, "yHallMagErr"); + dPARSE ( 50, 47, yAperMag3, "yAperMag3"); + dPARSE ( 51, 50, yAperMag3Err, "yAperMag3Err"); + dPARSE ( 52, 51, yAperMag4, "yAperMag4"); + dPARSE ( 53, 52, yAperMag4Err, "yAperMag4Err"); + dPARSE ( 58, 53, yPA, "yPA"); + iPARSE ( 61, 58, yClass, "yClass"); + iPARSE ( 63, 61, yppErrBits, "yppErrBits"); + iPARSE ( 64, 63, ySeqNum, "ySeqNum"); + dPARSE ( 68, 64, j1HallMag, "j1HallMag"); + dPARSE ( 69, 68, j1HallMagErr, "j1HallMagErr"); + dPARSE ( 72, 69, j1AperMag3, "j1AperMag3"); + dPARSE ( 73, 72, j1AperMag3Err, "j1AperMag3Err"); + dPARSE ( 74, 73, j1AperMag4, "j1AperMag4"); + dPARSE ( 75, 74, j1AperMag4Err, "j1AperMag4Err"); + dPARSE ( 80, 75, j1PA, "j1PA"); + iPARSE ( 83, 80, j1Class, "j1Class"); + iPARSE ( 85, 83, j1ppErrBits, "j1ppErrBits"); + iPARSE ( 86, 85, j1SeqNum, "j1SeqNum"); + dPARSE ( 90, 86, j2HallMag, "j2HallMag"); + dPARSE ( 91, 90, j2HallMagErr, "j2HallMagErr"); + dPARSE ( 94, 91, j2AperMag3, "j2AperMag3"); + dPARSE ( 95, 94, j2AperMag3Err, "j2AperMag3Err"); + dPARSE ( 96, 95, j2AperMag4, "j2AperMag4"); + dPARSE ( 97, 96, j2AperMag4Err, "j2AperMag4Err"); + dPARSE (102, 97, j2PA, "j2PA"); + iPARSE (105, 102, j2Class, "j2Class"); + iPARSE (107, 105, j2ppErrBits, "j2ppErrBits"); + iPARSE (108, 107, j2SeqNum, "j2SeqNum"); + dPARSE (112, 108, hHallMag, "hHallMag"); + dPARSE (113, 112, hHallMagErr, "hHallMagErr"); + dPARSE (116, 113, hAperMag3, "hAperMag3"); + dPARSE (117, 116, hAperMag3Err, "hAperMag3Err"); + dPARSE (118, 117, hAperMag4, "hAperMag4"); + dPARSE (119, 118, hAperMag4Err, "hAperMag4Err"); + dPARSE (124, 119, hPA, "hPA"); + iPARSE (127, 124, hClass, "hClass"); + iPARSE (129, 127, hppErrBits, "hppErrBits"); + iPARSE (130, 129, hSeqNum, "hSeqNum"); + dPARSE (134, 130, kHallMag, "kHallMag"); + dPARSE (135, 134, kHallMagErr, "kHallMagErr"); + dPARSE (138, 135, kAperMag3, "kAperMag3"); + dPARSE (139, 138, kAperMag3Err, "kAperMag3Err"); + dPARSE (140, 139, kAperMag4, "kAperMag4"); + dPARSE (141, 140, kAperMag4Err, "kAperMag4Err"); + dPARSE (146, 141, kPA, "kPA"); + iPARSE (149, 146, kClass, "kClass"); + iPARSE (151, 149, kppErrBits, "kppErrBits"); + iPARSE (152, 151, kSeqNum, "kSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (yHallMag, double, NELEM); + REALLOCATE (yHallMagErr, double, NELEM); + REALLOCATE (yAperMag3, double, NELEM); + REALLOCATE (yAperMag3Err, double, NELEM); + REALLOCATE (yAperMag4, double, NELEM); + REALLOCATE (yAperMag4Err, double, NELEM); + REALLOCATE (yPA, double, NELEM); + REALLOCATE (yClass, int, NELEM); + REALLOCATE (yppErrBits, int, NELEM); + REALLOCATE (ySeqNum, int, NELEM); + REALLOCATE (j1HallMag, double, NELEM); + REALLOCATE (j1HallMagErr, double, NELEM); + REALLOCATE (j1AperMag3, double, NELEM); + REALLOCATE (j1AperMag3Err, double, NELEM); + REALLOCATE (j1AperMag4, double, NELEM); + REALLOCATE (j1AperMag4Err, double, NELEM); + REALLOCATE (j1PA, double, NELEM); + REALLOCATE (j1Class, int, NELEM); + REALLOCATE (j1ppErrBits, int, NELEM); + REALLOCATE (j1SeqNum, int, NELEM); + REALLOCATE (j2HallMag, double, NELEM); + REALLOCATE (j2HallMagErr, double, NELEM); + REALLOCATE (j2AperMag3, double, NELEM); + REALLOCATE (j2AperMag3Err, double, NELEM); + REALLOCATE (j2AperMag4, double, NELEM); + REALLOCATE (j2AperMag4Err, double, NELEM); + REALLOCATE (j2PA, double, NELEM); + REALLOCATE (j2Class, int, NELEM); + REALLOCATE (j2ppErrBits, int, NELEM); + REALLOCATE (j2SeqNum, int, NELEM); + REALLOCATE (hHallMag, double, NELEM); + REALLOCATE (hHallMagErr, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (kHallMag, double, NELEM); + REALLOCATE (kHallMagErr, double, NELEM); + REALLOCATE (kAperMag3, double, NELEM); + REALLOCATE (kAperMag3Err, double, NELEM); + REALLOCATE (kAperMag4, double, NELEM); + REALLOCATE (kAperMag4Err, double, NELEM); + REALLOCATE (kPA, double, NELEM); + REALLOCATE (kClass, int, NELEM); + REALLOCATE (kppErrBits, int, NELEM); + REALLOCATE (kSeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // y measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = yHallMag[i]; + stars[Nstars].measure[0].dM = yHallMagErr[i]; + stars[Nstars].measure[0].Map = yAperMag3[i]; + stars[Nstars].measure[0].dMap = yAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = yAperMag4[i]; + stars[Nstars].measure[0].dMkron = yAperMag4Err[i]; + stars[Nstars].measure[0].posangle = yPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = yClass[i]; + stars[Nstars].measure[0].photFlags = yppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = ySeqNum[i]; + stars[Nstars].measure[0].photcode = codeY; + + // J_1 measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = j1HallMag[i]; + stars[Nstars].measure[1].dM = j1HallMagErr[i]; + stars[Nstars].measure[1].Map = j1AperMag3[i]; + stars[Nstars].measure[1].dMap = j1AperMag3Err[i]; + stars[Nstars].measure[1].Mkron = j1AperMag4[i]; + stars[Nstars].measure[1].dMkron = j1AperMag4Err[i]; + stars[Nstars].measure[1].posangle = j1PA[i]; // XXX units + stars[Nstars].measure[1].psfQF = j1Class[i]; + stars[Nstars].measure[1].photFlags = j1ppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = j1SeqNum[i]; + stars[Nstars].measure[1].photcode = codeJ; + + // J measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = j2HallMag[i]; + stars[Nstars].measure[2].dM = j2HallMagErr[i]; + stars[Nstars].measure[2].Map = j2AperMag3[i]; + stars[Nstars].measure[2].dMap = j2AperMag3Err[i]; + stars[Nstars].measure[2].Mkron = j2AperMag4[i]; + stars[Nstars].measure[2].dMkron = j2AperMag4Err[i]; + stars[Nstars].measure[2].posangle = j2PA[i]; // XXX units + stars[Nstars].measure[2].psfQF = j2Class[i]; + stars[Nstars].measure[2].photFlags = j2ppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = j2SeqNum[i]; + stars[Nstars].measure[2].photcode = codeJ; + + // H measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = hHallMag[i]; + stars[Nstars].measure[3].dM = hHallMagErr[i]; + stars[Nstars].measure[3].Map = hAperMag3[i]; + stars[Nstars].measure[3].dMap = hAperMag3Err[i]; + stars[Nstars].measure[3].Mkron = hAperMag4[i]; + stars[Nstars].measure[3].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[3].posangle = hPA[i]; // XXX units + stars[Nstars].measure[3].psfQF = hClass[i]; + stars[Nstars].measure[3].photFlags = hppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = hSeqNum[i]; + stars[Nstars].measure[3].photcode = codeH; + + // K measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = kHallMag[i]; + stars[Nstars].measure[4].dM = kHallMagErr[i]; + stars[Nstars].measure[4].Map = kAperMag3[i]; + stars[Nstars].measure[4].dMap = kAperMag3Err[i]; + stars[Nstars].measure[4].Mkron = kAperMag4[i]; + stars[Nstars].measure[4].dMkron = kAperMag4Err[i]; + stars[Nstars].measure[4].posangle = kPA[i]; // XXX units + stars[Nstars].measure[4].psfQF = kClass[i]; + stars[Nstars].measure[4].photFlags = kppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = kSeqNum[i]; + stars[Nstars].measure[4].photcode = codeK; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (yHallMag); + FREE (yHallMagErr); + FREE (yAperMag3); + FREE (yAperMag3Err); + FREE (yAperMag4); + FREE (yAperMag4Err); + FREE (yPA); + FREE (yClass); + FREE (yppErrBits); + FREE (ySeqNum); + + FREE (j1HallMag); + FREE (j1HallMagErr); + FREE (j1AperMag3); + FREE (j1AperMag3Err); + FREE (j1AperMag4); + FREE (j1AperMag4Err); + FREE (j1PA); + FREE (j1Class); + FREE (j1ppErrBits); + FREE (j1SeqNum); + + FREE (j2HallMag); + FREE (j2HallMagErr); + FREE (j2AperMag3); + FREE (j2AperMag3Err); + FREE (j2AperMag4); + FREE (j2AperMag4Err); + FREE (j2PA); + FREE (j2Class); + FREE (j2ppErrBits); + FREE (j2SeqNum); + + FREE (hHallMag); + FREE (hHallMagErr); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (kHallMag); + FREE (kHallMagErr); + FREE (kAperMag3); + FREE (kAperMag3Err); + FREE (kAperMag4); + FREE (kAperMag4Err); + FREE (kPA); + FREE (kClass); + FREE (kppErrBits); + FREE (kSeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c (revision 41170) @@ -0,0 +1,90 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +// one pass reads a single file at a time +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options) { + + // open the input file here and pass the pointer below + FILE *f = fopen (filename, "r"); + if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename); + + // we allocate one extra byte into which we never read so there will always be a NULL terminating the string + ALLOCATE_PTR (buffer, char, BUFFER_SIZE + 1); + bzero (buffer, BUFFER_SIZE + 1); + + // starting point of valid data in buffer; this is updated on each pass to readstars + int Nstart = 0; + + while (1) { + + // read the chunk from the file + fprintf (stderr, "loading %s\n", filename); + + // on each pass, we read a new chunk of data and receive Nstars + + int Nstars = 0; + UKIRT_Stars *stars = loadukirt_uhs_readstars (f, buffer, &Nstart, options, &Nstars); + if (!Nstars) { + fclose (f); + free (buffer); + free (stars); + return TRUE; // end of file reached + } + + fprintf (stderr, "writing %d stars to dvo\n", Nstars); + + // sort the stars by RA (Nmeasure agnostic) + loadukirt_uhs_sortStars (stars, Nstars); + + // scan through the stars, loading the containing catalogs + // skip through table for unsaved stars + for (int i = 0; i < Nstars; i++) { + if (stars[i].flag) continue; + + // scan forward until we read the UserPatch + if (stars[i].average.R < UserPatch.Rmin) continue; + if (stars[i].average.R > UserPatch.Rmax) break; + if (stars[i].average.D < UserPatch.Dmin) continue; + if (stars[i].average.D > UserPatch.Dmax) continue; + + // identify the relevant catalog + SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D); + if (skylist[0].Nregions == 0) { + SkyListFree (skylist); + continue; + } + SkyRegion *region = skylist[0].regions[0]; + + // select stars matching this region (Nmeasure agnostic) + int Nsubset; + UKIRT_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset); + + // In parallel mode, write out the subset to a disk file. Block until a remote host + // is available. In serial mode, just match against the appropriate region and save + // NOTE: disable parallel mode for now: + // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options); + // loadukirt_uhs_catalog (Nmeasure agnostic) + loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options); + free (subset); + SkyListFree (skylist); + } + for (int i = 0; i < Nstars; i++) { + free (stars[i].measure); + } + free (stars); + } + + // wait for last remote clients to finish + // NOTE: disable parallel mode for now: + // harvest_all (); + + // we should not actually reach this point + return FALSE; +} + +/* modifying to read in smaller blocks of the input file at a time + + each pass on readstars needs to ... + +*/ + Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h (revision 41170) @@ -31,5 +31,8 @@ int IMAGES_ONLY; int ACCEPT_MOTION; + +int ACCEPT_ASTROM; int RETAIN_AVE_PHOTOMETRY; + char CATDIR[DVO_MAX_PATH]; char GSCFILE[DVO_MAX_PATH]; Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c (revision 41170) @@ -112,9 +112,15 @@ } - /* limit the impact of a dvomerge -parallel */ + /* accept input database average astrometry motions */ ACCEPT_MOTION = FALSE; if ((N = get_argument (*argc, argv, "-accept-motion"))) { remove_argument (N, argc, argv); ACCEPT_MOTION = TRUE; + } + /* accept input database average astrometry information */ + ACCEPT_ASTROM = FALSE; + if ((N = get_argument (*argc, argv, "-accept-astrom"))) { + remove_argument (N, argc, argv); + ACCEPT_ASTROM = TRUE; } @@ -337,9 +343,15 @@ } - /* limit the impact of a dvomerge -parallel */ + /* accept input database average astrometry motions */ ACCEPT_MOTION = FALSE; if ((N = get_argument (*argc, argv, "-accept-motion"))) { remove_argument (N, argc, argv); ACCEPT_MOTION = TRUE; + } + /* accept input database average astrometry information */ + ACCEPT_ASTROM = FALSE; + if ((N = get_argument (*argc, argv, "-accept-astrom"))) { + remove_argument (N, argc, argv); + ACCEPT_ASTROM = TRUE; } Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 41170) @@ -328,4 +328,5 @@ if (FORCE_MERGE) { strextend (&command, "-force-merge"); } if (ACCEPT_MOTION) { strextend (&command, "-accept-motion"); } + if (ACCEPT_ASTROM) { strextend (&command, "-accept-astrom"); } if (RETAIN_AVE_PHOTOMETRY) { strextend (&command, "-retain-ave-photometry"); } if (MATCHED_TABLES) { strextend (&command, "-matched-tables"); } Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 41170) @@ -363,4 +363,5 @@ } + // XXX: add choice of secfilt // update the average properties to reflect the incoming entries: // if RETAIN_AVE_PHOTOMETRY is true and the original value is NAN, but the input value is not, accept the input: @@ -389,5 +390,5 @@ // we can choose to accept the proper-motion and parallax from the reference tgtcat - if (ACCEPT_MOTION) { + if (ACCEPT_MOTION || ACCEPT_ASTROM) { output[0].average[n].dR = input[0].average[N].dR; output[0].average[n].dD = input[0].average[N].dD; @@ -400,4 +401,8 @@ output[0].average[n].Tmean = input[0].average[N].Tmean; } + if (ACCEPT_ASTROM) { + output[0].average[n].R = input[0].average[N].R; + output[0].average[n].D = input[0].average[N].D; + } /* Nm is updated, but not written out in -update mode (for existing entries) @@ -461,5 +466,5 @@ // we can choose to accept the proper-motion and parallax from the reference tgtcat - if (ACCEPT_MOTION) { + if (ACCEPT_MOTION || ACCEPT_ASTROM) { output[0].average[Nave].dR = input[0].average[N].dR; output[0].average[Nave].dD = input[0].average[N].dD; @@ -471,4 +476,8 @@ output[0].average[Nave].dP = input[0].average[N].dP; output[0].average[Nave].Tmean = input[0].average[N].Tmean; + } + if (ACCEPT_ASTROM) { + output[0].average[Nave].R = input[0].average[N].R; + output[0].average[Nave].D = input[0].average[N].D; } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile (revision 41170) @@ -42,5 +42,5 @@ $(SRC)/bDrawObjects.$(ARCH).o $(SRC)/bDrawFrame.$(ARCH).o \ $(SRC)/bDrawLabels.$(ARCH).o $(SRC)/bDrawIt.$(ARCH).o \ -$(SRC)/bDrawImage.$(ARCH).o \ +$(SRC)/bDrawImage.$(ARCH).o $(SRC)/MemoryDump.$(ARCH).o \ $(SRC)/PNGit.$(ARCH).o $(SRC)/PPMit.$(ARCH).o \ $(SRC)/PSit.$(ARCH).o $(SRC)/CrossHairs.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt (revision 41170) @@ -0,0 +1,38 @@ + +This document outlines the way kapa works. + +Top-Level (kapa.c): + + * load arguments + * generate basic graphic structure (SetUpGraphic) + * this is held static by SetUpGraphic.c + * it currently does not carry all elements: sections are static elsewhere + * Graphic elements are a bit X-centric. + * generate basic window elements (InitLayout / Layout.c) + * run event loop + +Event Loop: + + * check pipe for messages + * in X context, check for various events (cursor interations, resizing, moving, raising) + +In X context, events which expose or reconfigure the window trigger Refresh(). + +In non-X context: + * PNGit & JPEGit trigger bDrawIt (equivalent to Refresh) + * PSit is equivalent to bDrawIt + +------ + +I would like to change from native X to native bDraw functions. This +would allow for opacity and anti-aliasing. To do this, I need to do +the following: + +* bDraw operations need to include a mask so we can distinguish layers + (e.g., text on image). anti-aliasing needs to be applied to text + and graph elements, but not to the images + +* Refresh currently calls DrawImage with generates both the main image + as well as the extra elements: colorbar, zoom window, pan window, + buttons. + Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h (revision 41170) @@ -3,7 +3,13 @@ int args PROTO((int *argc, char **argv)); void SetUpGraphic PROTO((int *argc, char **argv)); +void FreeGraphic PROTO((void)); void DefineLayout PROTO((int, char **)); int EventLoop PROTO((void)); void CloseDisplay PROTO((void)); +int MemoryDump PROTO((int sock)); +int MemoryDumpLines PROTO((int sock)); +int MemoryDumpOnExit PROTO((int sock)); +int MemoryDumpSetOnExit PROTO((int state)); +int MemoryDumpAndExit PROTO((void)); /* SetUpGraphic */ @@ -33,4 +39,6 @@ void DrawTextlines PROTO((KapaGraphWidget *graph)); void DrawConnect PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); +void DrawPolygon PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); +void DrawPolyfill PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); void DrawHistogram PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); int DrawObjectN PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); @@ -85,4 +93,5 @@ int SetColormapFromPipe PROTO((int sock)); int SetNanColorFromPipe PROTO((int sock)); +int SetSmoothSigma PROTO((int sock)); int LoadVectorData PROTO((int sock, KapaGraphWidget *graph, int N, char *type)); @@ -109,4 +118,5 @@ void InitLayout PROTO((int argc, char **argv)); +void FreeLayout PROTO((void)); /* PS drawing utilities */ @@ -142,4 +152,6 @@ void bDrawBars PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object, int mode)); void bDrawPoints PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); +void bDrawPolygons PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); +void bDrawFillPolygons PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawXErrors PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawYErrors PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); @@ -179,4 +191,5 @@ void InitButtonFunc PROTO((Button *button, int (*function)(Graphic *graphic, KapaImageWidget *image))); void DrawImage PROTO((KapaImageWidget *image)); +void DrawImageTool PROTO((KapaImageWidget *image)); void DrawButton PROTO((Graphic *graphic, Button *button)); void DrawBitmap PROTO((Graphic *graphic, int x, int y, int dx, int dy, unsigned char *bitmap, int mode)); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h (revision 41170) @@ -48,4 +48,6 @@ unsigned long back; // basic background color + float smooth_sigma; // anti-aliasing smoothing scale + unsigned long overlay_color[NOVERLAYS]; // image plotting colors } Graphic; @@ -131,9 +133,23 @@ } Overlay; -typedef struct { - float *x, *y, *z, *dxp, *dxm, *dyp, *dym; - int Npts; - int style, ptype, ltype, color, etype, ebar; - double lweight, size; +// a set of objects all have the same basic properties +typedef struct { + float *x; // x-coordinates of the points + float *y; // y-coordinates of the points + float *z; // size/colorscale for points + float *dxp; // lower-errorbar in x + float *dxm; // upper-errorbar in x + float *dyp; // lower-errorbar in y + float *dym; // upper-errorbar in y + int Npts; // number of points in this set + int style; // how are the object draw: CONNECT, HISTOGRAM, + int ptype; // shape of object at each point + // ptype is overloaded for NPOLYGON to be the number of points / polygon + int ltype; // style of line (solid, dot, dash, etc) + int color; // color for point (if not colorscaled) + int etype; // errorbars to draw (0x01 = y, 0x02 = x) + int ebar; // draw a cap on the error bar + double lweight; // line thickness + double size; // size of the object double x0, x1, y0, y1; /* limits for this object */ double alpha; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c (revision 41170) @@ -352,4 +352,28 @@ } + if (!strcmp (word, "SIGM")) { + SetSmoothSigma (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEMD")) { + MemoryDump (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEML")) { + MemoryDumpLines (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEMX")) { + MemoryDumpOnExit (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + fprintf (stderr, "unknown signal %s\n", word); KiiSendCommand (sock, 4, "DONE"); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c (revision 41170) @@ -97,26 +97,26 @@ switch (object[0].style) { case KAPA_PLOT_CONNECT: - // fprintf (stderr, "plot KAPA_PLOT_CONNECT: \n"); DrawConnect (graphic, graph, object); break; + case KAPA_PLOT_POLYGON: + DrawPolygon (graphic, graph, object); + break; + case KAPA_PLOT_POLYFILL: + DrawPolyfill (graphic, graph, object); + break; case KAPA_PLOT_HISTOGRAM: - // fprintf (stderr, "plot KAPA_PLOT_HISTOGRAM:\n"); DrawHistogram (graphic, graph, object); break; case KAPA_PLOT_BARS_SOLID: - // fprintf (stderr, "plot KAPA_PLOT_BARS_SOLID:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_SOLID); break; case KAPA_PLOT_BARS_OUTLINE: - // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTLINE:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTLINE); break; case KAPA_PLOT_BARS_OUTFILL: - // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTFILL:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTFILL); break; case KAPA_PLOT_POINTS: default: - // fprintf (stderr, "plot KAPA_PLOT_POINTS:\n"); DrawPoints (graphic, graph, object); break; @@ -172,5 +172,137 @@ sx0 = sx1; sy0 = sy1; } - +} + +/******/ +void DrawPolygon (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0, sx1, sy1; + double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx + XCENTER; + sy0 = x[k]*myi + y[k]*myj + by + YCENTER; + + // last point connects to first + if (j == object[0].ptype - 1) { + sx1 = x[i]*mxi + y[i]*mxj + bx + XCENTER; + sy1 = x[i]*myi + y[i]*myj + by + YCENTER; + } else { + sx1 = x[k+1]*mxi + y[k+1]*mxj + bx + XCENTER; + sy1 = x[k+1]*myi + y[k+1]*myj + by + YCENTER; + } + ClipLine (graphic, sx0, sy0, sx1, sy1, X0, Y0, X1, Y1); + } + } +} + +/******/ +void DrawPolyfill (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y, *z; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0; + // double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + /* + window boundary so we can clip objects + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + */ + + x = object[0].x; y = object[0].y; z = object[0].z; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + ALLOCATE_PTR (points, XPoint, object[0].ptype); + + // NOTE that LoadObject.c:45 limits the allow + // object styles which may have negative (scaled) colors + int scaleColor = (object[0].color < 0); + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx + XCENTER; + sy0 = x[k]*myi + y[k]*myj + by + YCENTER; + + points[j].x = sx0; + points[j].y = sy0; + } + + if (scaleColor) { + if (!finite(z[i])) continue; + int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); + XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); + } + XFillPolygon (graphic->display, graphic->window, graphic->gc, points, object[0].ptype, Convex, CoordModeOrigin); + } + + free (points); } @@ -580,4 +712,7 @@ int scaleColor = (object[0].color < 0); + // NOTE that LoadObject.c:45 limits the allow + // object styles which may have negative (scaled) colors + ds = 0.5 * (graphic->dx + graphic->dy) * 0.003 * object[0].size; dz = 0.5 * (graphic->dx + graphic->dy) * 0.010; @@ -916,8 +1051,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dxp[i]))) goto skip_dxp; if (D > fabs(dxp[i]*mxi)) goto skip_dxp; @@ -999,8 +1134,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dyp[i]))) goto skip_dyp; if (D > fabs(dyp[i]*myj)) goto skip_dyp; @@ -1023,6 +1158,6 @@ skip_dyp: if (!(finite(x[i]) && finite(y[i]) && finite(dym[i]))) continue; - if (object[0].ptype == 5) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } if (D > fabs(dym[i]*myj)) continue; sx0 = x[i]*mxi + y[i]*mxj + bx + XCENTER; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c (revision 41170) @@ -148,4 +148,21 @@ void DrawImage (KapaImageWidget *image) { + Graphic *graphic; + + if (image == NULL) return; + + graphic = GetGraphic (); + + if (image[0].picture.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].picture.pix, 0, 0, + image[0].picture.x + 1, image[0].picture.y + 1, + image[0].picture.dx, image[0].picture.dy); + } +} + +// add the zoom, pan, crosshairs, status box, buttons +void DrawImageTool (KapaImageWidget *image) { + int i; Graphic *graphic; @@ -160,17 +177,4 @@ image[0].picture.dx+1, image[0].picture.dy+1); - if (image[0].picture.pix) { - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].picture.pix, 0, 0, - image[0].picture.x + 1, image[0].picture.y + 1, - image[0].picture.dx, image[0].picture.dy); - } - - for (i = 0; i < NOVERLAYS; i++) { - if (image[0].overlay[i].active) { - PaintOverlay (graphic, image, i); - } - } - if (image[0].location) { if (image[0].cmapbar.pix) { @@ -211,6 +215,4 @@ StatusBox (graphic, image); } - - FlushDisplay (); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c (revision 41170) @@ -160,4 +160,6 @@ break; + case XK_plus: + case XK_equal: case XK_KP_Add: if (modstate & ControlMask) { @@ -171,4 +173,7 @@ Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; + + case XK_minus: + case XK_underscore: case XK_KP_Subtract: if (modstate & ControlMask) { Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c (revision 41170) @@ -212,4 +212,5 @@ } bDrawBufferFree (buffer); + free (palette); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c (revision 41170) @@ -45,2 +45,7 @@ AddSection ("default", 0.0, 0.0, 1.0, 1.0, -1); } + +void FreeLayout (void) { + FreeSections (); + FreeRotFonts (); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c (revision 41170) @@ -93,20 +93,4 @@ SetSectionSizes (section); Refresh(); - // if (USE_XWINDOW) DrawFrame (graph); - // FlushDisplay (); - - /* XXX why did I do this??? */ -# if (0) - status = TRUE; - if (status) { - for (i = 0; i < Nsection; i++) { - PositionPicture (§ion[i]); - } - if (USE_XWINDOW) XClearWindow (graphic.display, graphic.window); - Refresh (); - } else { - FlushDisplay (); - } -# endif return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c (revision 41170) @@ -26,4 +26,6 @@ label[Nbytes] = 0; + FREE (label); + c = GetRotFont (&size); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c (revision 41170) @@ -42,5 +42,11 @@ graph[0].objects[N].color = KapaColormapSize() - 1; } - if ((graph[0].objects[N].style != KAPA_PLOT_POINTS) && (graph[0].objects[N].color < 0)) { + + // XXX watch out for this restriction in DrawObjects / bDrawObjects + int canScaleColor = FALSE; + canScaleColor |= (graph[0].objects[N].style == KAPA_PLOT_POINTS); + canScaleColor |= (graph[0].objects[N].style == KAPA_PLOT_POLYFILL); + + if (!canScaleColor && (graph[0].objects[N].color < 0)) { graph[0].objects[N].color = 0; } @@ -103,8 +109,13 @@ if (USE_XWINDOW) { - Graphic *graphic = GetGraphic(); - DrawObjectN (graphic, graph, &graph[0].objects[graph[0].Nobjects-1]); - } - FlushDisplay (); + if (0) { + // use this if we are not using buffered plotting + Graphic *graphic = GetGraphic(); + DrawObjectN (graphic, graph, &graph[0].objects[graph[0].Nobjects-1]); + FlushDisplay (); + } else { + Refresh (); + } + } return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c (revision 41170) @@ -0,0 +1,41 @@ +# include "Ximage.h" + +static int DumpOnExit = FALSE; + +int MemoryDump (int sock) { + OHANA_UNUSED_PARAM(sock); + + ohana_memdump_file (stderr, TRUE); + + return TRUE; +} + +int MemoryDumpLines (int sock) { + + int Nlines; + KiiScanMessage (sock, "%d", &Nlines); + + ohana_memdump_set_maxlines (Nlines); + + return TRUE; +} + +int MemoryDumpOnExit (int sock) { + OHANA_UNUSED_PARAM(sock); + + int state; + KiiScanMessage (sock, "%d", &state); + + MemoryDumpSetOnExit (state); + return TRUE; +} + +int MemoryDumpSetOnExit (int state) { + DumpOnExit = state; + return TRUE; +} + +int MemoryDumpAndExit (void) { + if (DumpOnExit) MemoryDump(0); + exit (0); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c (revision 41170) @@ -98,4 +98,9 @@ png_write_info (png_ptr, info_ptr); + if (graphic->smooth_sigma == 0.0) { + fprintf (stderr, "making PNG without antialias smoothing\n"); + fprintf (stderr, "use 'antialias (value)' to set smoothing scale (0.4 - 0.7 recommended)\n"); + } + if (haveImage) { buffer = bDrawIt (palette, Npalette, 3); @@ -109,4 +114,6 @@ bDrawBufferFree (buffer); + free (palette); + fclose (f); return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c (revision 41170) @@ -46,4 +46,6 @@ bDrawBufferFree (buffer); + free (palette); + return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c (revision 41170) @@ -1,5 +1,68 @@ # include "Ximage.h" +# define USE_BUFFERED_DRAW 1 + +void bDrawXimage (bDrawBuffer *buffer); +void Refresh_Buffered (void); +void Refresh_Unbuffered (void); + void Refresh (void) { + if (USE_BUFFERED_DRAW) { + Refresh_Buffered(); + } else { + Refresh_Unbuffered(); + } +} + +void Refresh_Buffered (void) { + + int Npalette; + Graphic *graphic; + + if (!USE_XWINDOW) return; + // if (HAVE_BACKING) return; + + graphic = GetGraphic(); + + // limit the png window to the min needed to contain the active graphic regions + SectionMinBoundary (graphic); + + // is the palette reasonable in modern context? + png_color *palette = KapaPNGPalette (&Npalette); + + bDrawBuffer *buffer = bDrawIt (palette, Npalette, 3); + + /* XClearWindow (graphic.display, graphic.window); */ + XSetForeground (graphic->display, graphic->gc, graphic->back); + XFillRectangle (graphic->display, graphic->window, graphic->gc, 0, 0, graphic->dx, graphic->dy); + XSetForeground (graphic->display, graphic->gc, graphic->fore); + + // copy buffer to Xwindow as image? + bDrawXimage (buffer); + bDrawBufferFree (buffer); + free (palette); + + // draw image tool for all sections + int Nsection = GetNumberOfSections (); + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + + KapaImageWidget *image = section->image; + DrawImageTool (image); + + /*** PaintOverlay is called in DrawImage *** + if (!image) continue; + for (int j = 0; j < NOVERLAYS; j++) { + if (image[0].overlay[j].active) { + PaintOverlay (graphic, image, j); + } + } + */ + } + + FlushDisplay (); +} + +void Refresh_Unbuffered (void) { int i, Nsection; @@ -20,8 +83,21 @@ Nsection = GetNumberOfSections (); for (i = 0; i < Nsection; i++) { - section = GetSectionByNumber (i); - DrawSectionBG (graphic, section); - DrawImage (section->image); - DrawGraph (section->graph); + section = GetSectionByNumber (i); + DrawSectionBG (graphic, section); + + KapaImageWidget *image = section->image; + DrawImage (image); + DrawImageTool (image); + + /*** the overlay is added in DrawImage ***/ + if (image) { + for (int j = 0; j < NOVERLAYS; j++) { + if (image[0].overlay[j].active) { + PaintOverlay (graphic, image, j); + } + } + } + + DrawGraph (section->graph); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c (revision 41170) @@ -72,5 +72,5 @@ } } - fprintf (stderr, "ruff Npix: %d vs %d\n", i, graphic[0].Npixels); + // fprintf (stderr, "ruff Npix: %d vs %d\n", i, graphic[0].Npixels); // all other modes are 1D: set the flag: @@ -217,4 +217,5 @@ !strncmp (graphic->colormapName, "lgcy:", 5) || !strncmp (graphic->colormapName, "cetf:", 5) || + !strncmp (graphic->colormapName, "cetr:", 5) || !strncmp (graphic->colormapName, "csvf:", 5)) { @@ -233,9 +234,10 @@ int isCSV = !strncmp (graphic->colormapName, "csvf:", 5); int isCET = !strncmp (graphic->colormapName, "cetf:", 5); + int isCETRev = !strncmp (graphic->colormapName, "cetr:", 5); int isLegacy = !strncmp (graphic->colormapName, "lgcy:", 5); float fracIndex, fracRed, fracBlue, fracGreen; - if (isCET) fracIndex = 0.0; + if (isCET || isCETRev) fracIndex = 0.0; while (scan_line_maxlen (f, line, 1024) != EOF) { @@ -247,5 +249,5 @@ if (Nscan != 4) continue; } - if (isCET) { + if (isCET || isCETRev) { Nscan = sscanf (line, "%f,%f,%f", &fracRed, &fracGreen, &fracBlue); fracIndex += 1.0 / 256.0; @@ -256,5 +258,5 @@ if (Nscan != 4) continue; } - if (!isCSV && !isLegacy && !isCET) { + if (!isCSV && !isLegacy && !isCET && !isCETRev) { Nscan = sscanf (line, "%f %f %f %f", &fracIndex, &fracRed, &fracGreen, &fracBlue); if (Nscan != 4) continue; @@ -299,4 +301,16 @@ } } + + // reverse the color sequence: + if (isCETRev) { + for (i = 0; i < MaxValue / 2; i++) { + unsigned short tmp; + // fprintf (stderr, "swap: %d %d\n", graphic[0].cmap[i].red, graphic[0].cmap[MaxValue - 1 - i].red); + tmp = graphic[0].cmap[i].red; graphic[0].cmap[i].red = graphic[0].cmap[MaxValue - 1 - i].red; graphic[0].cmap[MaxValue - 1 - i].red = tmp; + tmp = graphic[0].cmap[i].blue; graphic[0].cmap[i].blue = graphic[0].cmap[MaxValue - 1 - i].blue; graphic[0].cmap[MaxValue - 1 - i].blue = tmp; + tmp = graphic[0].cmap[i].green; graphic[0].cmap[i].green = graphic[0].cmap[MaxValue - 1 - i].green; graphic[0].cmap[MaxValue - 1 - i].green = tmp; + } + } + goto store_colors; } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c (revision 41170) @@ -32,4 +32,6 @@ graphic->pixels = NULL; + graphic->smooth_sigma = 0.0; + if (!USE_XWINDOW) { ALLOCATE (graphic[0].pixels, unsigned long, NPIXELS_STATIC); @@ -50,5 +52,4 @@ CheckVisual (graphic, argc, argv); CheckColors (graphic, argc, argv); - icon.width = icon_width; @@ -84,2 +85,22 @@ } +int SetSmoothSigma (int sock) { + + float sigma; + KiiScanMessage (sock, "%f", &sigma); + + if (isfinite(sigma)) { + if ((sigma <= 1.1) && (sigma >= 0.0)) { + graphic->smooth_sigma = sigma; + } + } + return TRUE; +} + +void FreeGraphic () { + free (graphic->pixels); + free (graphic->cmap); + free (graphic->color); + free (graphic->colormapName); + free (graphic); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c (revision 41170) @@ -23,4 +23,9 @@ NAME_WINDOW = strcreate (argv[N]); remove_argument(N, argc, argv); + } + + if ((N = get_argument (*argc, argv, "-memdump"))) { + remove_argument(N, argc, argv); + MemoryDumpSetOnExit (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c (revision 41170) @@ -1,3 +1,6 @@ # include "Ximage.h" + +// XXX for the moment, this function does NOT set the mask bits. +// since we lay graphics on top of images, this is OK for now # define WHITE_R 255 @@ -54,10 +57,18 @@ // the created buffer is supposed to contain the output windows + if (Xs < 0) { + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); + abort(); + } if (buffer[0].Nx < Xs + dx) { - fprintf (stderr, "invalid condition\n"); + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); + abort(); + } + if (Ys < 0) { + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); abort(); } if (buffer[0].Ny < Ys + dy) { - fprintf (stderr, "invalid condition\n"); + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); abort(); } @@ -153,4 +164,27 @@ return (TRUE); +} + +void bDrawXimage (bDrawBuffer *buffer) { + + Graphic *graphic = GetGraphic (); + + ALLOCATE_PTR (data, char, 4*buffer->Nx*buffer->Ny); + + for (int iy = 0; iy < buffer->Ny; iy++) { + for (int ix = 0; ix < buffer->Nx; ix++) { + data[4*(iy*buffer->Nx + ix) + 0] = buffer->pixels[iy][3*ix + 2]; + data[4*(iy*buffer->Nx + ix) + 1] = buffer->pixels[iy][3*ix + 1]; + data[4*(iy*buffer->Nx + ix) + 2] = buffer->pixels[iy][3*ix + 0]; + data[4*(iy*buffer->Nx + ix) + 3] = 0; + } + } + + XImage *pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, + data, buffer->Nx, buffer->Ny, 32, 0); + + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, pix, 0, 0, 1, 1, buffer->Nx, buffer->Ny); + + free (data); } @@ -186,4 +220,5 @@ } bDrawBufferFree (buffer); + free (palette); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c (revision 41170) @@ -3,31 +3,47 @@ bDrawBuffer *bDrawIt (png_color *palette, int Npalette, int Nbyte) { - int i, j, Nsection; - bDrawBuffer *buffer; - bDrawColor black; - Graphic *graphic; - Section *section; + Graphic *graphic = GetGraphic(); + bDrawColor black = KapaColorByName ("black"); - graphic = GetGraphic(); + // get the number of sections + int Nsection = GetNumberOfSections (); - black = KapaColorByName ("black"); + // in order to allow the anti-aliasing to affect the text & graphs but not the images + // I need to generate the images in one buffer and the graphs in a second buffer + // then merge the two buffers. - // if we want to trim, we'll need to carry about the start in graphic coords and - // the dx,dy size. - buffer = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); - bDrawSetStyle (buffer, black, 0, 0, 1.0); + // base will hold the images + bDrawBuffer *base = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); + bDrawSetStyle (base, black, 0, 0, 1.0); - // reset the sizes for all sections - Nsection = GetNumberOfSections (); - for (i = 0; i < Nsection; i++) { - section = GetSectionByNumber (i); - bDrawImage (buffer, section->image, graphic); - for (j = 0; section->image && (j < NOVERLAYS); j++) { - if (section->image->overlay[j].active) bDrawOverlay (buffer, section->image, j); - } - bDrawGraph (buffer, section->graph); + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + bDrawImage (base, section->image, graphic); } - return (buffer); + // graph will hold the graphic overlay + bDrawBuffer *graph = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); + bDrawSetStyle (graph, black, 0, 0, 1.0); + + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + for (int j = 0; section->image && (j < NOVERLAYS); j++) { + if (section->image->overlay[j].active) bDrawOverlay (graph, section->image, j); + } + bDrawGraph (graph, section->graph); + } + + // apply anti-aliasing only to the graph + if (graphic->smooth_sigma > 0.0) { + // anything > 1.1 blurs the image too much + graphic->smooth_sigma = MIN (graphic->smooth_sigma, 1.1); + bDrawSmooth (graph, graphic->smooth_sigma); + } + + // place graph on base + bDrawMerge (base, graph); + bDrawBufferFree (graph); + + return (base); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c (revision 41170) @@ -9,4 +9,6 @@ # define OpenTriangle(BUF,X1,Y1,X2,Y2,X3,Y3) (bDrawTriOpen (BUF, (X1), (Y1), (X2), (Y2), (X3), (Y3))) +// I should not have a local static variable for this: +// I should just pass the graphic structure to the called functions below static Graphic *graphic; @@ -35,4 +37,10 @@ case KAPA_PLOT_CONNECT: bDrawConnect (buffer, graph, object); + break; + case KAPA_PLOT_POLYGON: + bDrawPolygons (buffer, graph, object); + break; + case KAPA_PLOT_POLYFILL: + bDrawFillPolygons (buffer, graph, object); break; case KAPA_PLOT_HISTOGRAM: @@ -103,4 +111,153 @@ sx0 = sx1; sy0 = sy1; } +} + +void bDrawPolygons (bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0, sx1, sy1; + double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx; + sy0 = x[k]*myi + y[k]*myj + by; + + // last point connects to first + if (j == object[0].ptype - 1) { + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + } else { + sx1 = x[k+1]*mxi + y[k+1]*mxj + bx; + sy1 = x[k+1]*myi + y[k+1]*myj + by; + } + bDrawClipLine (buffer, sx0, sy0, sx1, sy1, X0, Y0, X1, Y1); + } + } +} + +void bDrawFillPolygons (bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0; + // double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + /* + window boundary so we can clip objects + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + */ + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + ALLOCATE_PTR (xpts, double, object[0].ptype); + ALLOCATE_PTR (ypts, double, object[0].ptype); + + int scaleColor = (object[0].color < 0); + unsigned char *pixel1, *pixel2, *pixel3; + float *z = object[0].z; + + if (scaleColor) { + // scaled colors use the colormap defined for the graphic + ALLOCATE (pixel1, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel2, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel3, unsigned char, graphic[0].Npixels); + + /** cmap[i].pixel must be defined even if X is not used **/ + for (int i = 0; i < graphic[0].Npixels; i++) { /* set up pixel array */ + pixel1[i] = graphic[0].cmap[i].red >> 8; + pixel2[i] = graphic[0].cmap[i].green >> 8; + pixel3[i] = graphic[0].cmap[i].blue >> 8; + } + + } + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx; + sy0 = x[k]*myi + y[k]*myj + by; + + xpts[j] = sx0; + ypts[j] = sy0; + } + + if (scaleColor) { + if (!finite(z[i])) continue; + int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); + buffer->bColor_R = pixel1[pixel]; + buffer->bColor_G = pixel2[pixel]; + buffer->bColor_B = pixel3[pixel]; + } + bDrawPolyFill (buffer, xpts, ypts, object[0].ptype); + } + free (xpts); + free (ypts); } @@ -886,8 +1043,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dxp[i]))) goto skip_dxp; if (D > fabs(dxp[i]*mxi)) goto skip_dxp; @@ -969,8 +1126,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dyp[i]))) goto skip_dyp; if (D > fabs(dyp[i]*myj)) goto skip_dyp; @@ -993,6 +1150,6 @@ skip_dyp: if (!(finite(x[i]) && finite(y[i]) && finite(dym[i]))) continue; - if (object[0].ptype == 5) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } if (D > fabs(dym[i]*myj)) continue; sx0 = x[i]*mxi + y[i]*mxj + bx; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c (revision 41170) @@ -24,8 +24,13 @@ } - Xmin = 0; - Ymin = 0; - Xmax = image[0].picture.dx; - Ymax = image[0].picture.dy; + // Xmin = 0; + // Ymin = 0; + // Xmax = image[0].picture.dx; + // Ymax = image[0].picture.dy; + + Xmin = image[0].picture.x; + Ymin = image[0].picture.y; + Xmax = image[0].picture.x + image[0].picture.dx; // maybe this should be just dx? + Ymax = image[0].picture.y + image[0].picture.dy; if (N == INFRONT) { @@ -35,5 +40,7 @@ for (i = 0; i < image[0].overlay[N].Nobjects; i++) { - Image_to_Picture (&X, &Y, image[0].overlay[N].objects[i].x, image[0].overlay[N].objects[i].y, &image[0].picture); + // XXX the 0.5,0.5 offset is apparently needed here. + // work on rationalizing these functions in the context of their different plotting types + Image_to_Screen (&X, &Y, image[0].overlay[N].objects[i].x - 0.5, image[0].overlay[N].objects[i].y - 0.5, &image[0].picture); dX = image[0].overlay[N].objects[i].dx * expand; dY = image[0].overlay[N].objects[i].dy * expand; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c (revision 41170) @@ -11,4 +11,10 @@ CloseDisplay (); - exit (0); + + // free things + FreeLayout(); + FreeGraphic(); + FREE (NAME_WINDOW); + + MemoryDumpAndExit (); } Index: branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c (revision 41170) @@ -76,5 +76,6 @@ } -/* sort a coordinate pair (X,Y) and the associated index (S) */ +/* sort the index of a coordinate pair (X,Y) on X (vector pair stays unsorted) */ +// XXX isn't this function equivalent to dsort_indexonly? void sort_coords_indexonly (double *X, double *Y, off_t *S, off_t N) { OHANA_UNUSED_PARAM(Y); Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h (revision 41170) @@ -45,5 +45,7 @@ KAPA_PLOT_BARS_OUTLINE = 4, KAPA_PLOT_BARS_OUTFILL = 5, - KAPA_PLOT_INVALID_MAX = 6, + KAPA_PLOT_POLYGON = 6, + KAPA_PLOT_POLYFILL = 7, + KAPA_PLOT_INVALID_MAX = 8, } KapaPlotStyle; @@ -172,6 +174,8 @@ int Nx, Ny, Nbyte; bDrawColor **pixels; + char **mask; png_color *palette; int Npalette; + // current drawing values: int bWeight; @@ -256,4 +260,8 @@ int KapaGetImageData (int fd, KapaImageData *graphmode); int KapaSetToolbox (int fd, int location); +int KapaSetSmoothSigma (int fd, float sigma); +int KapaMemoryDump (int fd); +int KapaMemoryDumpLines (int fd, int Nlines); +int KapaMemoryDumpOnExit (int fd, int state); /* KapaColors */ @@ -272,4 +280,5 @@ /* RotFont.c */ void InitRotFonts PROTO((void)); +void FreeRotFonts PROTO((void)); int SetRotFont PROTO((char *name, int size)); char *GetRotFont PROTO((int *size)); @@ -292,4 +301,5 @@ bDrawBuffer *bDrawBufferCreate (int Nx, int Ny, int Nbyte, png_color *palette, int Npalette); void bDrawBufferFree (bDrawBuffer *buffer); +int bDrawMerge (bDrawBuffer *base, bDrawBuffer *layer); void bDrawSetBuffer (bDrawBuffer *buffer); void bDrawSetColor (bDrawBuffer *buffer, bDrawColor color); @@ -312,4 +322,7 @@ void bDrawTriOpen (bDrawBuffer *buffer, double x1, double y1, double x2, double y2, double x3, double y3); void bDrawTriFill (bDrawBuffer *buffer, double x1, double y1, double dx, double dy); +void bDrawPolyFill (bDrawBuffer *buffer, double *x, double *y, int Npoints); + +void bDrawSmooth (bDrawBuffer *buffer, float sigma); /* bDrawRotFont.c */ Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c (revision 41170) @@ -54,4 +54,7 @@ if (!strcasecmp (string, "outline")) return KAPA_PLOT_BARS_OUTLINE; if (!strcasecmp (string, "outfill")) return KAPA_PLOT_BARS_OUTFILL; + + if (!strcasecmp (string, "polygon")) return KAPA_PLOT_POLYGON; + if (!strcasecmp (string, "polyfill")) return KAPA_PLOT_POLYFILL; if (strlen(string) > 2) { Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c (revision 41170) @@ -455,2 +455,33 @@ } +int KapaSetSmoothSigma (int fd, float sigma) { + + KiiSendCommand (fd, 4, "SIGM"); + KiiSendMessage (fd, "%4.1f ", sigma); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDump (int fd) { + + KiiSendCommand (fd, 4, "MEMD"); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDumpLines (int fd, int Nlines) { + + KiiSendCommand (fd, 4, "MEML"); + KiiSendMessage (fd, "%d ", Nlines); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDumpOnExit (int fd, int state) { + + KiiSendCommand (fd, 4, "MEMX"); + KiiSendMessage (fd, "%d ", state); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c (revision 41170) @@ -32,4 +32,10 @@ strncpy (currentname, RotFonts[DEFFONT].name, 63); currentname[63] = 0; currentsize = RotFonts[DEFFONT].size; +} + +void FreeRotFonts (void) { + free (RotFonts); + RotFonts = FALSE; + RotFontInited = FALSE; } Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c (revision 41170) @@ -1,14 +1,34 @@ # include -// move these to the bDrawBuffer type -// static int bWeight; -// static int bType; -// static bDrawColor bColor; -// static bDrawColor bColor_R; -// static bDrawColor bColor_G; -// static bDrawColor bColor_B; -// static bDrawBuffer *bBuffer; +// buffer->pixels carries the plot image +// buffer->mask is 0 if the pixel is untouched, 1 if the pixel has data void bDrawCircleSingle (bDrawBuffer *buffer, double xc, double yc, double radius); + +int bDrawMerge (bDrawBuffer *base, bDrawBuffer *layer) { + + // the two bDrawBuffers must match in size and depth + + if (base->Nx != layer->Nx) return FALSE; + if (base->Ny != layer->Ny) return FALSE; + if (base->Nbyte != layer->Nbyte) return FALSE; + + for (int j = 0; j < base->Ny; j++) { + for (int i = 0; i < base->Nx; i++) { + + if (!layer->mask[j][i]) continue; + + // completely opaque top layer: + if (base->Nbyte == 1) { + base->pixels[j][i] = layer->pixels[j][i]; + } else { + base->pixels[j][3*i+0] = layer->pixels[j][3*i+0]; + base->pixels[j][3*i+1] = layer->pixels[j][3*i+1]; + base->pixels[j][3*i+2] = layer->pixels[j][3*i+2]; + } + } + } + return TRUE; +} // create a drawing buffer with either 1 or 3 byte colors @@ -35,6 +55,8 @@ ALLOCATE (buffer[0].pixels, bDrawColor *, Ny); + ALLOCATE (buffer[0].mask, char *, Ny); for (i = 0; i < Ny; i++) { ALLOCATE (buffer[0].pixels[i], bDrawColor, Nbyte*Nx); + ALLOCATE (buffer[0].mask[i], char, Nx); for (j = 0; j < Nx; j++) { if (Nbyte == 1) { @@ -45,4 +67,5 @@ buffer[0].pixels[i][3*j+2] = white_B; } + buffer[0].mask[i][j] = 0; // for now: 0 = no data, 1 = data } } @@ -63,7 +86,9 @@ for (i = 0; i < buffer[0].Ny; i++) { free (buffer[0].pixels[i]); + free (buffer[0].mask[i]); } free (buffer[0].pixels); - free (buffer[0].palette); + free (buffer[0].mask); + // free (buffer[0].palette); free (buffer); return; @@ -121,11 +146,9 @@ buffer[0].pixels[y][x] = buffer->bColor; } else { - // buffer[0].pixels[y][3*x+0] = buffer->bColor_R; - // buffer[0].pixels[y][3*x+1] = buffer->bColor_G; - // buffer[0].pixels[y][3*x+2] = buffer->bColor_B; - buffer[0].pixels[y][3*x+0] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+0] + alpha * buffer->bColor_R)); + buffer[0].pixels[y][3*x+0] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+0] + alpha * buffer->bColor_R)); // was just buffer->bColor_R, etc buffer[0].pixels[y][3*x+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+1] + alpha * buffer->bColor_G)); buffer[0].pixels[y][3*x+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+2] + alpha * buffer->bColor_B)); } + buffer[0].mask[y][x] = 1; return; } @@ -150,37 +173,51 @@ void bDrawRectOpen (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { - int X1, Y1, X2, Y2; - if (x1 > x2) SWAP (x1, x2); if (y1 > y2) SWAP (y1, y2); + int X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); + int X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); + + int Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); + int Y2 = MIN (MAX (ROUND (y2), 1), buffer[0].Ny - 1); + + int dNs = -0.5*(buffer->bWeight - 1); + /* 0, 0, 0, -1, -1, -2, -2 */ + + int dNe = +0.5*buffer->bWeight + 1; + /* 1, 1, 2, 2, 2, 3, 3 */ + + for (int dN = dNs; dN < dNe; dN ++) { + // line on the bottom needs to run longer for the negative dN values + bDrawLineHorizontal (buffer, X1 + dN, X2 + 1 - dN, Y1 + dN); + + // line on the top needs to run longer for the positive dN values + bDrawLineHorizontal (buffer, X1 + dN, X2 + 1 - dN, Y2 - dN); + + // line on the left needs to run longer for the negative dN values + bDrawLineVertical (buffer, X1 + dN, Y1 + dN, Y2 - dN); + + // line on the right needs to run longer for the positive dN values + bDrawLineVertical (buffer, X2 - dN, Y1 + dN, Y2 - dN); + } + return; +} + +void bDrawRectFill (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { + + int i; + int X1, Y1, X2, Y2; + + if (x1 > x2) SWAP (x1, x2); + if (y1 > y2) SWAP (y1, y2); + X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); - Y2 = MIN (MAX (ROUND (y2), 1), buffer[0].Ny - 1); - - bDrawLineHorizontal (buffer, X1, X2 + 1, Y1); - bDrawLineHorizontal (buffer, X1, X2 + 1, Y2); - bDrawLineVertical (buffer, X1, Y1, Y2); - bDrawLineVertical (buffer, X2, Y1, Y2); - return; -} - -void bDrawRectFill (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { - - int i; - int X1, Y1, X2, Y2; - - if (x1 > x2) SWAP (x1, x2); - if (y1 > y2) SWAP (y1, y2); - - X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); - X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); - - Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); Y2 = MIN (MAX (ROUND (y2), 0), buffer[0].Ny - 1); for (i = Y1; i < Y2; i++) { + // this should be a line of width 1 or we duplicate pixels bDrawLineHorizontal (buffer, X1, X2, i); } @@ -235,4 +272,5 @@ // use the Bresenham line drawing technique // integer-only Bresenham line-draw version which is fast +// bresenham assumes x1 < x2 and (y2 - y1) < (x2 - x1) void bDrawLineBresen (bDrawBuffer *buffer, int X1, int Y1, int X2, int Y2, int swapcoords) { @@ -299,8 +337,6 @@ buffer[0].pixels[Y][3*i+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[Y][3*i+1] + alpha * buffer->bColor_G)); buffer[0].pixels[Y][3*i+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[Y][3*i+2] + alpha * buffer->bColor_B)); - // buffer[0].pixels[Y][3*i+0] = buffer->bColor_R; - // buffer[0].pixels[Y][3*i+1] = buffer->bColor_G; - // buffer[0].pixels[Y][3*i+2] = buffer->bColor_B; - } + } + buffer[0].mask[Y][i] = 1; } return; @@ -326,8 +362,6 @@ buffer[0].pixels[i][3*X+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[i][3*X+1] + alpha * buffer->bColor_G)); buffer[0].pixels[i][3*X+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[i][3*X+2] + alpha * buffer->bColor_B)); - // buffer[0].pixels[i][3*X+0] = buffer->bColor_R; - // buffer[0].pixels[i][3*X+1] = buffer->bColor_G; - // buffer[0].pixels[i][3*X+2] = buffer->bColor_B; - } + } + buffer[0].mask[i][X] = 1; } return; @@ -384,4 +418,148 @@ return; +} + +# define BOT_LEFT 0 +# define BOT_RGHT 1 +# define TOP_LEFT 2 +# define TOP_RGHT 3 + +// I should probably look up an appropriate recipe for this +// source code for XDrawPolyFill? + +int GetNextSeqNumber (int seq, int Npoint, int Clockwise, int LeftSide); +int bDrawFillBetweenSegments (bDrawBuffer *buffer, int *x, int *y, int ystart); + +void bDrawPolyFill (bDrawBuffer *buffer, double *x, double *y, int Npoints) { + + // The coord list which is passed in (x,y) must be in order around the contour + for (int i = 0; i < Npoints; i++) { + fprintf (stderr, "%d : %f,%f\n", i, x[i], y[i]); + } + + // First, find the lowest point and start at that sequence number + int iMin = 0; + double yMin = y[iMin]; + for (int i = 1; i < Npoints; i++) { + if (y[i] < yMin) { iMin = i; yMin = y[iMin]; } + } + + // we generate two line segments and will fill between them + int xval[4]; + int yval[4]; + + // the starting point is a vertex + xval[BOT_LEFT] = x[iMin]; yval[BOT_LEFT] = y[iMin]; + xval[BOT_RGHT] = x[iMin]; yval[BOT_RGHT] = y[iMin]; + + // get iNext, iPrev assuming ClockWise, then test + int isCW = TRUE; + int iNextLeft = GetNextSeqNumber (iMin, Npoints, isCW, TRUE); // TRUE => left-side + int iNextRght = GetNextSeqNumber (iMin, Npoints, isCW, FALSE); + + // if this is true, the points are not clockwise + if (x[iNextLeft] > x[iNextRght]) { + isCW = FALSE; + int tmp = iNextLeft; + iNextLeft = iNextRght; + iNextRght = tmp; + } + + int iTL = iNextLeft; // sequence number of the top,left point + int iTR = iNextRght; // sequence number of the top,right point + xval[TOP_LEFT] = x[iTL]; yval[TOP_LEFT] = y[iTL]; + xval[TOP_RGHT] = x[iTR]; yval[TOP_RGHT] = y[iTR]; + + // we need to track the starting row, start at the bottom + int ystart = yval[BOT_LEFT]; + + while (TRUE) { + int isLeft = bDrawFillBetweenSegments (buffer, xval, yval, ystart); + + // the last pair of segments in the sequence end at the same top point: + if (iTL == iTR) break; + + // after one pass, if isLeft is true, then cycle the points on the left + // segment, otherwise cycle the right segment: + + if (isLeft) { + iTL = GetNextSeqNumber (iTL, Npoints, isCW, TRUE); + xval[BOT_LEFT] = xval[TOP_LEFT]; yval[BOT_LEFT] = yval[TOP_LEFT]; + xval[TOP_LEFT] = x[iTL]; yval[TOP_LEFT] = y[iTL]; + ystart = yval[BOT_LEFT]; + } else { + iTR = GetNextSeqNumber (iTR, Npoints, isCW, FALSE); + xval[BOT_RGHT] = xval[TOP_RGHT]; yval[BOT_RGHT] = yval[TOP_RGHT]; + xval[TOP_RGHT] = x[iTR]; yval[TOP_RGHT] = y[iTR]; + ystart = yval[BOT_RGHT]; + } + } + return; +} + +// we have two line segments defined by x[],y[]. + +// [0],[1] is the left-side segment +// [2],[3] is the right-side segment + +// Fill the region between the two segments until +// the lower of the two y coordinates y[1], y[3] + +// return which of y[1] or y[3] was the ending row + +int bDrawFillBetweenSegments (bDrawBuffer *buffer, int *x, int *y, int ystart) { + + int dY_L = y[TOP_LEFT] - y[BOT_LEFT]; + int dX_L = x[TOP_LEFT] - x[BOT_LEFT]; + + int dY_R = y[TOP_RGHT] - y[BOT_RGHT]; + int dX_R = x[TOP_RGHT] - x[BOT_RGHT]; + + double Slope_L = (dY_L == 0) ? NAN : dX_L / (double) dY_L; + double Slope_R = (dY_R == 0) ? NAN : dX_R / (double) dY_R; + + // stop at whichever comes first + // XXX not sure if I need to use < or <= here: + // with <=, the value of Y after the loop will be one too high. + int Y; + for (Y = ystart; (Y < y[TOP_LEFT]) && (Y < y[TOP_RGHT]); Y++) { + + // calculating X_L, X_R based on the slope and current y point + int X_L = (dY_L == 0) ? x[BOT_LEFT] : (Y - y[BOT_LEFT]) * Slope_L + x[BOT_LEFT]; + int X_R = (dY_R == 0) ? x[BOT_RGHT] : (Y - y[BOT_RGHT]) * Slope_R + x[BOT_RGHT]; + + // draw horizontal line from X_L to X_R at Y + bDrawLineHorizontal (buffer, X_L, X_R, Y); + } + + if (Y == y[TOP_LEFT]) { + return TRUE; // TRUE is LEFT SIDE + } else { + return FALSE; + } +} + +int GetNextSeqNumber (int seq, int Npoints, int Clockwise, int LeftSide) { + + int iNext; + + if (Clockwise) { + if (LeftSide) { + // next point is next in sequence + iNext = (seq < Npoints - 1) ? seq + 1 : 0; + } else { + // next point is prev in sequence + iNext = (seq > 0) ? seq - 1 : Npoints - 1; + } + } else { + if (LeftSide) { + // next point is prev in sequence + iNext = (seq > 0) ? seq - 1 : Npoints - 1; + } else { + // next point is next in sequence + iNext = (seq < Npoints - 1) ? seq + 1 : 0; + } + } + return iNext; } @@ -585,4 +763,5 @@ if (x == 0) { if (decrementY) { + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc , Xc + 1, Yc + y); bDrawLineHorizontal (buffer, Xc , Xc + 1, Yc - y); @@ -590,4 +769,5 @@ // center line + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc ); return; @@ -595,4 +775,5 @@ if (x == y) { + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc + y); bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc - y); @@ -602,4 +783,5 @@ // only draw these two lines if we decrement y if (decrementY) { + // this must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc + y); bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc - y); @@ -607,4 +789,5 @@ // always draw these two lines: + // this must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc + x); bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc - x); @@ -648,5 +831,110 @@ } +// run a smoothing kernel on each channel for anti-aliasing +// XXX I need to consider the mask as well +void bDrawSmooth (bDrawBuffer *buffer, float sigma) { + + // generate a temp buffer for storage of the smoothed result + int Nx = buffer[0].Nx; + int Ny = buffer[0].Ny; + ALLOCATE_PTR (temp, bDrawColor, Nx*Ny); + + //* build a 1D gaussian + int Nsigma = 2; + int Ns = (int) (Nsigma*sigma + 0.5); + int Ngauss = 2*Ns + 1; + ALLOCATE_PTR (gaussnorm, float, Ngauss); + float *gauss = &gaussnorm[Ns]; + for (int i = -Ns; i < Ns + 1; i++) { + gauss[i] = exp ((i*i)/(-2*sigma*sigma)); + } + + // NOTE XXX: need to handle Nbyte = 1 or 3 + int Nchannel = 3; + + // we need to smooth the 3 channels independently, do this as three passes + for (int ch = 0; ch < Nchannel; ch++) { + + // smooth in X direction + for (int j = 0; j < Ny; j++) { + bDrawColor *vi = (bDrawColor *) &buffer[0].pixels[j][ch]; + bDrawColor *vo = &temp[j*Nx]; + for (int i = 0; i < Nx; i++, vo++, vi += Nchannel) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (i+n < 0) continue; + if (i+n >= Nx) continue; + s += gauss[n]*vi[Nchannel*n]; + g += gauss[n]; + } + *vo = s / g; + } + } + + // NOTE: output maps back into original image + // careful on the counting + + /* smooth in Y direction */ + for (int i = 0; i < Nx; i++) { + bDrawColor *vi = &temp[i]; + for (int j = 0; j < Ny; j++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (j+n < 0) continue; + if (j+n >= Ny) continue; + s += gauss[n]*vi[(n+j)*Nx]; // vi is a single-index array, so this is stepping by rows + g += gauss[n]; + } + buffer[0].pixels[j][Nchannel*i + ch] = s / g; + } + } + } + + // smooth the mask + ALLOCATE_PTR (temp_mask, float, Nx*Ny); + + // smooth in X direction + for (int j = 0; j < Ny; j++) { + char *vi = (char *) &buffer[0].mask[j][0]; + float *vo = &temp_mask[j*Nx]; + for (int i = 0; i < Nx; i++, vo++, vi++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (i+n < 0) continue; + if (i+n >= Nx) continue; + s += gauss[n]*vi[n]; + g += gauss[n]; + } + *vo = s / g; + } + } + + // NOTE: output maps back into original image + // careful on the counting + + /* smooth in Y direction */ + for (int i = 0; i < Nx; i++) { + float *vi = &temp_mask[i]; + for (int j = 0; j < Ny; j++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (j+n < 0) continue; + if (j+n >= Ny) continue; + s += gauss[n]*vi[(n+j)*Nx]; // vi is a single-index array, so this is stepping by rows + g += gauss[n]; + } + buffer[0].mask[j][i] = (s / g > 0.05) ? 1 : 0; + } + } + + free (temp); + free (temp_mask); + free (gaussnorm); + + return; +} + /* +NOTES on Bresenham-style circles: the discriminant of inside or outside the circle is: Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h (revision 41170) @@ -44,4 +44,7 @@ void ohana_memcheck_block_func (const char *myFile, int myLine, const char *myFunc, void *in); void real_free (void *in); + +void ohana_memdump_strings_file (FILE *f, int VERBOSE); +void ohana_memdump_set_maxlines (int N); # define ohana_memcheck(X) ohana_memcheck_func (__FILE__, __LINE__, __func__, X) Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h (revision 41170) @@ -83,3 +83,6 @@ void isortfour (int *X, int *Y, int *Z, int *W, int N); +void dsort_indexonly (double *X, off_t *S, off_t N); +void dsort_int_indexonly (double *X, int *S, int N); + #endif Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c (revision 41170) @@ -45,4 +45,6 @@ static pthread_mutex_t memBlockListMutex = PTHREAD_MUTEX_INITIALIZER; + +static int NblockMaxDump = 100; void ohana_meminit () { @@ -435,5 +437,5 @@ fprintf (stderr, "memory corruption\n"); } - if (Nbad < 100) { + if (Nbad < NblockMaxDump) { fprintf (stderr, " file: %s, line: %d, func: %s\n", thisBlock->file, thisBlock->line, thisBlock->func); } @@ -613,5 +615,5 @@ Nbytes += thisBlock->size; - if (Ntotal < 100) { + if (Ntotal < NblockMaxDump) { if (good) { fprintf (f, " %zd %zd %zd GOOD %s %d, func: %s\n", Ntotal, thisBlock->size, Nbytes, thisBlock->file, thisBlock->line, thisBlock->func); @@ -677,2 +679,49 @@ return memstats; } + +void ohana_memdump_strings_file (FILE *f, int VERBOSE) { + + if (!lastBlock) { + if (VERBOSE) fprintf (f, "no memory allocated\n"); + return; + } + + OhanaMemblock *thisBlock = lastBlock; + + size_t Ntotal = 0; + size_t Nbytes = 0; + size_t Nstring = 0; + + fprintf (f, " entry | bytes | cumulative : line : string\n"); + + while (thisBlock) { + + // pointer to the start of the user memory + char *ptr = (char *)(thisBlock + 1); + + Ntotal ++; + Nbytes += thisBlock->size; + + int N = strlen(thisBlock->file); + if (N > 22) { + int found = !strcmp("libohana/src/string.c", &thisBlock->file[N - 21]); + if (found) { + if (Nstring < NblockMaxDump) { + fprintf (f, " %8zd | %8zd | %10zd : %4d : %s\n", Ntotal, thisBlock->size, Nbytes, thisBlock->line, ptr); + } + Nstring ++; + } + thisBlock = thisBlock->prevBlock; + } + } + + if (Nstring || VERBOSE) { + fprintf (f, "%zd strings allocated\n", Nstring); + } + + return; +} + +void ohana_memdump_set_maxlines (int N) { + NblockMaxDump = N; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c (revision 41170) @@ -196,2 +196,33 @@ } +/* sort the index of a vector (vector stays unsorted) */ +void dsort_indexonly (double *X, off_t *S, off_t N) { + +# define SWAPFUNC(A,B){ off_t itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ +} +# define COMPARE(A,B)((!isfinite(X[S[A]]) && isfinite(X[S[B]])) || (X[S[A]] < X[S[B]])) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +/* sort the index of a vector (vector stays unsorted) */ +void dsort_int_indexonly (double *X, int *S, int N) { + +# define SWAPFUNC(A,B){ int itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ +} +# define COMPARE(A,B)((!isfinite(X[S[A]]) && isfinite(X[S[B]])) || (X[S[A]] < X[S[B]])) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c (revision 41170) @@ -39,4 +39,11 @@ } + int binning = 1; + if ((N = get_argument (argc, argv, "-binning"))) { + remove_argument (N, &argc, argv); + binning = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int PSFTYPE = IS_DOT; if ((N = get_argument (argc, argv, "-psf"))) { @@ -80,6 +87,9 @@ Ymax = graphmode.ymax; Ymin = graphmode.ymin; - dX = (Xmax - Xmin) / (Xpix - 1); - dY = (Ymax - Ymin) / (Ypix - 1); + + // (dX, dY) are the pixel scale of the output image (output pixels / input pixels) + + dX = binning * (Xmax - Xmin) / (Xpix - 1); + dY = binning * (Ymax - Ymin) / (Ypix - 1); CHECKVAL(Xmin); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c (revision 41170) @@ -13,4 +13,11 @@ int kapa; Graphdata graphmode; + + int binning = 1; + if ((N = get_argument (argc, argv, "-binning"))) { + remove_argument (N, &argc, argv); + binning = atoi(argv[N]); + remove_argument (N, &argc, argv); + } range = NULL; @@ -79,6 +86,9 @@ Ymax = graphmode.ymax; Ymin = graphmode.ymin; - dX = (Xmax - Xmin) / (Xpix - 1); - dY = (Ymax - Ymin) / (Ypix - 1); + + // (dX, dY) are the pixel scale of the output image (output pixels / input pixels) + + dX = binning * (Xmax - Xmin) / (Xpix - 1); + dY = binning * (Ymax - Ymin) / (Ypix - 1); CHECKVAL(Xmin); @@ -90,6 +100,6 @@ CHECKVAL(dY); - int Nx = (Xmax - Xmin) / dX + 1; - int Ny = (Ymax - Ymin) / dY + 1; + int Nx = abs((Xmax - Xmin) / dX) + 1; + int Ny = abs((Ymax - Ymin) / dY) + 1; Coords newcoords = graphmode.coords; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c (revision 41170) @@ -3,5 +3,5 @@ int region (int argc, char **argv) { - double Ra, Dec, Radius; + double Ra, Dec; float dx, dy; int N, kapa, NoClear, dXpix, dYpix; @@ -54,4 +54,26 @@ } + float XSIZE = NAN; + if ((N = get_argument (argc, argv, "-xsize"))) { + remove_argument (N, &argc, argv); + XSIZE = atof (argv[N]); + remove_argument (N, &argc, argv); + if (XSIZE <= 0) { + gprint (GP_ERR, "ERROR: xsize <= 0\n"); + return FALSE; + } + } + + float YSIZE = NAN; + if ((N = get_argument (argc, argv, "-ysize"))) { + remove_argument (N, &argc, argv); + YSIZE = atof (argv[N]); + remove_argument (N, &argc, argv); + if (YSIZE <= 0) { + gprint (GP_ERR, "ERROR: xsize <= 0\n"); + return FALSE; + } + } + if ((N = get_argument (argc, argv, "-ew"))) { remove_argument (N, &argc, argv); @@ -81,6 +103,11 @@ } - if ((argc != 4) && (argc != 5)) { - gprint (GP_ERR, "USAGE: region Ra Dec Radius [projection] [orientation]\n"); + int RadiusArg, CtypeArg; + if (!isnan(XSIZE) || !isnan(YSIZE)) { + RadiusArg = -1; + CtypeArg = 3; + if ((argc != 3) && (argc != 4)) { + gprint (GP_ERR, "USAGE: region Ra Dec [projection] [-xsize deg] [-ysize deg]\n"); + gprint (GP_ERR, " OR: region Ra Dec Radius [projection]\n"); gprint (GP_ERR, " [-image] [-ew] [+ew] [-ns] [+ns] [-no-clear] [-angle theta]\n"); gprint (GP_ERR, " current: %f %f (%f x %f) (%s)\n", @@ -90,28 +117,51 @@ &graphmode.coords.ctype[5]); return (FALSE); - } - + } + } else { + RadiusArg = 3; + CtypeArg = 4; + if ((argc != 4) && (argc != 5)) { + gprint (GP_ERR, "USAGE: region Ra Dec Radius [projection]\n"); + gprint (GP_ERR, " OR: region Ra Dec [projection] [-xsize deg] [-ysize deg]\n"); + gprint (GP_ERR, " [-image] [-ew] [+ew] [-ns] [+ns] [-no-clear] [-angle theta]\n"); + gprint (GP_ERR, " current: %f %f (%f x %f) (%s)\n", + graphmode.coords.crval1, graphmode.coords.crval2, + fabs(graphmode.xmax - graphmode.xmin), + fabs(graphmode.ymax - graphmode.ymin), + &graphmode.coords.ctype[5]); + return (FALSE); + } + } if (!ohana_str_to_radec (&Ra, &Dec, argv[1], argv[2])) return (FALSE); - Radius = atof (argv[3]); + + // region 0 0 sin -- should raise an error (radius = 0 or non-numeric) + + // I want to be able to support the old style call in which both of these were valid: + // region 0 0 90 ait + // region 0 0 90 <- uses existing, sticky projection type + // but I also want to be able to use + // region 0 0 ait -xsize 5 + // region 0 0 ait -ysize 5 + + double Radius = NAN; + if (RadiusArg >= 0) { + Radius = atof (argv[RadiusArg]); + } + InitCoords (&graphmode.coords, "DEC--TAN"); - - if (argc == 5) { - if (!strcasecmp (argv[4], "TAN")) - strcpy (graphmode.coords.ctype, "DEC--TAN"); - if (!strcasecmp (argv[4], "SIN")) - strcpy (graphmode.coords.ctype, "DEC--SIN"); - if (!strcasecmp (argv[4], "ARC")) - strcpy (graphmode.coords.ctype, "DEC--ARC"); - if (!strcasecmp (argv[4], "STG")) - strcpy (graphmode.coords.ctype, "DEC--STG"); - if (!strcasecmp (argv[4], "ZEA")) - strcpy (graphmode.coords.ctype, "DEC--ZEA"); - if (!strcasecmp (argv[4], "AIT")) - strcpy (graphmode.coords.ctype, "DEC--AIT"); - if (!strcasecmp (argv[4], "GLS")) - strcpy (graphmode.coords.ctype, "DEC--GLS"); - if (!strcasecmp (argv[4], "PAR")) - strcpy (graphmode.coords.ctype, "DEC--PAR"); - } + if (argc == CtypeArg + 1) { + if (!strcasecmp (argv[CtypeArg], "TAN")) { strcpy (graphmode.coords.ctype, "DEC--TAN"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "SIN")) { strcpy (graphmode.coords.ctype, "DEC--SIN"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "ARC")) { strcpy (graphmode.coords.ctype, "DEC--ARC"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "STG")) { strcpy (graphmode.coords.ctype, "DEC--STG"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "ZEA")) { strcpy (graphmode.coords.ctype, "DEC--ZEA"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "AIT")) { strcpy (graphmode.coords.ctype, "DEC--AIT"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "GLS")) { strcpy (graphmode.coords.ctype, "DEC--GLS"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "PAR")) { strcpy (graphmode.coords.ctype, "DEC--PAR"); goto got_ctype; } + gprint (GP_ERR, "ERROR: invalid projection type %s\n", argv[CtypeArg]); + gprint (GP_ERR, "allowed values: TAN, SIN, ARC, STG, ZEA, AIT, GLS, PAR\n"); + return FALSE; + } +got_ctype: graphmode.coords.crval1 = Ra; @@ -126,5 +176,6 @@ graphmode.coords.pc2_2 = cos(Angle*RAD_DEG)*pc2_2; - /* ask kapa for coordinate limits, to get the right aspect ratio */ + // ask kapa for coordinate limits, to get the right aspect ratio + // dx, dy are the size of the graph region in pixels KapaGetLimits (kapa, &dx, &dy); dx = fabs (dx); @@ -132,15 +183,38 @@ /* define limits for Ra, Dec at center, grid in degrees */ - if (dy < dx) { - graphmode.xmin = -(dx/dy)*Radius; - graphmode.ymin = -Radius; - graphmode.xmax = (dx/dy)*Radius; - graphmode.ymax = Radius; + if (RadiusArg >= 0) { + // force non-anamorphic projection with Radius set to smaller axis + if (dy < dx) { + graphmode.xmin = -(dx/dy)*Radius; + graphmode.ymin = -Radius; + graphmode.xmax = (dx/dy)*Radius; + graphmode.ymax = Radius; + } else { + graphmode.xmin = -Radius; + graphmode.ymin = -(dy/dx)*Radius; + graphmode.xmax = Radius; + graphmode.ymax = (dy/dx)*Radius; + } } else { - graphmode.xmin = -Radius; - graphmode.ymin = -(dy/dx)*Radius; - graphmode.xmax = Radius; - graphmode.ymax = (dy/dx)*Radius; - } + if (isnan(XSIZE)) { + graphmode.xmin = -(dx/dy)*YSIZE/2.0; + graphmode.ymin = -YSIZE/2.0; + graphmode.xmax = (dx/dy)*YSIZE/2.0; + graphmode.ymax = YSIZE/2.0; + } + if (isnan(YSIZE)) { + graphmode.xmin = -XSIZE/2.0; + graphmode.ymin = -(dy/dx)*XSIZE/2.0; + graphmode.xmax = XSIZE/2.0; + graphmode.ymax = (dy/dx)*XSIZE/2.0; + } + // anamorphic projection: + if (!isnan(XSIZE) && !isnan(YSIZE)) { + graphmode.xmin = -XSIZE/2.0; + graphmode.ymin = -YSIZE/2.0; + graphmode.xmax = XSIZE/2.0; + graphmode.ymax = YSIZE/2.0; + } + } set_variable ("XMIN", graphmode.xmin); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c (revision 41170) @@ -73,4 +73,6 @@ {1, "??", list_vars, "list variables *"}, {1, "#", nop, "a NOP function"}, + {1, "##", nop, "a NOP function"}, + {1, "###", nop, "a NOP function"}, {1, "local", local, "define local variables"}, {1, "macro", macro, "deal with the macros *"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c (revision 41170) @@ -5,4 +5,11 @@ if (argc < 2) goto usage; + + if (!strcasecmp (argv[1], "max-lines")) { + if (argc != 3) goto usage; + int MaxLines = atoi(argv[2]); + ohana_memdump_set_maxlines (MaxLines); + return TRUE; + } if (!strcasecmp (argv[1], "all")) { @@ -43,6 +50,13 @@ } + if (!strncasecmp ("strings", argv[1], strlen(argv[1]))) { + ohana_memdump_strings_file (stderr, FALSE); + return (TRUE); + } + usage: - gprint (GP_ERR, "USAGE: memory (all/leaks)\n"); + gprint (GP_ERR, "USAGE: memory (mode) [-max-lines]\n"); + gprint (GP_ERR, "USAGE: memory (max-lines) (N)\n"); + gprint (GP_ERR, " mode options: all, leaks, check, checkfree, variables, vectors, buffers, macros, commands, strings\n"); return (FALSE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh (revision 41170) @@ -320,2 +320,82 @@ end end + +# check memleaks +macro memtest1 + + local a b i N + + $i = 0 + $a = 1 + + memory check + output /dev/null + for i 0 1000 + if ($a == 1) + echo "run" + end + end + output stdout + memory check +end + +# check memleaks +macro memtest2 + + local a b i N + + $i = 0 + $a = 1 + $b = 2 + + memory check + output /dev/null + for i 0 1000 + if (($a == 1) && ($b == 2)) + echo "run" + end + end + output stdout + memory check +end + +# memory test using continue +macro memtest3 + + local a b i N + + $i = 0 + $a = 1 + $b = 2 + + memory check + output /dev/null + for i 0 1000 + if (($a == 1) && ($b == 2)) + continue + echo "run" + end + end + output stdout + memory check +end + +# memory test using continue +macro memtest4 + + local a b i N bool + + + $i = 0 + $a = 1 + $b = 2 + $bool = 0 + + memory check + output /dev/null + for i 0 1000 + $bool = (($a == 1) && ($b == 2)) + end + output stdout + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh (revision 41170) @@ -7,4 +7,13 @@ # Is the macro working? macro test_prep + $test_var1 = check1 + $test_var2 = check2 + $var_count = $0 +end + +# Is the macro working? +macro test_local + local test_var test_var2 var_count + $test_var1 = check1 $test_var2 = check2 @@ -47,2 +56,30 @@ end end + +# Memory Test for macro +macro memtest2 + + $i = 0 + test_prep + + memory check + for i 0 10000 + test_prep + end + memory check + +end + +# Memory Test for macro +macro memtest3 + + $i = 0 + test_local + + memory check + for i 0 10000 + test_local + end + memory check + +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile (revision 41170) @@ -19,4 +19,5 @@ $(SRC)/init.$(ARCH).o \ $(SRC)/accum.$(ARCH).o \ +$(SRC)/antialias.$(ARCH).o \ $(SRC)/applyfit1d.$(ARCH).o \ $(SRC)/applyfit2d.$(ARCH).o \ @@ -78,4 +79,5 @@ $(SRC)/join.$(ARCH).o \ $(SRC)/jpeg.$(ARCH).o \ +$(SRC)/kapamemory.$(ARCH).o \ $(SRC)/kern.$(ARCH).o \ $(SRC)/keyword.$(ARCH).o \ @@ -89,8 +91,10 @@ $(SRC)/lookup.$(ARCH).o \ $(SRC)/matrix.$(ARCH).o \ +$(SRC)/match1d.$(ARCH).o \ $(SRC)/match2d.$(ARCH).o \ $(SRC)/mkrgb.$(ARCH).o \ $(SRC)/mcreate.$(ARCH).o \ $(SRC)/medacc.$(ARCH).o \ +$(SRC)/mgaussdev.$(ARCH).o \ $(SRC)/mget.$(ARCH).o \ $(SRC)/mget3d.$(ARCH).o \ @@ -99,4 +103,5 @@ $(SRC)/medimage.$(ARCH).o \ $(SRC)/medimage_commands.$(ARCH).o \ +$(SRC)/medimage_calc.$(ARCH).o \ $(SRC)/mset.$(ARCH).o \ $(SRC)/needles.$(ARCH).o \ @@ -169,4 +174,5 @@ $(SRC)/vgauss.$(ARCH).o \ $(SRC)/vlorentz.$(ARCH).o \ +$(SRC)/vsigmoid.$(ARCH).o \ $(SRC)/vellipse.$(ARCH).o \ $(SRC)/vmaxwell.$(ARCH).o \ @@ -176,4 +182,5 @@ $(SRC)/vzload.$(ARCH).o \ $(SRC)/vpeaks.$(ARCH).o \ +$(SRC)/vtransitions.$(ARCH).o \ $(SRC)/vpop.$(ARCH).o \ $(SRC)/vroll.$(ARCH).o \ @@ -182,4 +189,6 @@ $(SRC)/vsmooth.$(ARCH).o \ $(SRC)/vstats.$(ARCH).o \ +$(SRC)/virls.$(ARCH).o \ +$(SRC)/vwtmean.$(ARCH).o \ $(SRC)/xsection.$(ARCH).o \ $(SRC)/vsh.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c (revision 41170) @@ -0,0 +1,26 @@ +# include "data.h" + +int antialias (int argc, char **argv) { + + int N, kapa; + Graphdata graphmode; + + char *name = NULL; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + name = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetGraph (&graphmode, &kapa, name)) return (FALSE); + FREE (name); + + if (argc != 2) { + gprint (GP_ERR, "USAGE: antialias (sigma)\n"); + return (FALSE); + } + + float sigma = atof (argv[1]); + + KapaSetSmoothSigma (kapa, sigma); + return (TRUE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c (revision 41170) @@ -1,10 +1,71 @@ # include "data.h" -enum {SUM, MEAN, MEDIAN}; +enum {SUM, MEAN, MEDIAN, INNER, NGOOD}; + +double cutstat (float *value, int Nvalue, int mode) { + + double output = 0.0; + + if (Nvalue == 0) return NAN; + + if (mode == MEDIAN) { + fsort (value, Nvalue); + if (Nvalue % 2) { + int Ncenter = Nvalue / 2; + myAssert ((Ncenter >= 0) && (Ncenter < Nvalue), "cutstat"); + output = value[Ncenter]; + } else { + int Ncenter = Nvalue / 2 - 1; + myAssert ((Ncenter >= 0) && (Ncenter < Nvalue - 1), "cutstat"); + output = 0.5*(value[Ncenter] + value[Ncenter + 1]); + } + return output; + } + if (mode == INNER) { + fsort (value, Nvalue); + + int Ns = 0, Ne = 0; + if (Nvalue % 2) { + // for an odd number of points take the same number below + // and above the center value + int Ncenter = Nvalue / 2; + int Nquarter = 0.25*Nvalue; + Ns = Ncenter - Nquarter; + Ne = Ncenter + Nquarter; + } else { + // for an even number, the middle lies between two points + // take the same number below as above + int Ncenter = (int)(Nvalue / 2) - 1; + int Nquarter = 0.25*Nvalue; + Ns = Ncenter - Nquarter; + Ne = Ncenter + 1 + Nquarter; + } + int Nv = 0; + for (int i = Ns; i <= Ne; i++) { + myAssert ((i >= 0) && (i < Nvalue), "cutstat"); + output += value[i]; + Nv ++; + } + output /= Nv; + return output; + } + if (mode == NGOOD) { + return Nvalue; + } + if ((mode == MEAN) || (mode == SUM)) { + for (int i = 0; i < Nvalue; i++) { + myAssert ((i >= 0) && (i < Nvalue), "cutstat"); + output += value[i]; + } + if (mode == MEAN) { output /= Nvalue; } + return output; + } + return NAN; +} int cut (int argc, char **argv) { int i, j, N, Nx, Ny, Mode; - float *Vin, *Vbuf, value; + float *Vin, *Vbuf; int sx, sy, nx, ny; Vector *xvec, *yvec; @@ -12,4 +73,8 @@ Mode = SUM; + if ((N = get_argument (argc, argv, "-sum"))) { + remove_argument (N, &argc, argv); + Mode = SUM; + } if ((N = get_argument (argc, argv, "-median"))) { remove_argument (N, &argc, argv); @@ -19,4 +84,12 @@ remove_argument (N, &argc, argv); Mode = MEAN; + } + if ((N = get_argument (argc, argv, "-inner"))) { + remove_argument (N, &argc, argv); + Mode = INNER; + } + if ((N = get_argument (argc, argv, "-ngood"))) { + remove_argument (N, &argc, argv); + Mode = NGOOD; } @@ -36,6 +109,7 @@ Ny = buf[0].matrix.Naxis[1]; - if ((sx < 0) || (sy < 0) || (sx+nx > Nx) || (sy+ny > Ny)) { - gprint (GP_ERR, "region out of range\n"); + // ny & nx do not need to be constrained by the buffer, but they do need to be sensible + if ((nx < 0) || (ny < 0) || (nx > 1e8) || (ny > 1e8)) { + gprint (GP_ERR, "warning : extraction size is crazy: %d,%d\n", nx, ny); return (FALSE); } @@ -57,21 +131,31 @@ for (i = 0; i < nx; i++) { + // for out-of-range areas, set the output pixels to NAN + if (i + sx < 0) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + if (i + sx >= Nx) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + /* accumulate values */ + // skip out-of-range areas in y + if (sy < 0) { + ny += sy; + sy = 0; + } + if (sy > Ny) sy = Ny; + if (sy + ny >= Ny) ny = Ny - sy; + Vin = (float *)(buf[0].matrix.buffer) + sy*Nx + sx + i; + int Npix = 0; for (j = 0; j < ny; j++, Vin += Nx) { - Vbuf[j] = *Vin; - } - /* apply stat of choice */ - if (Mode == MEDIAN) { - fsort (Vbuf, ny); - value = Vbuf[(int)(0.5*ny)]; - } else { - value = 0; - for (j = 0; j < ny; j++) { - value += Vbuf[j]; - } - if (Mode == MEAN) { value /= ny; } - } - yvec[0].elements.Flt[i] = value; + if (!isfinite(*Vin)) continue; + Vbuf[Npix] = *Vin; + Npix ++; + } + yvec[0].elements.Flt[i] = cutstat (Vbuf, Npix, Mode); } free (Vbuf); @@ -89,21 +173,31 @@ for (i = 0; i < ny; i++) { + // for out-of-range areas, set the output pixels to NAN + if (i + sy < 0) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + if (i + sy >= Ny) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + /* accumulate values */ + // skip out-of-range areas in x + if (sx < 0) { + nx += sx; + sx = 0; + } + if (sx > Nx) sx = Nx; + if (sx + nx >= Nx) nx = Nx - sx; + Vin = (float *)(buf[0].matrix.buffer) + (sy + i)*Nx + sx; + int Npix = 0; for (j = 0; j < nx; j++, Vin ++) { - Vbuf[j] = *Vin; - } - /* apply stat of choice */ - if (Mode == MEDIAN) { - fsort (Vbuf, nx); - value = Vbuf[(int)(0.5*nx)]; - } else { - value = 0; - for (j = 0; j < nx; j++) { - value += Vbuf[j]; - } - if (Mode == MEAN) { value /= nx; } - } - yvec[0].elements.Flt[i] = value; + if (!isfinite(*Vin)) continue; + Vbuf[Npix] = *Vin; + Npix ++; + } + yvec[0].elements.Flt[i] = cutstat (Vbuf, Npix, Mode); } free (Vbuf); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c (revision 41170) @@ -1,11 +1,25 @@ # include "data.h" + +/* : source image, must exist + : target image -- if it does not exist, it is created of size (Nx,Ny) + sx, sy : source starting coordinate -- need not be on a valid image pixel + nx, ny : number of source pixels -- currently must not go out of bounds -> allow out-of-bounds + Sx, Sy : target starting coordinate -- need not be on a valid image pixel + Nx, Ny : redundant information UNLESS does exist (in which case it is required information) -> make Nx,Ny optional if exists? + */ int extract (int argc, char **argv) { - int i, j; - float *Vin, *Vout; - int sx, sy, nx, ny, NX, NY; - int Sx, Sy, Nx, Ny; + int N; Buffer *in, *out; + + float initValue = 0.0; + int initOutput = FALSE; + if ((N = get_argument (argc, argv, "-init"))) { + remove_argument (N, &argc, argv); + initValue = atof (argv[N]); + remove_argument (N, &argc, argv); + initOutput = TRUE; + } if (argc != 11) { @@ -15,22 +29,24 @@ if ((in = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - NX = in[0].matrix.Naxis[0]; - NY = in[0].matrix.Naxis[1]; + int NX = in[0].matrix.Naxis[0]; + int NY = in[0].matrix.Naxis[1]; - sx = atof (argv[3]); - sy = atof (argv[4]); - nx = atof (argv[5]); - ny = atof (argv[6]); + int sx = atof (argv[3]); + int sy = atof (argv[4]); + int nx = atof (argv[5]); + int ny = atof (argv[6]); - Sx = atof (argv[7]); - Sy = atof (argv[8]); - Nx = atof (argv[9]); - Ny = atof (argv[10]); + int Sx = atof (argv[7]); + int Sy = atof (argv[8]); + int Nx = atof (argv[9]); + int Ny = atof (argv[10]); if ((Sy + ny > Ny) || (Sx + nx > Nx)) { - gprint (GP_ERR, "mismatch between source and dest regions\n"); + gprint (GP_ERR, "source pixels extend beyond target pixels\n"); gprint (GP_ERR, "%d + %d > %d or %d + %d > %d\n", Sy, ny, Ny, Sx, nx, Nx); - return (FALSE); + // return (FALSE); } + + // XXX : allow source region to fall outside source image /* region is not on first image */ @@ -39,14 +55,10 @@ (sy > in[0].matrix.Naxis[1])) { gprint (GP_ERR, "region outside of source image\n"); - return (FALSE); + // return (FALSE); } - if ((Sx + nx > Nx) || (Sy + ny > Ny)) { - gprint (GP_ERR, "source region larger than dest region\n"); - return (FALSE); - } if ((Sx < 0) || (Sy < 0)) { gprint (GP_ERR, "dest region out of range\n"); - return (FALSE); + // return (FALSE); } @@ -76,13 +88,61 @@ } - for (j = 0; j < ny; j++) { - if (j + sy < 0) continue; - if (j + sy >= NY) continue; - Vin = (float *)(in[0].matrix.buffer) + (j + sy)*in[0].matrix.Naxis[0] + sx; - Vout = (float *)(out[0].matrix.buffer) + (j + Sy)*out[0].matrix.Naxis[0] + Sx; - for (i = 0; i < nx; i++, Vin++, Vout++) { + // (NX,NY) : source image dimensions + // (Nx,Ny) : target image dimensions + + // allow (sx, sy), (Sx, Sy) to be off image + // allow (sx+nx, sy+ny) to be off image + + // if (sx < 0) { + // nx += sx; + // sx = 0; + // } + // if (sx > NX) sx = NX; + // + // if (Sx < 0) { + // Nx += sx; + // sx = 0; + // } + // if (sx > NX) sx = NX; + + float *Vin = (float *)(in[0].matrix.buffer); + float *Vout = (float *)(out[0].matrix.buffer); + + if (initOutput) { + for (int j = 0; j < Ny; j++) { + for (int i = 0; i < Nx; i++) { + Vout[i + Nx*j] = initValue; + } + } + } + + int Nps = NX*NY; + int Npt = Nx*Ny; + + for (int j = 0; j < ny; j++) { + if (j + sy < 0) continue; // not yet on source image + if (j + Sy < 0) continue; // not yet on target image + if (j + sy >= NY) continue; // past edge of source image + if (j + Sy >= Ny) continue; // past edge of target image + + // Vin = (float *)(in[0].matrix.buffer) + (j + sy)*in[0].matrix.Naxis[0] + sx; + // Vout = (float *)(out[0].matrix.buffer) + (j + Sy)*out[0].matrix.Naxis[0] + Sx; + + for (int i = 0; i < nx; i++) { if (i + sx < 0) continue; if (i + sx >= NX) continue; - *Vout = *Vin; + if (i + Sx < 0) continue; + if (i + Sx >= Nx) continue; + + int ps = i + sx + (j + sy)*NX; + int pt = i + Sx + (j + Sy)*Nx; + + myAssert (pt >= 0, "oops 1"); + myAssert (pt < Npt, "oops 2"); + + myAssert (ps >= 0, "oops 3"); + myAssert (ps < Nps, "oops 4"); + + Vout[pt] = Vin[ps]; } } @@ -91,3 +151,2 @@ } - Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c (revision 41170) @@ -3,4 +3,5 @@ int accum PROTO((int, char **)); +int antialias PROTO((int, char **)); int applyfit PROTO((int, char **)); int applyfit1d PROTO((int, char **)); @@ -67,4 +68,5 @@ int join PROTO((int, char **)); int jpeg PROTO((int, char **)); +int kapamemory PROTO((int, char **)); int kern PROTO((int, char **)); int keyword PROTO((int, char **)); @@ -79,8 +81,10 @@ int lookup PROTO((int, char **)); int matrix PROTO((int, char **)); +int match1d PROTO((int, char **)); int match2d PROTO((int, char **)); int mkrgb PROTO((int, char **)); int mcreate PROTO((int, char **)); int medacc PROTO((int, char **)); +int mgaussdev PROTO((int, char **)); int mget PROTO((int, char **)); int mget3d PROTO((int, char **)); @@ -156,4 +160,5 @@ int vgauss PROTO((int, char **)); int vlorentz PROTO((int, char **)); +int vsigmoid PROTO((int, char **)); int vellipse PROTO((int, char **)); int vmaxwell PROTO((int, char **)); @@ -163,7 +168,11 @@ int vzload PROTO((int, char **)); int vstats PROTO((int, char **)); +int vstats PROTO((int, char **)); +int virls PROTO((int, char **)); +int vwtmean PROTO((int, char **)); int vroll PROTO((int, char **)); int vshift PROTO((int, char **)); int vpeaks PROTO((int, char **)); +int vtransitions PROTO((int, char **)); int vpop PROTO((int, char **)); int vsmooth PROTO((int, char **)); @@ -187,4 +196,5 @@ static Command cmds[] = { {1, "accum", accum, "accumulate vector values in another vector"}, + {1, "antialias", antialias, "set anti-alias sigma value for display"}, {1, "applyfit", applyfit1d, "apply 1-d fit to new vector"}, {1, "applyfit1d", applyfit1d, "apply 1-d fit to new vector"}, @@ -255,4 +265,5 @@ {1, "join", join, "find the join of two ID vectors"}, {1, "jpeg", jpeg, "convert display image to JPEG"}, + {1, "kapamemory", kapamemory, "manage kapa memory dump options"}, {1, "kern", kern, "convolve with 3x3 kernel"}, {1, "keyword", keyword, "extract a FITS keyword from image header"}, @@ -265,4 +276,5 @@ {1, "imcreate", mcreate, "create an image"}, {1, "medacc", medacc, "accumulate vector values in another vector"}, + {1, "mgaussdev", mgaussdev, "generate a gaussian deviate image"}, {1, "mget", mget, "extract a vector from an image"}, {1, "mget3d", mget3d, "extract a vector from a 3D image"}, @@ -273,4 +285,5 @@ {1, "medimage", medimage_command, "median image manipulation"}, {1, "matrix", matrix, "matrix math operations"}, + {1, "match1d", match1d, "match 2 vectors and return matched indexes"}, {1, "match2d", match2d, "match 2 pairs of X,Y vectors and return matched indexes"}, {1, "mkrgb", mkrgb, "convert 3 images to rgb jpeg (use Kapa for better control)"}, @@ -347,4 +360,5 @@ {1, "vgauss", vgauss, "fit a Gaussian to a vector"}, {1, "vlorentz", vlorentz, "fit a Lorentzian to a vector"}, + {1, "vsigmoid", vsigmoid, "fit a Sigmoid to a vector"}, {1, "vellipse", vellipse, "fit a Ellipse to a vector pair"}, {1, "vgrid", vgrid, "generate an image from a triplet of vectors"}, @@ -354,5 +368,6 @@ {1, "vload", vload, "load vectors as overlay on image display"}, {1, "vmaxwell", vmaxwell, "fit a Maxwellian to a vector"}, - {1, "vpeaks", vpeaks, "fine coord and flux of peaks in vector"}, + {1, "vpeaks", vpeaks, "find coord and flux of peaks in vector"}, + {1, "vtransitions", vtransitions, "find points in vector that cross the transition value"}, {1, "vpop", vpop, "remove first element of a vector"}, {1, "vroll", vroll, "roll vector elements by 1 entry"}, @@ -360,4 +375,6 @@ {1, "vsmooth", vsmooth, "Gaussian smooth of a vector"}, {1, "vstats", vstats, "statistics on a vector"}, + {1, "vwtmean", vwtmean, "weighted mean of a vector"}, + {1, "virls", virls, "IRLS mean of a vector"}, {1, "vzload", vzload, "load vectors as overlay on image display (scaled points)"}, {1, "vsh", vsh, "Vector Spherical Harmonics"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c (revision 41170) @@ -7,4 +7,11 @@ double x0, dx, dy, y0; Vector *xout, *yout, *xin, *yin; + + int N; + int FillEnds = FALSE; + if ((N = get_argument (argc, argv, "-fill-ends"))) { + FillEnds = TRUE; + remove_argument (N, &argc, argv); + } /** check basic syntax **/ @@ -36,10 +43,12 @@ y0 = yin[0].elements.Flt[0]; - /* in and out vectors are sorted */ - j = 0; + // fill in the start with NANs + for (i = 0; (i < xout[0].Nelements) && (xout[0].elements.Flt[i] < xin[0].elements.Flt[0]); i++) { + yout[0].elements.Flt[i] = FillEnds ? yin[0].elements.Flt[0] : NAN; + } // every output pixel should get a value unless // we are below the lowest in or above the highest in - for (i = 0, j = 0; (i < xout[0].Nelements) && (j < xin[0].Nelements - 1); ) { + for (j = 0; (i < xout[0].Nelements) && (j < xin[0].Nelements - 1); ) { yout[0].elements.Flt[i] = NAN; @@ -66,10 +75,10 @@ // fill in the rest with NANs + int NinLast = xin[0].Nelements - 1; while (i < yout[0].Nelements) { - yout[0].elements.Flt[i] = NAN; + yout[0].elements.Flt[i] = FillEnds ? yin[0].elements.Flt[NinLast] : NAN; i++; } return (TRUE); - } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c (revision 41170) @@ -0,0 +1,56 @@ +# include "data.h" + +int kapamemory (int argc, char **argv) { + + int N, kapa; + + /* display source */ + char *name = NULL; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + name = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetImage (NULL, &kapa, name)) return (FALSE); + FREE (name); + + if (argc < 2) { + gprint (GP_ERR, "USAGE: kapamemory (mode)\n"); + gprint (GP_ERR, " mode = dump, nlines, exit\n"); + return (FALSE); + } + + if (!strcasecmp(argv[1], "dump")) { + KapaMemoryDump (kapa); + return TRUE; + } + + if (!strcasecmp(argv[1], "exit")) { + if (argc != 3) goto exit_usage; + if (!strcasecmp(argv[2], "on")) { + KapaMemoryDumpOnExit (kapa, TRUE); + return TRUE; + } + if (!strcasecmp(argv[2], "off")) { + KapaMemoryDumpOnExit (kapa, FALSE); + return TRUE; + } + exit_usage: + gprint (GP_ERR, "USAGE: kapamemory exit (status)\n"); + gprint (GP_ERR, " statue = 'on' or 'off'\n"); + return FALSE; + } + + if (!strcasecmp(argv[1], "nlines")) { + if (argc != 3) { + gprint (GP_ERR, "USAGE: kapamemory nlines (N)\n"); + return FALSE; + } + int Nlines = atoi (argv[2]); + KapaMemoryDumpLines (kapa, Nlines); + return TRUE; + } + gprint (GP_ERR, "ERROR: unknown kapamemory mode %s\n", argv[1]); + return FALSE; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c (revision 41170) @@ -0,0 +1,248 @@ +# include "data.h" + +int find_matches1d (Vector *X1, Vector *X2, double Radius, Vector *index1, Vector *index2, Vector *radiusMatch); +int find_matches1d_closest (Vector *X1, Vector *X2, double Radius, Vector *index); + +// match1d (X1) (X2) (Radius) [-index1 (index1)] [-index2 (index2)] [-nomatch1 nomatch1] [-nomatch2 nomatch2] +int match1d (int argc, char **argv) { + + int N, CLOSEST; + double Radius; + char *endptr; + Vector *X1vec, *X2vec; + Vector *index1, *index2; + + if ((N = get_argument (argc, argv, "-h"))) goto usage; + if ((N = get_argument (argc, argv, "--help"))) goto usage; + + CLOSEST = FALSE; + if ((N = get_argument (argc, argv, "-closest"))) { + remove_argument (N, &argc, argv); + CLOSEST = TRUE; + } + + if ((N = get_argument (argc, argv, "-index1"))) { + remove_argument (N, &argc, argv); + if ((index1 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index1 = SelectVector ("index1", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + if ((N = get_argument (argc, argv, "-index2"))) { + remove_argument (N, &argc, argv); + if ((index2 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index2 = SelectVector ("index2", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + Vector *radiusMatch = NULL; + if ((N = get_argument (argc, argv, "-radius"))) { + if (CLOSEST) { + gprint (GP_ERR, "error: -radius and -closest are currently incompatible\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); + if ((radiusMatch = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 4) { + gprint (GP_ERR, "USAGE: match1d X1 X2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); + gprint (GP_ERR, " use -h or --help for more detail\n"); + return (FALSE); + } + + /* + we have two modes of operation: + + without -closest, we are finding all matched pairs within the match radius. in this + case, the two index vectors have the same length, one entry per matched pair. + x1[index1],y1[index1] matches to x2[index2],y2[index2]. + + with -closest selected, we are finding the closest element of set 1 to each of set 2 + and vice versa. in this case, index1 is always the same length as x1,y1, while index2 + is the same lengths as x2,y2. x2[index1],y2[index1] matches x1,y1 while + x1[index2],y1[index2] matches x2,y2 + + */ + + if ((X1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((X2vec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + REQUIRE_VECTOR_FLT (X1vec, FALSE); + REQUIRE_VECTOR_FLT (X2vec, FALSE); + + Radius = strtod (argv[3], &endptr); + if (*endptr) { + gprint (GP_ERR, "Radius must be numerical (%s)\n", argv[3]); + return (FALSE); + } + + if (CLOSEST) { + find_matches1d_closest (X1vec, X2vec, Radius, index1); + find_matches1d_closest (X2vec, X1vec, Radius, index2); + } else { + find_matches1d (X1vec, X2vec, Radius, index1, index2, radiusMatch); + } + return (TRUE); + +usage: + gprint (GP_ERR, "we have two modes of operation:\n\n"); + + gprint (GP_ERR, "without -closest, we are finding all matched pairs within the match radius. in this\n"); + gprint (GP_ERR, "case, the two index vectors have the same length, one entry per matched pair.\n"); + gprint (GP_ERR, "x1[index1] matches to x2[index2].\n\n"); + + gprint (GP_ERR, "with -closest selected, we are finding the closest element of set 1 to each of set 2\n"); + gprint (GP_ERR, "and vice versa. in this case, index1 is always the same length as x1, while index2\n"); + gprint (GP_ERR, "is the same lengths as x2. x2[index1] matches x1 while\n"); + gprint (GP_ERR, "x1[index2] matches x2\n\n"); + + gprint (GP_ERR, "if -index1 or -index2 is not supplied, the vectors are created with names index1 or index2\n"); + gprint (GP_ERR, "use 'reindex' to generate new vectors based on these index vectors\n"); + + gprint (GP_ERR, "if -radius (vector) is supplied, the vector will be filled with the distance between the matched pairs\n"); + gprint (GP_ERR, " not valid with -closest\n"); + return FALSE; +} + +// Radius is a 1-D separation +int find_matches1d (Vector *X1, Vector *X2, double Radius, Vector *index1, Vector *index2, Vector *radiusMatch) { + + off_t i, j, first_j, I, J, *N1, *N2, Nmatch, NMATCH, DMATCH; + double dX, dR; + + NMATCH = MAX(MAX(0.01*X1->Nelements, 0.01*X2->Nelements), 100); + DMATCH = NMATCH; + + ResetVector (index1, OPIHI_INT, NMATCH); + ResetVector (index2, OPIHI_INT, NMATCH); + if (radiusMatch) ResetVector (radiusMatch, OPIHI_FLT, NMATCH); + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + dsort_indexonly (X1->elements.Flt, N1, X1->Nelements); + dsort_indexonly (X2->elements.Flt, N2, X2->Nelements); + + Nmatch = 0; + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + if (!isfinite(X1->elements.Flt[I])) { i++; continue; } + if (!isfinite(X2->elements.Flt[J])) { j++; continue; } + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { i++; continue; } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for all matches of list2() to list1(i) + first_j = j; + for (j = first_j; (dX > -1.02*Radius) && (j < X2->Nelements); j++) { + J = N2[j]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dR = fabs(dX); + if (dR < Radius) { + index1->elements.Int[Nmatch] = I; + index2->elements.Int[Nmatch] = J; + if (radiusMatch) radiusMatch->elements.Flt[Nmatch] = dR; + + // XXX track matches 1 and 2 with internal vector, save new nomatch index vectors + // after this loop + + Nmatch ++; + if (Nmatch >= NMATCH) { + NMATCH += DMATCH; + REALLOCATE (index1->elements.Int, opihi_int, NMATCH); + REALLOCATE (index2->elements.Int, opihi_int, NMATCH); + if (radiusMatch) { REALLOCATE (radiusMatch->elements.Flt, opihi_flt, NMATCH); } + } + } + } + j = first_j; + i++; + } + index1->Nelements = Nmatch; + index2->Nelements = Nmatch; + if (radiusMatch) radiusMatch->Nelements = Nmatch; + + free (N1); + free (N2); + + return (TRUE); +} + +// find the elements of X2,Y2 which are closest to each element of X1,Y1 (-1 if no match) +int find_matches1d_closest (Vector *X1, Vector *X2, double Radius, Vector *index) { + + off_t i, j, Jmin, Ji, I, J, *N1, *N2, NMATCH; + double dX, dR, Rmin; + + NMATCH = X1->Nelements; + ResetVector (index, OPIHI_INT, NMATCH); + + for (i = 0; i < index->Nelements; i++) { index->elements.Int[i] = -1; } + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + dsort_indexonly (X1->elements.Flt, N1, X1->Nelements); + dsort_indexonly (X2->elements.Flt, N2, X2->Nelements); + + // find the closest entry in list 2 to the current entry in list 1: + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { + // no match in list 2 to this entry + index->elements.Int[I] = -1; + i++; + continue; + } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for closest matches of list2() to list1(i) + Jmin = -1; + for (Ji = j; (dX > -1.02*Radius) && (Ji < X2->Nelements); Ji++) { + J = N2[Ji]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dR = fabs(dX); + if (dR > Radius) continue; + if (dR < Rmin) { + Rmin = dR; + Jmin = J; + } + } + + // no match in list 2 to this entry + if (Jmin == -1) { + index->elements.Int[I] = -1; + i++; + continue; + } + index->elements.Int[I] = Jmin; + i++; + } + index->Nelements = NMATCH; + + free (N1); + free (N2); + + return (TRUE); +} + + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c (revision 41170) @@ -0,0 +1,365 @@ +# include "data.h" + +float weight_cauchy_square_flt (float x2); +float irls_mean (float *val, float *wgt, int N, float *outvar, int *Npt); +float irls_fraction_interpolate (float *values, float fraction, int Npts); +float irls_robust_stdev (float *values, int Npts); +float irls_wtmean (float *val, float *wgt, int *idx, int Npt, float *variance); +void irls_bootstrap (int *idx, int Npts); +void irls_init (int N); +void irls_free (void); + +# define IRLS_TOLERANCE 1e-4 +int BOOTSTRAP = FALSE; +int BOOTSTRAP_NITER = 100; + +static float *irls_valsub = NULL; +static float *irls_wgtsub = NULL; +static int *irls_idx = NULL; +static float *irls_testval = NULL; + +enum {CALC_MEDIAN, CALC_MEAN, CALC_IRLS, CALC_WTMEAN}; + +int medimage_calc (int argc, char **argv) { + + int ix, iy, n, N; + Buffer *output; + + BOOTSTRAP = FALSE; + if ((N = get_argument (argc, argv, "-bootstrap"))) { + BOOTSTRAP = TRUE; + remove_argument (N, &argc, argv); + } + BOOTSTRAP_NITER = 100; + if ((N = get_argument (argc, argv, "-bootstrap-iter"))) { + remove_argument (N, &argc, argv); + BOOTSTRAP_NITER = atoi (argv[N]); + remove_argument (N, &argc, argv); + BOOTSTRAP = TRUE; + } + + int mode = CALC_MEDIAN; + if ((N = get_argument (argc, argv, "-mean"))) { + mode = CALC_MEAN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-irls"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_IRLS; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-wtmean"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_WTMEAN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-median"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_MEDIAN; + remove_argument (N, &argc, argv); + } + + Buffer *variance = NULL; + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((variance = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage calc (name) (output) [-mean,-irls,-wtmean,-median] [-variance output] [-bootstrap] [-bootstrap-iter (N)]\n"); + gprint (GP_ERR, " calculate the median image for the median image set\n"); + gprint (GP_ERR, " This function calculates the 'average' image for the supplied set of images.\n"); + gprint (GP_ERR, " Four options are available to calculate the average:\n"); + gprint (GP_ERR, " -mean (straight arithmetic mean)\n"); + gprint (GP_ERR, " -median\n"); + gprint (GP_ERR, " -wtmean (arithmetic mean, weighted by supplied variances)\n"); + gprint (GP_ERR, " -irls (iteratively-reweighted least-squares)\n"); + gprint (GP_ERR, " If -variance is supplied the returned image represents the estimate of the variance on the average image\n"); + gprint (GP_ERR, " For the four average methods above, the sqrt of the variance has the following meanings:\n"); + gprint (GP_ERR, " -mean : standard deviation / sqrt(Npts)\n"); + gprint (GP_ERR, " -median : standard deviation / sqrt(Npts)\n"); + gprint (GP_ERR, " -wtmean : formal error on the weighted mean [sum (1 / variances)]\n"); + gprint (GP_ERR, " -irls : formal error on the pixels used (excluding ~3 sigma outliers)\n"); + gprint (GP_ERR, " note that the irls formal error slightly underestimates the observed standard deviation\n"); + return FALSE; + } + + MedImageType *median = FindMedImage (argv[1]); + if (!median) { + gprint (GP_ERR, "median image %s not found\n", argv[1]); + return FALSE; + } + + if ((output = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) return (FALSE); + + int Ninput = median->Ninput; + int Nx = median->Nx; + int Ny = median->Ny; + + ALLOCATE_PTR (val, float, Ninput); + ALLOCATE_PTR (wgt, float, Ninput); + + if (mode == CALC_IRLS) irls_init (Ninput); + + gfits_free_matrix (&output->matrix); + gfits_free_header (&output->header); + if (!CreateBuffer (output, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + + float *outvalue = (float *) output->matrix.buffer; + float *varvalue = NULL; + if (variance) { + gfits_free_matrix (&variance->matrix); + gfits_free_header (&variance->header); + if (!CreateBuffer (variance, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + + varvalue = (float *) variance->matrix.buffer; + } + + // save the number of points per pixel + Buffer *nptbuf = SelectBuffer ("irls_npt", ANYBUFFER, TRUE); + gfits_free_matrix (&nptbuf->matrix); + gfits_free_header (&nptbuf->header); + if (!CreateBuffer (nptbuf, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + float *NptVal = (float *) nptbuf->matrix.buffer; + + for (iy = 0; iy < Ny; iy++) { + for (ix = 0; ix < Nx; ix++) { + + int N = 0; + int Npix = ix + Nx*iy; + for (n = 0; n < Ninput; n++) { + float v = median->flx[n][Npix]; + if (!isfinite(v)) continue; + val[N] = v; + wgt[N] = 1.0; + if (median->var[n]) { + float s = median->var[n][Npix]; + if (!isfinite(s)) continue; + if (fabs(s) < 2*FLT_MIN) s = 2*FLT_MIN; + wgt[N] = 1.0 / s; + } + N++; + } + if (N == 0) continue; + + switch (mode) { + case CALC_MEDIAN: + fsort (val, N); + outvalue[Npix] = val[(int)(0.5*N)]; + if (varvalue) { + float sum = 0.0; + for (n = 0; n < N; n++) { + sum += SQ(val[n] - outvalue[Npix]); + } + // variance on the mean (stdev / sqrt(N))^2 + varvalue[Npix] = sum / (N - 1) / N; + } + break; + case CALC_MEAN: { + float sum = 0.0; + for (n = 0; n < N; n++) { + sum += val[n]; + } + outvalue[Npix] = sum / (float) N; + + if (varvalue) { + float Sum = 0.0; + for (n = 0; n < N; n++) { + Sum += SQ(val[n] - outvalue[Npix]); + } + // variance on the mean (stdev / sqrt(N))^2 + varvalue[Npix] = Sum / (N - 1) / N; + } + break; + } + case CALC_WTMEAN: { + float variance; + outvalue[Npix] = irls_wtmean (val, wgt, NULL, N, &variance); + if (varvalue) { varvalue[Npix] = variance; } + break; + } + case CALC_IRLS: + if (varvalue) { + int Npts = 0; + outvalue[Npix] = irls_mean (val, wgt, N, &varvalue[Npix], &Npts); + NptVal[Npix] = Npts; + } else { + int Npts = 0; + outvalue[Npix] = irls_mean (val, wgt, N, NULL, &Npts); + NptVal[Npix] = Npts; + } + } + } + } + irls_free(); + return TRUE; +} + +// allocate temporary vectors +void irls_init (int N) { + + ALLOCATE (irls_valsub, float, N); + ALLOCATE (irls_wgtsub, float, N); + ALLOCATE (irls_idx, int, N); + ALLOCATE (irls_testval, float, BOOTSTRAP_NITER); +} + +// free temporary vectors +void irls_free (void) { + FREE (irls_valsub); + FREE (irls_wgtsub); + FREE (irls_idx); + FREE (irls_testval); + irls_valsub = NULL; + irls_wgtsub = NULL; + irls_idx = NULL; + irls_testval = NULL; +} + +float irls_mean (float *val, float *wgt, int N, float *outvar, int *Npt) { + + // calculate weighted mean + float Value = irls_wtmean (val, wgt, NULL, N, NULL); + + int converged = FALSE; + for (int i = 0; (i < 10) && !converged; i++) { + float ValueLast = Value; + float S1 = 0.0, S2 = 0.0; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int n = 0; n < N; n++) { + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + S1 += Mod * wgt[n] * val[n]; + S2 += Mod * wgt[n]; + } + Value = S1 / S2; + + float delta = fabs(Value - ValueLast); + if (delta < Value * IRLS_TOLERANCE) converged = TRUE; + } + + if (outvar) { + if (BOOTSTRAP) { + // generate a subset vector of just the accepted points + + // calculate stdev of high-weight points + int npt = 0; + for (int n = 0; n < N; n++) { + + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) continue; // totally ad-hoc number + + irls_valsub[npt] = val[n]; + irls_wgtsub[npt] = wgt[n]; + npt ++; + } + + for (int iter = 0; iter < BOOTSTRAP_NITER; iter++) { + irls_bootstrap (irls_idx, npt); // fill idx with the index of the resampled points + irls_testval[iter] = irls_wtmean (irls_valsub, irls_wgtsub, irls_idx, npt, NULL); + } + + float sigma = irls_robust_stdev (irls_testval, BOOTSTRAP_NITER); + + *outvar = SQ(sigma); + *Npt = npt; + } else { + // calculate stdev of high-weight points + float S1 = 0.0, S2 = 0.0; + int npt = 0; + for (int n = 0; n < N; n++) { + + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) continue; // totally ad-hoc number + + S1 += SQ(dV); + S2 += wgt[n]; + npt ++; + } + *Npt = npt; + *outvar = 1 / S2; + } + } + + return Value; +} + +void irls_bootstrap (int *idx, int Npts) { + // generate an index (idx) containing the index values + // for the Npts randomly resampled points + for (int i = 0; i < Npts; i++) { + idx[i] = Npts * drand48(); + } +} + +float irls_wtmean (float *val, float *wgt, int *idx, int Npt, float *variance) { + float S1 = 0.0, S2 = 0.0; + for (int n = 0; n < Npt; n++) { + int N = idx ? idx[n] : n; + S1 += wgt[N] * val[N]; + S2 += wgt[N]; + } + float Value = S1 / S2; + if (variance) *variance = 1.0 / S2; + return Value; +} + +float irls_robust_stdev (float *values, int Npts) { + + fsort (values, Npts); + + float Slo = irls_fraction_interpolate (values, 0.158655, Npts); + float Shi = irls_fraction_interpolate (values, 0.841345, Npts); + float sigma = (Shi - Slo) / 2.0; + + return sigma; +} + +float irls_fraction_interpolate (float *values, float fraction, int Npts) { + + float F = fraction * Npts; + int N = fraction * Npts; + + if (N < 0 ) return NAN; + if (N >= Npts - 2) return NAN; + + // interpolate between N,N+1 + + float S = (F - N) * (values[N+1] - values[N]) + values[N]; + return S; +} + + +// exp(-(x^2/s^2)/2) = (1/2) +// -(x^2/s^2)/2 = ln(1/2) +// (x^2/s^2)/2 = ln(2) +// (x^2/s^2) = 2ln(2) +// (x /s) = sqrt(2ln(2)) : half-width at half-max +// FWHM = 2sqrt(2ln(2)) + +// R2 = (X / 2.385)^2 = (X^2 / 2.385^2) + +# define CAUCY_FACTOR 2.0 + +/* +float weight_cauchy_square_flt (float x2) { + return (1.0); +} +*/ + +float weight_cauchy_square_flt (float x2) { + float r2 = x2 / CAUCY_FACTOR; + return (1.0 / (1.0 + r2)); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c (revision 41170) @@ -15,13 +15,30 @@ int medimage_add (int argc, char **argv) { + int N; Buffer *image; + Buffer *var = NULL; + + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((var = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage add (name) (image)\n"); + gprint (GP_ERR, "USAGE: medimage add (name) (image) [-variance variance]\n"); gprint (GP_ERR, " add the given image to the set of images to be medianed\n"); + gprint (GP_ERR, " optionally supply variance image (for weighted calculations)\n"); return FALSE; } if ((image = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + if (var) { + if ((var->matrix.Naxis[0] != image->matrix.Naxis[0]) || + (var->matrix.Naxis[1] != image->matrix.Naxis[1])) { + gprint (GP_ERR, "variance buffer does not match image buffer dimensions\n"); + return FALSE; + } + } MedImageType *median = FindMedImage (argv[1]); @@ -46,76 +63,17 @@ // new image should match existing medimage dimensions + // AddMedImage (median, image, var); int Ninput = median->Ninput; median->Ninput ++; - REALLOCATE (median->buffers, float *, median->Ninput); + REALLOCATE (median->flx, float *, median->Ninput); + REALLOCATE (median->var, float *, median->Ninput); - ALLOCATE (median->buffers[Ninput], float, median->Nx*median->Ny); - memcpy (median->buffers[Ninput], image->matrix.buffer, sizeof(float)*median->Nx*median->Ny); + ALLOCATE (median->flx[Ninput], float, median->Nx*median->Ny); + memcpy (median->flx[Ninput], image->matrix.buffer, sizeof(float)*median->Nx*median->Ny); - return TRUE; -} - -int medimage_calc (int argc, char **argv) { - - int ix, iy, n, N; - Buffer *output; - - int CALC_MEAN = FALSE; - if ((N = get_argument (argc, argv, "-mean"))) { - CALC_MEAN = TRUE; - remove_argument (N, &argc, argv); - } - - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage calc (name) (output) [-mean]\n"); - gprint (GP_ERR, " calculate the median image for the median image set\n"); - return FALSE; - } - - MedImageType *median = FindMedImage (argv[1]); - if (!median) { - gprint (GP_ERR, "median image %s not found\n", argv[1]); - return FALSE; - } - - if ((output = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) return (FALSE); - - int Ninput = median->Ninput; - int Nx = median->Nx; - int Ny = median->Ny; - - float *value = NULL; - ALLOCATE (value, float, Ninput); - - gfits_free_matrix (&output->matrix); - gfits_free_header (&output->header); - if (!CreateBuffer (output, Nx, Ny, -32, 0.0, 1.0)) return FALSE; - - float *outvalue = (float *) output->matrix.buffer; - - for (iy = 0; iy < Ny; iy++) { - for (ix = 0; ix < Nx; ix++) { - - int N = 0; - int Npix = ix + Nx*iy; - for (n = 0; n < Ninput; n++) { - float v = median->buffers[n][Npix]; - if (!isfinite(v)) continue; - value[N] = v; - N++; - } - if (N == 0) continue; - - if (CALC_MEAN) { - float sum = 0.0; - for (n = 0; n < N; n++) { - sum += value[n]; - } - outvalue[Npix] = sum / (float) N; - } else { - fsort (value, N); - outvalue[Npix] = value[(int)(0.5*N)]; - } - } + median->var[Ninput] = NULL; + if (var) { + ALLOCATE (median->var[Ninput], float, median->Nx*median->Ny); + memcpy (median->var[Ninput], var->matrix.buffer, sizeof(float)*median->Nx*median->Ny); } @@ -124,39 +82,39 @@ /* -int medimage_save (int argc, char **argv) { + int medimage_save (int argc, char **argv) { - int N; + int N; - int APPEND = FALSE; - if ((N = get_argument (argc, argv, "-append"))) { - APPEND = TRUE; - remove_argument (N, &argc, argv); - } + int APPEND = FALSE; + if ((N = get_argument (argc, argv, "-append"))) { + APPEND = TRUE; + remove_argument (N, &argc, argv); + } - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage save (name) (filename) [-append]\n"); - return FALSE; - } + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage save (name) (filename) [-append]\n"); + return FALSE; + } - if (!SaveMedImage(argv[2], argv[1], APPEND)) { - gprint (GP_ERR, "failed to save medimage %s\n", argv[1]); - return (FALSE); - } - return TRUE; -} + if (!SaveMedImage(argv[2], argv[1], APPEND)) { + gprint (GP_ERR, "failed to save medimage %s\n", argv[1]); + return (FALSE); + } + return TRUE; + } -int medimage_load (int argc, char **argv) { + int medimage_load (int argc, char **argv) { - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage load (name) (filename)\n"); - return FALSE; - } + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage load (name) (filename)\n"); + return FALSE; + } - if (!LoadMedImage(argv[2], argv[1])) { - gprint (GP_ERR, "failed to load medimage %s\n", argv[1]); - return (FALSE); - } - return TRUE; -} + if (!LoadMedImage(argv[2], argv[1])) { + gprint (GP_ERR, "failed to load medimage %s\n", argv[1]); + return (FALSE); + } + return TRUE; + } */ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c (revision 41170) @@ -0,0 +1,34 @@ +# include "data.h" + +int mgaussdev (int argc, char **argv) { + + Buffer *buf; + + if (argc != 6) goto usage; + + if ((buf = SelectBuffer (argv[1], ANYBUFFER, TRUE)) == NULL) return (FALSE); + int Nx = atof (argv[2]); + int Ny = atof (argv[3]); + + double mean = atof (argv[4]); + double sigma = atof (argv[5]); + + /* I should encapsulate this in a create_default_buffer */ + gfits_free_matrix (&buf[0].matrix); + gfits_free_header (&buf[0].header); + + // 3D CUBE OPTION: if (!CreateBuffer3D (buf, Nx, Ny, Nz, -32, 1.0, 0.0)) return FALSE; + if (!CreateBuffer (buf, Nx, Ny, -32, 1.0, 0.0)) return FALSE; + + ohana_gaussdev_init (); + + float *v = (float *) buf[0].matrix.buffer; + for (int i = 0; i < Nx*Ny; i++, v++) { + *v = ohana_gaussdev_rnd (mean, sigma); + } + return (TRUE); + + usage: + gprint (GP_ERR, "USAGE: mgaussdev (buff) Nx Ny mean sigma\n"); + return (FALSE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c (revision 41170) @@ -39,4 +39,5 @@ if (*X < start) continue; if (*X > end) continue; + if (!isfinite(*Y)) continue; if (*Y < ymax) continue; xmax = *X; @@ -63,4 +64,5 @@ if (*X < start) continue; if (*X > end) continue; + if (!isfinite(*Y)) continue; if (*Y < ymax) continue; xmax = *X; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c (revision 41170) @@ -3,7 +3,9 @@ int roll (int argc, char **argv) { - int Nbytes, Nextra; - int dX, dx, dy, nx, ny; Buffer *buf; + + // this function is probably not needed + gprint (GP_ERR, "ERROR: use 'shift' instead of 'roll'\n"); + return (FALSE); if (argc != 4) { @@ -14,32 +16,71 @@ if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - dx = atof (argv[2]); - dy = atof (argv[3]); - if (dy != 0) { - gprint (GP_ERR, "only x rolls implemented for now\n"); + int dx = atoi (argv[2]); + int dy = atoi (argv[3]); + + /* if (dx,dy < 0), we are moving the start position backwards, + if (dx,dy > 0), we are moving the start position forward */ + + int Nx = buf[0].matrix.Naxis[0]; + int Ny = buf[0].matrix.Naxis[1]; + + if (dy == 0) { + // moves in just dx can be done row-by-row + + int dX = abs(dx); + int Nbytes = (Nx * Ny - dX) * sizeof (float); + int Nextra = dX * sizeof (float); + + if (dx < 0) { + memmove (buf[0].matrix.buffer, &buf[0].matrix.buffer[dX*sizeof(float)], Nbytes); + memset (&buf[0].matrix.buffer[Nbytes], 0, Nextra); + } else { + memmove (&buf[0].matrix.buffer[dX*sizeof(float)], buf[0].matrix.buffer, Nbytes); + memset (buf[0].matrix.buffer, 0, Nextra); + } + + return TRUE; } - /* if (dx < 0), we are moving the start position back by dx pixels, - if (dx > 0), we are moving the start position forward by dx pixels */ - - nx = buf[0].matrix.Naxis[0]; - ny = buf[0].matrix.Naxis[1]; + if (dx == 0) { + // moves in just dy can be done row-by-row - dX = abs(dx); - Nbytes = nx * ny * sizeof (float); - Nextra = (nx * ny + dX) * sizeof (float); + ALLOCATE_PTR (output, float, Nx*Ny); - REALLOCATE (buf[0].matrix.buffer, char, Nextra); + float *input = (float *) buf[0].matrix.buffer; - if (dx < 0) { - memmove (buf[0].matrix.buffer, &buf[0].matrix.buffer[dX*sizeof(float)], Nbytes); - } else { - memmove (&buf[0].matrix.buffer[dX*sizeof(float)], buf[0].matrix.buffer, Nbytes); + for (int iy = 0; iy < Ny; iy++) { + if (iy + dy < 0) continue; + if (iy + dy >= Ny) continue; + memcpy (&output[iy*Nx], &input[(iy + dy)*Nx], Nx); + } + if (dy > 0) { + memset (output, 0, dy*Nx*sizeof(float)); + } else { + int Nbytes = Nx * (Ny - abs(dy)) * sizeof (float); + int Nextra = abs(dy) * Nx * sizeof (float); + memset (&output[Nbytes], 0, Nextra); + } + free (buf[0].matrix.buffer); + buf[0].matrix.buffer = (char *) output; + return TRUE; } - REALLOCATE (buf[0].matrix.buffer, char, Nbytes); + ALLOCATE_PTR (output, float, Nx*Ny); + float *input = (float *) buf[0].matrix.buffer; - return (TRUE); - + for (int iy = 0; iy < Ny; iy++) { + if (iy + dy < 0) continue; + if (iy + dy >= Ny) continue; + for (int ix = 0; ix < Nx; ix++) { + if (ix + dx < 0) continue; + if (ix + dx >= Nx) continue; + output[ix + iy*Nx] = input[(ix + dx) + (iy + dy)*Nx]; + } + } + free (buf[0].matrix.buffer); + buf[0].matrix.buffer = (char *) output; + return TRUE; } + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c (revision 41170) @@ -3,19 +3,19 @@ int rotate (int argc, char **argv) { - int i, j, NX, NY, X, Y, Lx, Ly, N; + int i, j, NX, NY; float *in_buff, *out_buff, *c; - double angle, CosAngle, SinAngle, Xo, Yo, dX, dY, fx, fy, x, y, X1, Y1; + double Xo, Yo, x, y, X1, Y1; double pc11, pc12, pc21, pc22, PC11, PC12, PC21, PC22; Buffer *buf; - Xo = 0; - Yo = 0; - if ((N = get_argument (argc, argv, "-center"))) { - remove_argument (N, &argc, argv); - Xo = atof(argv[N]); - remove_argument (N, &argc, argv); - Yo = atof(argv[N]); - remove_argument (N, &argc, argv); - } +// Xo = 0; +// Yo = 0; +// if ((N = get_argument (argc, argv, "-center"))) { +// remove_argument (N, &argc, argv); +// Xo = atof(argv[N]); +// remove_argument (N, &argc, argv); +// Yo = atof(argv[N]); +// remove_argument (N, &argc, argv); +// } if (argc != 3) { @@ -221,30 +221,26 @@ } - angle = atof (argv[2]); - CosAngle = cos (angle*RAD_DEG); - SinAngle = sin (angle*RAD_DEG); + double angle = atof (argv[2]); + double CosAngle = cos (angle*RAD_DEG); + double SinAngle = sin (angle*RAD_DEG); gprint (GP_ERR, "rotating: %f %f %f\n", angle, CosAngle, SinAngle); - Lx = NX*fabs(CosAngle) + NY*fabs(SinAngle); - Ly = NX*fabs(SinAngle) + NY*fabs(CosAngle); - dX = MAX(0,NY*SinAngle); - dY = MAX(0,-NX*SinAngle); - /* - gprint (GP_ERR, "%f %f --> ", Xo, Yo); - X1 = Xo*CosAngle - Yo*SinAngle + dX; - Y1 = Xo*SinAngle + Yo*CosAngle + dY; - gprint (GP_ERR, "%f %f\n", X1, Y1); - */ + // we are rotating about the center pixel, (NX/2, NY/2), + // but we are then putting the result in a new image + // of size (Lx,Ly). + + // (x,y) = (i - Nx/2),(j - Ny/2) + // (x',y') = R(theta) (x,y) + // (I,J) = (x' + Lx/2),(y' + Ly/2) + + int Lx = NX*fabs(CosAngle) + NY*fabs(SinAngle); + int Ly = NX*fabs(SinAngle) + NY*fabs(CosAngle); /* fix reference pixel */ gfits_scan (&buf[0].header, "CRPIX1", "%lf", 1, &Xo); gfits_scan (&buf[0].header, "CRPIX2", "%lf", 1, &Yo); - /* - X1 = (Xo - dX)*CosAngle - (Yo - dY)*SinAngle; - Y1 = (dX - Xo)*SinAngle + (Yo - dY)*CosAngle; - */ - X1 = Xo*CosAngle - Yo*SinAngle + dX; - Y1 = Xo*SinAngle + Yo*CosAngle + dY; + X1 = (Xo - NX/2)*CosAngle + (Yo - NY/2)*SinAngle + Lx/2; + Y1 = (NX/2 - Xo)*SinAngle + (Yo - NY/2)*CosAngle + Ly/2; gfits_modify (&buf[0].header, "CRPIX1", "%lf", 1, X1); gfits_modify (&buf[0].header, "CRPIX2", "%lf", 1, Y1); @@ -255,8 +251,8 @@ gfits_scan (&buf[0].header, "PC002001", "%lf", 1, &pc21); gfits_scan (&buf[0].header, "PC002002", "%lf", 1, &pc22); - PC11 = pc11*CosAngle - pc21*SinAngle; - PC12 = pc12*CosAngle - pc22*SinAngle; - PC21 = pc21*CosAngle + pc11*SinAngle; - PC22 = pc22*CosAngle + pc12*SinAngle; + PC11 = pc11*CosAngle + pc21*SinAngle; + PC12 = pc12*CosAngle + pc22*SinAngle; + PC21 = pc21*CosAngle - pc11*SinAngle; + PC22 = pc22*CosAngle - pc12*SinAngle; gfits_modify (&buf[0].header, "PC001001", "%le", 1, PC11); gfits_modify (&buf[0].header, "PC001002", "%le", 1, PC12); @@ -270,22 +266,32 @@ gfits_create_matrix (&buf[0].header, &buf[0].matrix); gfits_print_alt (&buf[0].header, "HISTORY", "%S", 1, "WARNING: rotated image!"); + out_buff = (float *)buf[0].matrix.buffer; for (j = 0; j < Ly; j++) { for (i = 0; i < Lx; i++, out_buff++) { - x = (i - dX)*CosAngle + (j - dY)*SinAngle; - y = (dX - i)*SinAngle + (j - dY)*CosAngle; - X = (int) x; - Y = (int) y; - - if (X < 0) continue; - if (X >= NX - 1) continue; - if (Y < 0) continue; - if (Y >= NY - 1) continue; - - c = &in_buff[X + NX*Y]; - fx = x - X; - fy = y - Y; + float xo = (i - Lx/2); + float yo = (j - Ly/2); + + x = xo*CosAngle + yo*SinAngle; + y = -xo*SinAngle + yo*CosAngle; + + int I = x + NX/2; + int J = y + NY/2; + + if (I < 0) continue; + if (I >= NX - 1) continue; + if (J < 0) continue; + if (J >= NY - 1) continue; + + c = &in_buff[I + NX*J]; + + int X = (int) x; + int Y = (int) y; + double fx = x - X; + double fy = y - Y; + *out_buff = (c[0]*(1-fx) + c[1]*fx)*(1-fy) + (c[NX+1]*fx + c[NX]*(1-fx))*fy; +// *out_buff = c[0]; } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c (revision 41170) @@ -7,4 +7,5 @@ int spline_load (int argc, char **argv); int spline_save (int argc, char **argv); +int spline_print (int argc, char **argv); int spline_delete (int argc, char **argv); int spline_rename (int argc, char **argv); @@ -18,4 +19,5 @@ {1, "load", spline_load, "write a spline to a FITS file"}, {1, "save", spline_save, "read a spline from a FITS file"}, + {1, "print", spline_print, "print a spline to stdout"}, {1, "delete", spline_delete, "delete a spline"}, {1, "rename", spline_rename, "rename a spline"}, @@ -35,4 +37,5 @@ gprint (GP_ERR, " spline load (spline) (filename) : load a spline from a FITS file\n"); gprint (GP_ERR, " spline save (spline) (filename) [-append] : save a spline in FITS format\n"); + gprint (GP_ERR, " spline print (spline) : print a spline to stdout\n"); return FALSE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c (revision 41170) @@ -28,4 +28,9 @@ if (xvec->Nelements != yvec->Nelements) { gprint (GP_ERR, "x and y vectors must be the same length\n"); + return FALSE; + } + + if (xvec->Nelements < 3) { + gprint (GP_ERR, "cannot make a spline with fewer than 3 knots\n"); return FALSE; } @@ -93,4 +98,26 @@ } +int spline_print (int argc, char **argv) { + + int i; + + if (argc != 2) { + gprint (GP_ERR, "USAGE: spline print (name)\n"); + return FALSE; + } + + Spline *myspline = FindSpline (argv[1]); + if (!myspline) { + gprint (GP_ERR, "spline %s not found\n", argv[1]); + return (FALSE); + } + + for (i = 0; i < myspline->Nknots; i++) { + gprint (GP_LOG, "%lf : %lf : %lf\n", myspline->xk[i], myspline->yk[i], myspline->y2[i]); + } + + return TRUE; +} + int spline_load (int argc, char **argv) { @@ -109,6 +136,11 @@ int spline_delete (int argc, char **argv) { - int status; - Spline *spline; + int N, status; + + int QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } if (argc != 2) { @@ -117,6 +149,7 @@ } - spline = FindSpline (argv[1]); + Spline *spline = FindSpline (argv[1]); if (spline == NULL) { + if (QUIET) return TRUE; gprint (GP_ERR, "spline %s not found\n", argv[1]); return FALSE; @@ -130,6 +163,4 @@ int spline_rename (int argc, char **argv) { - Spline *spline; - if (argc != 3) { gprint (GP_ERR, "USAGE: spline rename (spline) (newname)\n"); @@ -137,8 +168,13 @@ } - spline = FindSpline (argv[1]); + Spline *spline = FindSpline (argv[1]); if (spline == NULL) { gprint (GP_ERR, "spline %s not found\n", argv[1]); return FALSE; + } + + Spline *newname = FindSpline (argv[2]); + if (newname != NULL) { + DeleteSpline (newname); } @@ -147,24 +183,2 @@ return TRUE; } - -/* -int spline_listspline (int argc, char **argv) { - - Spline *spline; - - if (argc != 2) { - gprint (GP_ERR, "USAGE: spline listspline (spline)\n"); - return FALSE; - } - - spline = FindSpline (argv[1]); - if (spline == NULL) { - gprint (GP_ERR, "spline %s not found\n", argv[1]); - return FALSE; - } - - ListPages (spline); - return TRUE; -} -*/ - Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh (revision 41170) @@ -4,4 +4,46 @@ memtest1 end + +macro test2 + + mgaussdev z 50 100 0.0 1.0 + + cut z x y_sum x 0 0 z[][0] z[0][] + cut z x y_median x 0 0 z[][0] z[0][] -median + cut z x y_mean x 0 0 z[][0] z[0][] -mean + cut z x y_inner x 0 0 z[][0] z[0][] -inner + +end + +macro checkrange + if ($0 != 2) + echo "USAGE: checkrange (Npts)" + break + end + + $Npts = $1 + if ($Npts % 2) + $Ncenter = int($Npts / 2) + $Nquarter = int(0.25 * $Npts) + $Ns = $Ncenter - $Nquarter + $Ne = $Ncenter + $Nquarter + else + $Ncenter = int($Npts / 2) - 1 + $Nquarter = int(0.25 * $Npts) + $Ns = $Ncenter - $Nquarter + $Ne = $Ncenter + $Nquarter + 1 + end + if ($Ns < 0) + echo "error: Ns < 0: $Ns $Ncenter $Nquarter" + end + if ($Ne >= $Npts) + echo "error: Ne >= Npts: $Ne $Ncenter $Nquarter" + end + echo "$Npts : $Ncenter : $Nquarter : $Ns $Ne" + for i 0 $Npts + echo $i {($i >= $Ns) && ($i <= $Ne)} + end +end + # Test if cut works @@ -37,2 +79,42 @@ end end + +macro memtest1 + + $i = 0 + mcreate tim 100 100 + cut tim xdir imx X 40 4 60 6 + # do one to set up memory that should stay used + + memory check + for i 0 100 + cut tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut tim xdir imx y 40 4 60 6 + end + memory check + + for i 0 100 + cut -median tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut -median tim xdir imx y 40 4 60 6 + end + memory check + + for i 0 100 + cut -mean tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut -mean tim xdir imx y 40 4 60 6 + end + memory check + +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh (revision 41170) @@ -49,2 +49,13 @@ end + +# Test delete +macro test2 + + $PASS = 1 + + for i 0 1000 + create v 0 200 + delete v + end +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh (revision 41170) @@ -0,0 +1,15 @@ + +macro init.test + create x 0 10 + set y = x^2 + mcreate blank 0 0 + keyword blank TE0 -wd 56 + keyword blank TE1 -w "TEST LINE" + keyword blank TE2 -wf 5.12342341 + keyword blank TE3 -wb T + keyword blank TE4 -wc "not sure what do say" + keyword blank TE4 -wf 4.5 + keyword blank TE5 -ws "this is a long comment" + keyword blank TE2 -wc "this is a short comment" + write -fits DATA test2.fits x y -header blank +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh (revision 41170) @@ -1,11 +1,540 @@ -macro go - mcreate a 30 30 - for i 0 40 - set a$i = zero(a) + $i - end - - for i 0 40 - medimage add t1 a$i - end -end +macro test.mean + + $Nsample = 16 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 1.0 + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + medimage calc t1 T -mean + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "expect {1/sqrt($Nsample)} : $C1" + plot -c red -x line x yf +end + +macro test.median + + $Nsample = 16 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 1.0 + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "expect {1/sqrt($Nsample)} : $C1 (actually should be a bit higher)" + plot -c red -x line x yf +end + +macro test.wtmean + + $Nsample = 8 + $sig1 = 1.0 + $sig2 = 3.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig1 + set v = $sig1^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig2 + set v = $sig2^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -wtmean + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + $S1 = $Nsample / $sig1^2 + $Nsample / $sig2^2 + echo "expect {1/sqrt($S1)} : $C1" + plot -c red -x line x yf +end + +macro test.irls + medimage delete -q t1 + $Nsample = 16 + $sig = 1.0 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf +end + + +###################33 + + +macro test.mean.var + + $Nsample = 64 + $sig = 2.0 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 100 100 0.0 $sig + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + medimage calc t1 T -mean -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + imhist Tv xv yv -range -1 4 -delta 0.1 + lim -n 2 xv yv; clear; box; plot xv yv -x hist + + stat Tv + echo "$C1 vs {sqrt($MEDIAN)} : expect {$sig/sqrt($Nsample)}" +end + +macro test.median.var + + $Nsample = 64 + $sig = 2.0 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + imhist Tv xv yv -range -1 4 -delta 0.1 + lim -n 2 xv yv; clear; box; plot xv yv -x hist + + stat Tv + echo "$C1 vs {sqrt($MEDIAN)} : expect {$sig/sqrt($Nsample)}" +end + +macro test.wtmean.var + + $Nsample = 32 + $sig1 = 1.0 + $sig2 = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig1 + set v = $sig1^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig2 + set v = $sig2^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -wtmean -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + stat -q Tv + $S1 = $Nsample / $sig1^2 + $Nsample / $sig2^2 + echo $C1 vs {sqrt($MEDIAN)} : expect {1/sqrt($S1)} +end + +macro test.irls.var + + $Nsample = 16 + $sig = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range -1 4 -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} (ideal is {$sig/sqrt($Nsample)})" +end + +macro test.irls.boot.var + + $Nsample = 64 + $sig = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 200 200 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + mgaussdev noise 200 200 0.0 0.5 + set ts = ts + noise + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.02 + lim -n 1 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig / sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + date + medimage calc t1 Ti -irls -variance Tv -bootstrap-iter 100 + date + + imhist -q Ti x y -range -10 10 -delta 0.02 + lim -n 2 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig / sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range 0 {5*$sig/sqrt($Nsample)} -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} (ideal is {$sig/sqrt($Nsample)})" +end + +############################## +macro test.irls.boot.test + + $Nsample = 100 + $sig1 = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 100 100 0.0 $sig1 + set v = $sig1^2 + zero(t) + + medimage add t1 t -variance v + end + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv -bootstrap + + imhist -q Ti x y -range {-10*$sig1/sqrt($Nsample)} {10*$sig1/sqrt($Nsample)} -delta 0.01 + lim -n 2 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig1/sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss x y con yf + echo "sigma from irls: $C1 (ideal is {$sig1/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range 0 {5*$sig1/sqrt($Nsample)} -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q irls_npt + $Npix = $MEAN + + stat -q Tv + echo "sigma of irls average: $C1, sqrt(mean) of irls variance: {sqrt($MEAN)}, (ideal is {$sig1/sqrt($Npix)})" +end + +macro test.irls.range.var + medimage delete -q t1 + for i 0 8 + mgaussdev t 50 50 0.0 1.0 + set v = 1.0 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + end + + for i 0 8 + mgaussdev t 50 50 0.0 3.0 + set v = 3.0 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + stats Ti + + plot -c red -x line x yf + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} + + set dTv = sqrt(Tv) + imhist dTv xv yv -range -1 4 -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh (revision 41170) @@ -0,0 +1,43 @@ + +macro memtest1 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = exp(x) + end + memory check + +end + +macro memtest2 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = (x < 0) ? x : exp(x) + end + memory check + +end + +macro memtest3 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = (x < 5 + 2*$i) ? x : exp(x) + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh (revision 41170) @@ -0,0 +1,11 @@ + +macro test1 + + for i 0 100 + create x 0 50 + set y = x^2 + spline create t$i x y + spline rename t$i test + spline delete test + end +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh (revision 41170) @@ -0,0 +1,263 @@ + +list tests + test1 + test2 + test3 +end + +macro test1 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + + vsigmoid -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +# noise of 0.1 +macro test2 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 50 + 1000 / (1 + exp((x - 3)/2)) + set dy = (rnd(y) - 0.5)/0.5 + set y = y + dy + + $C0 = 2 + $C1 = 3 + $C2 = 900 + $C3 = 1 + + vsigmoid -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 1000.0) > 0.1) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.1) + $PASS = 0 + end +end + +# poisson-distributed noise +macro test3 + $PASS = 1 + break -auto off + + $C0o = 3 + $C1o = 2 + $C2o = 20 + $C3o = -2 + + create x -10 10 0.1 + set y = $C3o + $C2o / (1 + exp((x - $C0o)/$C1o)) + + gaussdev dY y[] 0.0 1.0 + set dy = dY * sqrt(abs(y)) + set y = y + dy + + $C0 = $C0o + 2 + $C1 = $C1o + 2 + $C2 = $C2o + 50 + $C3 = $C3o + 10 + + vsigmoid x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + $dS = 3.0 / sqrt(y[]) + + if (abs($C0 - $C0o) > $dS) + $PASS = 0 + end + if (abs($C1 - $C1o)/$C1o > $dS) + $PASS = 0 + end + if (abs($C2 - $C2o)/$C2o > $dS) + $PASS = 0 + end + if (abs($C3 - $C3o) > $dS) + $PASS = 0 + end +end + +macro test1rev + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 5 / (1 + exp(-(x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(abs(y)) + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +# noise of 0.1 +macro test2rev + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 50 + 1000 / (1 + exp(-(x - 3)/2)) + set dy = (rnd(y) - 0.5)/0.5 + set y = y + dy + + $C0 = 2 + $C1 = 3 + $C2 = 900 + $C3 = 1 + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 1000.0) > 0.1) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.1) + $PASS = 0 + end +end + +# poisson-distributed noise +macro test3rev + $PASS = 1 + break -auto off + + $C0o = 3 + $C1o = 2 + $C2o = 20 + $C3o = -2 + + create x -10 10 0.1 + set y = $C3o + $C2o / (1 + exp(-(x - $C0o)/$C1o)) + + gaussdev dY y[] 0.0 1.0 + set dy = dY * sqrt(abs(y)) + set y = y + dy + + $C0 = $C0o + 2 + $C1 = $C1o + 2 + $C2 = $C2o + 50 + $C3 = $C3o + 10 + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + $dS = 3.0 / sqrt(y[]) + + if (abs($C0 - $C0o) > $dS) + $PASS = 0 + end + if (abs($C1 - $C1o)/$C1o > $dS) + $PASS = 0 + end + if (abs($C2 - $C2o)/$C2o > $dS) + $PASS = 0 + end + if (abs($C3 - $C3o) > $dS) + $PASS = 0 + end +end + +macro memtest1 + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + set yfit = zero(y) + + $i = 0 + + memory check + for i 0 1000 + vsigmoid -q x y dy yfit + end + memory check + +end + +macro memtest2 + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + set yfit = zero(y) + + $i = 0 + + memory check + for i 0 1000 + vsigmoid -q -r x y dy yfit + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh (revision 41170) @@ -0,0 +1,28 @@ + +macro memtest1 + + # preset variables so memory usage does not change + $PMIN = -10 + $PMAX = 9.9 + $MIN = -10 + $MAX = 9.9 + $MEDIAN = -0.1097 + $MEAN = -0.05 + $MODE = -10 + $TOTAL = -10 + $NPIX = 200 + $NPTS = 200 + $NUSED = 200 + $SIGMA = 5.7879184514 + + create x -10 10 0.1 + $i = 0 + + memory check + for i 0 1000 + vstat -q x + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c (revision 41170) @@ -25,4 +25,14 @@ remove_argument (N, &argc, argv); BinMax = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + float ValMin = NAN; + float ValMax = NAN; + if ((N = get_argument (argc, argv, "-vrange"))) { + remove_argument (N, &argc, argv); + ValMin = atof (argv[N]); + remove_argument (N, &argc, argv); + ValMax = atof (argv[N]); remove_argument (N, &argc, argv); } @@ -64,4 +74,16 @@ } + // ValMin, ValMax not compatible with BinMin, BinMax + if (isfinite(ValMin)) { + int binTest = ohana_bisection_double (vecx[0].elements.Flt, vecx[0].Nelements, ValMin); + BinMin = (vecx[0].elements.Flt[binTest+1] == ValMin) ? binTest : binTest + 1; + // XXX dangerous: check that this does not run off the end, etc + } + if (isfinite(ValMax)) { + int binTest = ohana_bisection_double (vecx[0].elements.Flt, vecx[0].Nelements, ValMax); + BinMax = (vecx[0].elements.Flt[binTest+1] == ValMax) ? binTest : binTest + 1; + // XXX dangerous: check that this does not run off the end, etc + } + // use bisection to find the value @@ -71,4 +93,13 @@ if (!REVERSE) { + // this algorithm assumes vecy[BinMax] > threshold, vecy[BinMin] < threshold + if (vecy[0].elements.Flt[BinMin] > value) { + gprint (GP_ERR, "ERROR: all values above threshold\n"); + return FALSE; + } + if (vecy[0].elements.Flt[BinMax] < value) { + gprint (GP_ERR, "ERROR: all values below threshold\n"); + return FALSE; + } while (Nhi - Nlo > 10) { N = 0.5*(Nlo + Nhi); @@ -89,4 +120,13 @@ } } else { + // this algorithm assumes vecy[BinMin] > threshold, vecy[BinMax] < threshold + if (vecy[0].elements.Flt[BinMin] < value) { + gprint (GP_ERR, "ERROR: all values below threshold\n"); + return FALSE; + } + if (vecy[0].elements.Flt[BinMax] > value) { + gprint (GP_ERR, "ERROR: all values above threshold\n"); + return FALSE; + } while (Nhi - Nlo > 10) { N = 0.5*(Nlo + Nhi); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c (revision 41170) @@ -3,4 +3,5 @@ /* local private functions */ opihi_flt fgaussOD (opihi_flt, opihi_flt *, int, opihi_flt *); +int vgauss_apply (int argc, char **argv); # define GET_VAR(V,A) \ @@ -20,4 +21,10 @@ Vector *xvec, *yvec, *svec, *ovec; char *c, name[16]; + + if ((N = get_argument (argc, argv, "-apply"))) { + remove_argument (N, &argc, argv); + int status = vgauss_apply (argc, argv); + return status; + } Quiet = FALSE; @@ -104,4 +111,6 @@ } if (!Quiet) gprint (GP_ERR, "%d iterations\n", i); + set_variable ("Chisq", chisq); + set_variable ("ChisqNu", chisq / (float) (Npts - Npar)); for (i = 0; i < Npts; i++) { @@ -125,4 +134,38 @@ } +int vgauss_apply (int argc, char **argv) { + + opihi_flt par[4]; + Vector *xvec, *ovec; + char *c; + + // NOTE: -apply has already been stripped by vgauss call + if (argc != 3) { + gprint (GP_ERR, "USAGE: vgauss -apply \n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ovec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + CastVector (xvec, OPIHI_FLT); + + GET_VAR (par[0], "C0"); + GET_VAR (par[1], "C1"); + GET_VAR (par[2], "C2"); + GET_VAR (par[3], "C3"); + int Npar = 4; + + int Npts = xvec[0].Nelements; + ResetVector (ovec, OPIHI_FLT, Npts); + + for (int i = 0; i < Npts; i++) { + ovec[0].elements.Flt[i] = fgaussOD (xvec[0].elements.Flt[i], par, Npar, NULL); + } + ovec[0].Nelements = Npts; + + return TRUE; +} + /* pars: x_o, sigma, I, back */ opihi_flt fgaussOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { @@ -135,8 +178,10 @@ f = par[2]*r + par[3]; - dpar[0] = par[2]*r*z/par[1]; - dpar[1] = par[2]*r*z*z/par[1]; - dpar[2] = r; - dpar[3] = 1; + if (dpar) { + dpar[0] = par[2]*r*z/par[1]; + dpar[1] = par[2]*r*z*z/par[1]; + dpar[2] = r; + dpar[3] = 1; + } return (f); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c (revision 41170) @@ -0,0 +1,103 @@ +# include "data.h" + +double opihi_irls_mean_dbl (double *val, double *wgt, int N, int *Converged); +double weight_cauchy_square_dbl (double x2); + +# define IRLS_TOLERANCE 1e-4 + +int virls (int argc, char **argv) { + + if (argc != 3) { + gprint (GP_ERR, "USAGE: virls (values) (sigmas)\n"); + gprint (GP_ERR, "NOTE: sigmas with abs values < %e will be truncated to that value\n", FLT_MIN); + return (FALSE); + } + + Vector *value, *sigma; + + if ((value = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((sigma = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if (value->Nelements != sigma->Nelements) { + gprint (GP_ERR, "vectors are not the same length\n"); + return FALSE; + } + if (value->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[1]); + return FALSE; + } + if (sigma->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[2]); + return FALSE; + } + + ALLOCATE_PTR (val, opihi_flt, value->Nelements); + ALLOCATE_PTR (wgt, opihi_flt, value->Nelements); + + int N = 0; + for (int i = 0; i < value->Nelements; i++) { + if (!isfinite(value->elements.Flt[i])) continue; + + double s = sigma->elements.Flt[i]; + if (!isfinite(s)) continue; + + if (fabs(s) < FLT_MIN) s = FLT_MIN; + + val[N] = value->elements.Flt[i]; + wgt[N] = 1.0 / SQ(s); + N ++; + } + + int converged = TRUE; + double Value = opihi_irls_mean_dbl (val, wgt, N, &converged); + + set_variable ("WTMEAN", Value); + set_int_variable ("NUSED", N); + set_int_variable ("CONVERGED", converged); + return (TRUE); +} + +double opihi_irls_mean_dbl (double *val, double *wgt, int N, int *Converged) { + + // calculate weighted mean + double S1 = 0.0, S2 = 0.0; + for (int n = 0; n < N; n++) { + S1 += wgt[n] * val[n]; + S2 += wgt[n]; + } + double Value = S1 / S2; + + int converged = FALSE; + for (int i = 0; (i < 10) && !converged; i++) { + double ValueLast = Value; + + double S1 = 0.0, S2 = 0.0; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int n = 0; n < N; n++) { + double dV = (val[n] - Value); + double d2 = SQ(dV) * wgt[n]; + + double Mod = weight_cauchy_square_dbl (d2); + S1 += Mod * wgt[n] * val[n]; + S2 += Mod * wgt[n]; + fprintf (stderr, "%f %f : %f %f : %f\n", val[n], wgt[n], dV, sqrt(d2), Mod); + } + Value = S1 / S2; + + double delta = fabs(Value - ValueLast); + if (delta < Value * IRLS_TOLERANCE) converged = TRUE; + + // XXX if the answer is close to zero, we might not converge + } + *Converged = converged; + return Value; +} + +# define CAUCY_FACTOR 1.0 + +double weight_cauchy_square_dbl (double x2) { + double r2 = x2 / CAUCY_FACTOR; + return (1.0 / (1.0 + r2)); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c (revision 41170) @@ -0,0 +1,192 @@ +# include "data.h" + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); + +/* local private functions */ +opihi_flt fsigmoidOD (opihi_flt, opihi_flt *, int, opihi_flt *); +opihi_flt rsigmoidOD (opihi_flt, opihi_flt *, int, opihi_flt *); + +# define GET_VAR(V,A) { \ + char *c = get_variable (A); \ + if (c == NULL) { \ + gprint (GP_ERR, "missing fit parameter A\n"); \ + return (FALSE); \ + } \ + V = atof (c); \ + free (c); } + +// XXX this fitting function works but fails if the input vectors have NANs +int vsigmoid (int argc, char **argv) { + + int N; + Vector *xvec, *yvec, *svec, *ovec; + + FitFunc *FUNC = fsigmoidOD; + if ((N = get_argument (argc, argv, "-r"))) { + FUNC = rsigmoidOD; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-rev"))) { + FUNC = rsigmoidOD; + remove_argument (N, &argc, argv); + } + + int Quiet = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: vsigmoid [-q] [-r] [-rev] (out)\n"); + gprint (GP_ERR, " may be the words 'con[stant]' or 'poi[sson]', in which case the error is constructed'\n"); + gprint (GP_ERR, " uses guesses: C0 (center), C1 (sigma), C2 (top), C3 (bottom)\n"); + gprint (GP_ERR, " forward sigmoid : bottom + top / (1 + exp(-z)), z = (x - center) / sigma\n"); + gprint (GP_ERR, " -r / -rev : reverse sigmoid : bottom + top / (1 + exp(z))\n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ovec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + CastVector (xvec, OPIHI_FLT); + CastVector (yvec, OPIHI_FLT); + + int Nsvec = strlen(argv[3]); + + if ((Nsvec >= 3) && (!strncasecmp ("poisson", argv[3], Nsvec) || !strncasecmp ("constant", argv[3], Nsvec))) { + svec = SelectVector ("vsigmoid_err", ANYVECTOR, TRUE); + if (svec == NULL) return (FALSE); + MatchVector (svec, xvec, OPIHI_FLT); + opihi_flt *v1 = svec[0].elements.Flt; + opihi_flt *v2 = yvec[0].elements.Flt; + if (!strncasecmp ("poisson", argv[3], Nsvec)) { + for (int i = 0; i < svec[0].Nelements; i++) { + v1[i] = (isfinite(v2[i]) && (v2[i] > 0)) ? sqrt(v2[i]) : 1.0; + } + } else { + for (int i = 0; i < svec[0].Nelements; i++) { + v1[i] = 1.0; + } + } + } else { + if ((svec = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + } + + CastVector (svec, OPIHI_FLT); + // XXX Cast is failing. + + int Npts = xvec[0].Nelements; + ResetVector (ovec, OPIHI_FLT, Npts); + + ALLOCATE_PTR (dy, opihi_flt, Npts); + + opihi_flt par[4]; + GET_VAR (par[0], "C0"); + GET_VAR (par[1], "C1"); + GET_VAR (par[2], "C2"); + GET_VAR (par[3], "C3"); + int Npar = 4; + + // mrqmin takes the inverse variance (do not generate NANs) + opihi_flt *v1 = svec[0].elements.Flt; + opihi_flt *v2 = dy; + for (int i = 0; i < Npts; i++, v1++, v2++) { + *v2 = (*v1 == 0.0) ? 0.0 : 1.0 / (*v1 * *v1); + } + + opihi_flt ochisq = mrqinit (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, FUNC, !Quiet); + opihi_flt dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + opihi_flt chisq = mrqmin (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, FUNC, !Quiet); + dchisq = ochisq - chisq; + ochisq = chisq; + if (!Quiet) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (!Quiet) gprint (GP_ERR, "%d iterations\n", Niter); + + for (int i = 0; i < Npts; i++) { + ovec[0].elements.Flt[i] = FUNC (xvec[0].elements.Flt[i], par, Npar, dy); + } + ovec[0].Nelements = Npts; + /* set output *before* variable renomalization */ + + opihi_flt **covar = mrqcovar (Npar); + for (int i = 0; i < Npar; i++) { + char name[16]; + sprintf (name, "C%d", i); + set_variable (name, par[i]); + if (!Quiet) gprint (GP_ERR, "%d %f %f\n", i, par[i], sqrt(covar[i][i])); + sprintf (name, "dC%d", i); + set_variable (name, sqrt(covar[i][i])); + } + + free (dy); + mrqfree (Npar); + return (TRUE); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt rsigmoidOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + dpar[0] = par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + + return (f); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt fsigmoidOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (-z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = -r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = -par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = -par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + dpar[0] = -par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + + return (f); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c (revision 41170) @@ -0,0 +1,175 @@ +# include "data.h" + +static int isFltX = FALSE; +static int isFltY = FALSE; +static double interpolate_position (Vector *vecx, Vector *vecy, int bin, float value); + +int vtransitions (int argc, char **argv) { + + int N; + Vector *vecx, *vecy, *outbin, *outval, *outpos; + + int QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + + int UP_TRANSITIONS = TRUE; + if ((N = get_argument (argc, argv, "-dn"))) { + UP_TRANSITIONS = FALSE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-down"))) { + UP_TRANSITIONS = FALSE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-up"))) { + UP_TRANSITIONS = TRUE; + remove_argument (N, &argc, argv); + } + + int BinMin = -1; + int BinMax = -1; + if ((N = get_argument (argc, argv, "-range"))) { + remove_argument (N, &argc, argv); + BinMin = atoi (argv[N]); + remove_argument (N, &argc, argv); + BinMax = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 4) { + gprint (GP_ERR, "USAGE: vtransitions (value) (-up|-down,-dn) [-q] [-range BinMin BinMax]\n"); + gprint (GP_ERR, " find the x coordinates at which we pass the specified value going up\n"); + gprint (GP_ERR, " -q : quiet mode\n"); + gprint (GP_ERR, " -r : reverse (find downward transition)\n"); + return (FALSE); + } + + if ((vecx = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((vecy = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if (vecx[0].Nelements != vecy[0].Nelements) return FALSE; + int Nelements = vecx[0].Nelements; + + if ((outbin = SelectVector ("vtr_bin", ANYVECTOR, TRUE)) == NULL) return (FALSE); // bins containing transitions + if ((outval = SelectVector ("vtr_val", ANYVECTOR, TRUE)) == NULL) return (FALSE); // value of bins containing transitions + if ((outpos = SelectVector ("vtr_pos", ANYVECTOR, TRUE)) == NULL) return (FALSE); // interpolated coordinate of transitions + + double value = atof (argv[3]); + + isFltX = (vecx[0].type == OPIHI_FLT); + isFltY = (vecy[0].type == OPIHI_FLT); + + // BinMin = -1 -> range not provide + if (BinMin == -1) { + BinMin = 0; + BinMax = Nelements - 1; + } else { + if ((BinMin >= vecx[0].Nelements) || (BinMin < -1*vecx[0].Nelements)) { + gprint (GP_ERR, "vector subscript out of range\n"); + return FALSE; + } + if (BinMin < 0) BinMin += vecx[0].Nelements; + if ((BinMax >= vecx[0].Nelements) || (BinMax < -1*vecx[0].Nelements)) { + gprint (GP_ERR, "vector subscript out of range\n"); + return FALSE; + } + if (BinMax < 0) BinMax += vecx[0].Nelements; + } + + int Nout = 0; + int NOUT = 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + + // starting disposition + int isUp = isFltY ? (vecy[0].elements.Flt[BinMin] > value) : (vecy[0].elements.Int[BinMin] > value); + + for (int i = BinMin; i <= BinMax; i++) { + double testval = isFltY ? vecy[0].elements.Flt[i] : vecy[0].elements.Int[i]; + if (isUp && (testval < value)) { + isUp = FALSE; + if (!UP_TRANSITIONS) { + // save this value + outbin->elements.Int[Nout] = i; + if (isFltY) { + outval->elements.Flt[Nout] = testval; + } else { + outval->elements.Int[Nout] = testval; + } + outpos->elements.Flt[Nout] = interpolate_position (vecx, vecy, i, value); + Nout ++; + if (Nout == NOUT) { + NOUT += 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + } + } + continue; + } + if (!isUp && (testval > value)) { + isUp = TRUE; + if (UP_TRANSITIONS) { + // save this value + outbin->elements.Int[Nout] = i; + if (isFltY) { + outval->elements.Flt[Nout] = testval; + } else { + outval->elements.Int[Nout] = testval; + } + outpos->elements.Flt[Nout] = interpolate_position (vecx, vecy, i, value); + Nout ++; + if (Nout == NOUT) { + NOUT += 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + } + } + } + } + + ResetVector (outbin, OPIHI_INT, Nout); + ResetVector (outval, vecy[0].type, Nout); + ResetVector (outpos, OPIHI_FLT, Nout); + + if (!QUIET) gprint (GP_LOG, "found %d transitions\n", Nout); + + return (TRUE); +} + +static double interpolate_position (Vector *vecx, Vector *vecy, int bin, float value) { + + double x0, x1, y0, y1, Xvalue; + + if (vecy[0].Nelements == 0) return NAN; + if (vecy[0].Nelements == 1) return isFltY ? vecy[0].elements.Flt[0] : vecy[0].elements.Int[0]; + + if (bin == 0) { + // interpolate to value: + y0 = isFltY ? vecy[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + y1 = isFltY ? vecy[0].elements.Flt[bin+1] : vecy[0].elements.Int[bin+1]; + x0 = isFltX ? vecx[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + x1 = isFltX ? vecx[0].elements.Flt[bin+1] : vecy[0].elements.Int[bin+1]; + if (y0 == y1) { + Xvalue = 0.5*(x0 + x1); + } else { + Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0; + } + } else { + // interpolate to value: + y0 = isFltY ? vecy[0].elements.Flt[bin-1] : vecy[0].elements.Int[bin-1]; + y1 = isFltY ? vecy[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + x0 = isFltX ? vecx[0].elements.Flt[bin-1] : vecy[0].elements.Int[bin-1]; + x1 = isFltX ? vecx[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + if (y0 == y1) { + Xvalue = 0.5*(x0 + x1); + } else { + Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0; + } + } + return Xvalue; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c (revision 41170) @@ -0,0 +1,70 @@ +# include "data.h" + +double opihi_wt_mean_dbl (double *val, double *wgt, int N, double *Sigma); + +int vwtmean (int argc, char **argv) { + + if (argc != 3) { + gprint (GP_ERR, "USAGE: vwtmean (values) (sigmas)\n"); + gprint (GP_ERR, "NOTE: sigmas with abs values < %e will be truncated to that value\n", FLT_MIN); + return (FALSE); + } + + Vector *value, *sigma; + + if ((value = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((sigma = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if (value->Nelements != sigma->Nelements) { + gprint (GP_ERR, "vectors are not the same length\n"); + return FALSE; + } + if (value->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[1]); + return FALSE; + } + if (sigma->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[2]); + return FALSE; + } + + ALLOCATE_PTR (val, opihi_flt, value->Nelements); + ALLOCATE_PTR (wgt, opihi_flt, value->Nelements); + + int N = 0; + for (int i = 0; i < value->Nelements; i++) { + if (!isfinite(value->elements.Flt[i])) continue; + + double s = sigma->elements.Flt[i]; + if (!isfinite(s)) continue; + + if (fabs(s) < FLT_MIN) s = FLT_MIN; + + val[N] = value->elements.Flt[i]; + wgt[N] = 1.0 / SQ(s); + N ++; + } + + double Sigma = NAN; + double Value = opihi_wt_mean_dbl (val, wgt, N, &Sigma); + + set_variable ("WTMEAN", Value); + set_variable ("WTSIGMA", Sigma); + set_int_variable ("NUSED", N); + return (TRUE); +} + +double opihi_wt_mean_dbl (double *val, double *wgt, int N, double *Sigma) { + + // calculate weighted mean + double S1 = 0.0, S2 = 0.0; + for (int n = 0; n < N; n++) { + S1 += wgt[n] * val[n]; + S2 += wgt[n]; + } + double Value = S1 / S2; + *Sigma = 1.0 / S2; + + return Value; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c (revision 41170) @@ -32,12 +32,23 @@ } - /* option generate a FITS output table */ + // option generate a FITS output table (FITS holds the filename) + // in FITS output context, -header is interpretted as a buffer containing + // header keywords to supplement the FITS table header char *FITS = NULL; + Header *fitsheader = NULL; + Buffer *headbuffer = NULL; if ((N = get_argument (argc, argv, "-fits"))) { remove_argument (N, &argc, argv); FITS = strcreate (argv[N]); remove_argument (N, &argc, argv); - } - + + if ((N = get_argument (argc, argv, "-header"))) { + remove_argument (N, &argc, argv); + if ((headbuffer = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + fitsheader = &headbuffer->header; + remove_argument (N, &argc, argv); + } + } + /* option generate a FITS output table */ int CSV = FALSE; @@ -124,5 +135,5 @@ if (FITS) { - int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, compress, format, Ntile); + int status = WriteVectorTableFITS (argv[1], FITS, fitsheader, vec, Nvec, append, compress, format, Ntile); free (vec); return status; @@ -260,4 +271,6 @@ gprint (GP_ERR, " -append : write to the end of the existing file\n"); gprint (GP_ERR, " -fits NAME : write a fits table (extention name is NAME, column names match vector names)\n"); + gprint (GP_ERR, " in FITS output context, -header takes an additional argument which is interpretted\n"); + gprint (GP_ERR, " as a buffer containing header keywords to supplement the FITS table header\n"); gprint (GP_ERR, " -csv : write a comma-separated values file (eg, to read in excel)\n"); gprint (GP_ERR, " -f \"format\" : provide formatting codes for output:\n"); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile (revision 41170) @@ -67,4 +67,5 @@ $(SRC)/gstar.$(ARCH).o \ $(SRC)/hosts.$(ARCH).o \ +$(SRC)/cimages.$(ARCH).o \ $(SRC)/images.$(ARCH).o \ $(SRC)/imbox.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c (revision 41170) @@ -272,5 +272,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (RESULT_FILE && !SKIP_RESULTS) { - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nreturn, FALSE, FALSE, NULL, 0); if (!status) { goto escape; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c (revision 41170) @@ -114,5 +114,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -301,5 +301,5 @@ vec[i][0].Nelements = Nfound; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nfields + 1, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nfields + 1, FALSE, FALSE, NULL, 0); free (vec[Nfields]->elements.Int); free (vec[Nfields]); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c (revision 41170) @@ -99,5 +99,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; free (vec); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c (revision 41170) @@ -132,5 +132,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -327,5 +327,5 @@ vec[NfieldsOut-1] = RADvec; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, NfieldsOut, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, NfieldsOut, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c (revision 41170) @@ -0,0 +1,379 @@ +# include "dvoshell.h" +# define BETA 0.41421 + +int wordhash (char *word); + +int cimages (int argc, char **argv) { + + off_t i, Nimage, Nmosaic; + int j, status, InPic, leftside, TimeSelect, ByName; + int WITH_MOSAIC, SOLO_MOSAIC; + time_t tzero, tend; + int n, N, NPTS, Npts, kapa, *foundMosaic; + Vector Xvec, Yvec, Zvec; + double r[8], d[8], x[8], y[8], Rmin, Rmax, Rmid, trange, Radius; + Image *image; + Graphdata graphmode; + char name[256]; + int typehash; + PhotCode *photcode; + int Nsec, photcodeEquiv; + + if (!InitPhotcodes ()) return (FALSE); + if (!style_args (&graphmode, &argc, argv, &kapa)) goto usage; + + WITH_MOSAIC = FALSE; + if ((N = get_argument (argc, argv, "+mosaic"))) { + remove_argument (N, &argc, argv); + WITH_MOSAIC = TRUE; + } + + SOLO_MOSAIC = FALSE; + foundMosaic = NULL; + if ((N = get_argument (argc, argv, "-mosaic"))) { + remove_argument (N, &argc, argv); + SOLO_MOSAIC = TRUE; + WITH_MOSAIC = TRUE; + } + + // int HIDDEN = FALSE; + // if ((N = get_argument (argc, argv, "-hidden"))) { + // remove_argument (N, &argc, argv); + // HIDDEN = TRUE; + // } + + photcode = NULL; + photcodeEquiv = FALSE; + if ((N = get_argument (argc, argv, "-p"))) { + remove_argument (N, &argc, argv); + photcode = GetPhotcodebyName (argv[N]); + if (photcode == NULL) { + gprint (GP_ERR, "photcode %s not found\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + Nsec = GetPhotcodeNsec (photcode->code); + if (Nsec != -1) photcodeEquiv = TRUE; + } + if ((N = get_argument (argc, argv, "-photcode"))) { + remove_argument (N, &argc, argv); + photcode = GetPhotcodebyName (argv[N]); + if (photcode == NULL) { + gprint (GP_ERR, "photcode %s not found\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + Nsec = GetPhotcodeNsec (photcode->code); + if (Nsec != -1) photcodeEquiv = TRUE; + } + + ByName = FALSE; + if ((N = get_argument (argc, argv, "-name"))) { + remove_argument (N, &argc, argv); + strcpy (name, argv[N]); + remove_argument (N, &argc, argv); + ByName = TRUE; + } + + Radius = 45; + if ((N = get_argument (argc, argv, "-radius"))) { + remove_argument (N, &argc, argv); + Radius = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + TimeSelect = FALSE; + if ((N = get_argument (argc, argv, "-time"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tzero)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_dtime (argv[N], &trange)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (trange < 0) { + trange = fabs (trange); + tzero -= trange; + } + TimeSelect = TRUE; + } + if ((N = get_argument (argc, argv, "-trange"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tzero)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tend)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + trange = tend - tzero; + if (trange < 0) { + trange = fabs (trange); + tzero -= trange; + } + TimeSelect = TRUE; + } + + if (argc != 3) goto usage; + float vmin = atof (argv[1]); + float vmax = atof (argv[2]); + + /* it is not an error for the database not to have any images */ + if ((image = LoadImagesDVO (&Nimage)) == NULL) return (TRUE); + // BuildChipMatch (image, Nimage); + + if (SOLO_MOSAIC && photcode) { + ALLOCATE(foundMosaic, int, Nimage); + memset(foundMosaic, 0, Nimage*sizeof(int)); + } + + Rmin = graphmode.coords.crval1 - 180.0; + Rmax = graphmode.coords.crval1 + 180.0; + Rmid = 0.5*(Rmin + Rmax); + + int DistortImage = wordhash ("-DIS"); + int ChipImage = wordhash ("-WRP"); + int TriangleUp = wordhash ("TRP-"); + int TriangleDn = wordhash ("TRM-"); + int TrianglePts = wordhash ("TRI-"); + + NPTS = 200; + SetVector (&Xvec, OPIHI_FLT, NPTS); + SetVector (&Yvec, OPIHI_FLT, NPTS); + SetVector (&Zvec, OPIHI_FLT, NPTS); + + Image *mosaic = NULL; + + /******** stage 1 : find the images to plot *********/ + + n = 0; // tracks number of images + N = 0; // tracks number of vertices + for (i = 0; i < Nimage; i++) { + if (ByName && strncmp (image[i].name, name, strlen(name))) continue; + if (TimeSelect && ((image[i].tzero < tzero) || (image[i].tzero+image[i].trate*image[i].NY > tzero + trange))) continue; + + mosaic = image[i].parent; + Nmosaic = (SOLO_MOSAIC && mosaic) ? mosaic - image : -1; + + if (photcode) { + if ( photcodeEquiv && (photcode[0].code != GetPhotcodeEquivCodebyCode(image[i].photcode))) continue; + if (!photcodeEquiv && (photcode[0].code != image[i].photcode)) continue; + } + + Npts = 4; + status = TRUE; + leftside = FALSE; + + typehash = wordhash (&image[i].coords.ctype[4]); + + if (photcode && SOLO_MOSAIC && (Nmosaic >= 0)) { + // mosaic (DIS) images are not currently given a photcode : plot these via the WRP entries + /* DIS images represent a field, not a chip */ + if (typehash != ChipImage) continue; + if (foundMosaic[Nmosaic]) continue; + x[0] = -0.5*mosaic->NX; y[0] = -0.5*mosaic->NY; + x[1] = +0.5*mosaic->NX; y[1] = -0.5*mosaic->NY; + x[2] = +0.5*mosaic->NX; y[2] = +0.5*mosaic->NY; + x[3] = -0.5*mosaic->NX; y[3] = +0.5*mosaic->NY; + for (j = 0; j < Npts; j++) { + status = XY_to_RD (&r[j], &d[j], x[j], y[j], &mosaic->coords); + if (!status) break; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + } + foundMosaic[Nmosaic] = TRUE; + goto plot_points; + } + + /* DIS images represent a field, not a chip */ + if ((typehash == DistortImage) && !WITH_MOSAIC) continue; // do not plot the mosaic images + if ((typehash != DistortImage) && SOLO_MOSAIC) continue; // plot only the mosaic images + + if (typehash == DistortImage) { + x[0] = -0.5*image[i].NX; y[0] = -0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = -0.5*image[i].NY; + x[2] = +0.5*image[i].NX; y[2] = +0.5*image[i].NY; + x[3] = -0.5*image[i].NX; y[3] = +0.5*image[i].NY; + goto got_type; + } + + typehash = wordhash (image[i].coords.ctype); + if (typehash == TriangleUp) { + Npts = 3; + x[0] = 0; y[0] = +0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = -0.5*image[i].NY; + x[2] = -0.5*image[i].NX; y[2] = -0.5*image[i].NY; + goto got_type; + } + if (typehash == TriangleDn) { + Npts = 3; + x[0] = 0; y[0] = -0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = +0.5*image[i].NY; + x[2] = -0.5*image[i].NX; y[2] = +0.5*image[i].NY; + goto got_type; + } + // For 'TrianglePts' (TRI-), we are absurdly overloaded values otherwise not used + if (typehash == TrianglePts) { + Npts = 3; + x[0] = image[0].dXpixSys; y[0] = image[0].dYpixSys; + x[1] = image[0].dMagSys; y[1] = image[0].nFitAstrom; + x[2] = image[0].photom_map_id; y[2] = image[0].astrom_map_id; + goto got_type; + } + + // default layout + { + x[0] = 0; y[0] = 0; + x[1] = image[i].NX; y[1] = 0; + x[2] = image[i].NX; y[2] = image[i].NY; + x[3] = 0; y[3] = image[i].NY; + } + + got_type: + + /**** project this image to screen display coords ****/ + + // check for boundary overlap? + for (j = 0; j < Npts; j++) { + status = XY_to_RD (&r[j], &d[j], x[j], y[j], &image[i].coords); + if (!status) break; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + } + + // extremely large-scale images with certain projections will have odd boundaries. + // eg, the ASCA images are essentially circles of radius ~60 degrees. plot these as + // octagons with some dummy size. + if (!status) { + int jp, xo, yo; + double rc, dc; + xo = 0.5*image[i].NX; + yo = 0.5*image[i].NY; + // is the image center on the screen? + status = XY_to_RD (&rc, &dc, 0.5*x[2], 0.5*y[2], &image[i].coords); + if (status && (image[i].NX * image[i].coords.cdelt2 > 90)) { + // draw an octagon with radius 45 degrees + double dX = Radius / image[i].coords.cdelt2; + x[0] = xo+dX; y[0] = yo+BETA*dX; + x[1] = xo+BETA*dX; y[1] = yo+dX; + x[2] = xo-BETA*dX; y[2] = yo+dX; + x[3] = xo-dX; y[3] = yo+BETA*dX; + x[4] = xo-dX; y[4] = yo-BETA*dX; + x[5] = xo-BETA*dX; y[5] = yo-dX; + x[6] = xo+BETA*dX; y[6] = yo-dX; + x[7] = xo+dX; y[7] = yo-BETA*dX; + Npts = 8; + j = 0; + for (jp = 0; jp < 8; jp++) { + status = XY_to_RD (&r[j], &d[j], x[jp], y[jp], &image[i].coords); + if (!status) continue; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + j++; + } + Npts = j; + } else { + continue; + } + } + if (Npts == 0) continue; + + plot_points: + + /**** transform corners to screen coordinates ******/ + + status = FALSE; + for (j = 0; j < Npts; j++) { + status |= RD_to_XY (&Xvec.elements.Flt[N+j], &Yvec.elements.Flt[N+j], r[j], d[j], &graphmode.coords); + Zvec.elements.Flt[N+j] = MIN(1.0, MAX(0.01, (image[i].exptime - vmin) / (vmax - vmin))); + } + if (!status) continue; + // if none of the points are on the visible side of the projection, do not plot the image + + InPic = FALSE; + for (j = 0; j < Npts; j++) { + if ((Xvec.elements.Flt[N+j] >= graphmode.xmin) && + (Xvec.elements.Flt[N+j] <= graphmode.xmax) && + (Yvec.elements.Flt[N+j] >= graphmode.ymin) && + (Yvec.elements.Flt[N+j] <= graphmode.ymax)) + InPic = TRUE; + } + // if (!status && HIDDEN) status = TRUE; + if (!InPic) continue; + + n ++; + N += Npts; + if (N + 16 >= NPTS) { /* need to leave room for 8 point image */ + NPTS += 400; + REALLOCATE (Xvec.elements.Flt, opihi_flt, NPTS); + REALLOCATE (Yvec.elements.Flt, opihi_flt, NPTS); + REALLOCATE (Zvec.elements.Flt, opihi_flt, NPTS); + } + } + + gprint (GP_ERR, "plotting %d images\n", n); + Xvec.Nelements = Yvec.Nelements = Zvec.Nelements = N; + if (N > 0) { + // XXX note that ptype limits us to plot only a single type of + // image in a single pass + graphmode.style = KAPA_PLOT_POLYFILL; /* points */ + graphmode.ptype = Npts; /* connect pairs of points */ + graphmode.etype = 0; + graphmode.size = -1; /* point size determined by Zvec */ + graphmode.color = -1; // use z-vec for color + PlotVectorTriplet (kapa, &Xvec, &Yvec, &Zvec, NULL, &graphmode); +// PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); + } + + FREE (Xvec.elements.Flt); + FREE (Yvec.elements.Flt); + FREE (Zvec.elements.Flt); + FreeImagesDVO (image); + FREE (foundMosaic); + return (TRUE); + + + usage: + gprint (GP_ERR, "USAGE: cimage (vmin) (vmax) [options]\n"); + gprint (GP_ERR, " +mosaic : show mosaic outline\n"); + gprint (GP_ERR, " -mosaic : only mosaic outline\n"); + gprint (GP_ERR, " -hidden : (deprecated)\n"); + gprint (GP_ERR, " -name : only names matching (start of name)\n"); + gprint (GP_ERR, " -radius : display all-sky images with given radius octagon\n"); + gprint (GP_ERR, " -time (start) (range) : show images for time and interval\n"); + gprint (GP_ERR, " -trange (start) (stop) : show images within time range\n"); + return (FALSE); +} + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 41170) @@ -233,5 +233,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (ResultFile) { - int status = WriteVectorTableFITS (ResultFile, "RESULT", vec, Nvec, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (ResultFile, "RESULT", NULL, vec, Nvec, FALSE, FALSE, NULL, 0); if (!status) { gprint (GP_ERR, "failed to write result file %s\n", ResultFile); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c (revision 41170) @@ -12,8 +12,9 @@ off_t i, Nimage, Nmosaic; - int j, status, InPic, leftside, *plist, TimeSelect, ByName; + int j, status, InPic, leftside, TimeSelect, ByName; + // int *plist, n, npts; -- what is/was plist? int WITH_MOSAIC, SOLO_MOSAIC; time_t tzero, tend; - int N, NPTS, n, npts, Npts, kapa, *foundMosaic; + int n, N, NPTS, Npts, kapa, *foundMosaic; Vector Xvec, Yvec; double r[8], d[8], x[8], y[8], Rmin, Rmax, Rmid, trange, Radius; @@ -148,5 +149,6 @@ int TrianglePts = wordhash ("TRI-"); - npts = NPTS = 200; + // npts = 200 + NPTS = 200; SetVector (&Xvec, OPIHI_FLT, NPTS); SetVector (&Yvec, OPIHI_FLT, NPTS); @@ -154,6 +156,9 @@ Image *mosaic = NULL; - ALLOCATE (plist, int, NPTS); - n = N = 0; + /******** stage 1 : find the images to plot *********/ + + // ALLOCATE (plist, int, NPTS); + n = 0; // tracks number of images + N = 0; // tracks number of vertices for (i = 0; i < Nimage; i++) { if (ByName && strncmp (image[i].name, name, strlen(name))) continue; @@ -335,11 +340,12 @@ if (!InPic) continue; - plist[n] = i; - n++; - if (n > npts - 1) { - npts += 200; - REALLOCATE (plist, int, npts); - } - N+=2*Npts; + // plist[n] = i; + // n++; + // if (n > npts - 1) { + // npts += 200; + // REALLOCATE (plist, int, npts); + // } + n ++; + N += 2*Npts; if (N + 16 >= NPTS) { /* need to leave room for 8 point image */ NPTS += 400; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c (revision 41170) @@ -14,4 +14,5 @@ int catdir_define PROTO((int, char **)); int ccd PROTO((int, char **)); +int cimages PROTO((int, char **)); int cmatch PROTO((int, char **)); int cmd PROTO((int, char **)); @@ -80,4 +81,5 @@ {1, "catdir", catdir_define,"re-define CATDIR"}, // {1, "ccd", ccd, "plot color-color diagram"}, + {1, "cimages", cimages, "fill image boxes with a color"}, {1, "cmatch", cmatch, "match two catalogs"}, // {1, "cmd", cmd, "plot cmd of stars in current region"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c (revision 41170) @@ -352,5 +352,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (RESULT_FILE && !SKIP_RESULTS) { - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nreturn, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c (revision 41170) @@ -148,5 +148,5 @@ // XXX this is now set for both cases... CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -349,5 +349,5 @@ vec[Nfields-1] = IDXvec; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nfields, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nfields, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h (revision 41170) @@ -48,5 +48,6 @@ int Nx; int Ny; - float **buffers; + float **flx; + float **var; } MedImageType; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h (revision 41170) @@ -7,5 +7,5 @@ } DeimosResult; -float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *trace, int Nx, int Ny, int row); +float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, Spline *psf_trace, int Nx, int Ny, int row); void deimos_make_kernel (float stilt, int Nx); void deimos_free_kernel (); @@ -13,8 +13,8 @@ // internal to make_model: -float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, int Nx, int Ny, int row); +float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, Spline *psf_trace, int Nx, int Ny, int row); float *deimos_apply_tilt (float *input, int Nx, int Ny); void deimos_apply_profile (Vector *profile, float *out, int Nx, int Ny); void deimos_add_background (opihi_flt *backgnd, float *out, int Nx, int Ny, int row); -float *deimos_apply_trace (Spline *trace, float *input, int Nx, int Ny, int row); +float *deimos_apply_trace (Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, float *input, int Nx, int Ny, int row); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h (revision 41170) @@ -169,5 +169,5 @@ /* vector IO functions */ -int WriteVectorTableFITS PROTO((char *filename, char *extname, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile)); +int WriteVectorTableFITS PROTO((char *filename, char *extname, Header *extraheader, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile)); Vector **ReadVectorTableFITS PROTO((char *filename, char *extname, int *Nvec)); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c (revision 41170) @@ -32,7 +32,9 @@ free (medimage[0].name); for (i = 0; i < medimage[0].Ninput; i++) { - free (medimage[0].buffers[i]); + free (medimage[0].flx[i]); + FREE (medimage[0].var[i]); } - free (medimage[0].buffers); + free (medimage[0].flx); + free (medimage[0].var); free (medimage); } @@ -75,5 +77,6 @@ medimage->Nx = Nx; medimage->Ny = Ny; - ALLOCATE (medimage->buffers, float *, 1); + ALLOCATE (medimage->flx, float *, 1); + ALLOCATE (medimage->var, float *, 1); medimages[N] = medimage; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c (revision 41170) @@ -41,7 +41,9 @@ ALLOCATE (spline[0].y2, opihi_flt, spline[0].Nknots); } - memset (spline[0].xk, 0, spline[0].Nknots * sizeof(opihi_flt)); - memset (spline[0].yk, 0, spline[0].Nknots * sizeof(opihi_flt)); - memset (spline[0].y2, 0, spline[0].Nknots * sizeof(opihi_flt)); + if (spline[0].Nknots) { + memset (spline[0].xk, 0, spline[0].Nknots * sizeof(opihi_flt)); + memset (spline[0].yk, 0, spline[0].Nknots * sizeof(opihi_flt)); + memset (spline[0].y2, 0, spline[0].Nknots * sizeof(opihi_flt)); + } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c (revision 41170) @@ -266,4 +266,5 @@ int close_kapa (char *name) { + struct timespec request, remain; int N; @@ -274,4 +275,24 @@ } DelKapaDevice (name); + + // avoid blocking on waitpid, test every 1 msec, up to 500 msec + request.tv_sec = 0; + request.tv_nsec = 1000000; + + // try to harvest the child PID + int waitstatus = 0; + int result = waitpid (-1, &waitstatus, WNOHANG); + for (int i = 0; (i < 500) && (result == 0); i++) { + nanosleep (&request, &remain); + result = waitpid (-1, &waitstatus, WNOHANG); + } + + if ((result == -1) && (errno != ECHILD)) { + fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno); + } + if (result == 0) { + fprintf (stderr, "child did not exit (close_kapa), timeout"); + } + return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c (revision 41170) @@ -7,4 +7,7 @@ float dy, dx, *tmp; + // spline is not valid with < 3 points + if (N < 3) return; + ALLOCATE (tmp, float, N); @@ -69,4 +72,7 @@ opihi_flt dy, dx, *tmp; + // spline is not valid with < 3 points + if (N < 3) return; + ALLOCATE (tmp, opihi_flt, N); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c (revision 41170) @@ -75,4 +75,6 @@ FWHMx = 2.355*sqrt (fabs(x2 / I - x*x)); FWHMy = 2.355*sqrt (fabs(y2 / I - y*y)); + + fprintf (stderr, "Mxx, Myy: %f, %f\n", x2/I - x*x, y2/I - y*y); Sxy = xy / I - x*y; mag = -2.5*log10(I); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c (revision 41170) @@ -72,4 +72,14 @@ } + if ((graphmode[0].style == KAPA_PLOT_POLYGON) || (graphmode[0].style == KAPA_PLOT_POLYFILL)) { + if ((N = get_argument (*argc, argv, "-npoint"))) { + remove_argument (N, argc, argv); + graphmode[0].ptype = atoi (argv[N]); + remove_argument (N, argc, argv); + } else { + gprint (GP_ERR, "polygon & polyfill styles require number of points argument: -npoint N\n"); + return FALSE; + } + } return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c (revision 41170) @@ -162,6 +162,50 @@ } +// insert a new header line, return end pointer +char *gfits_insert_header_line (Header *header, char *endptr, char *newline) { + + if (0) { + char temp1[32], temp2[32]; + memset (temp1, 0, 32); + memset (temp2, 0, 32); + + if (endptr) { + memcpy (temp1, endptr, 8); + } + memcpy (temp2, newline, 8); + fprintf (stderr, "insert: %s : %s\n", temp1, temp2); + } + + /* find the END of the header, if not supplied */ + if (!endptr) { + endptr = gfits_header_field (header, "END", 1); + if (endptr == NULL) return NULL; + } + + /* is there enough space for 1 more line? */ + if (header[0].datasize - (endptr - (header[0].buffer)) < 2*FT_LINE_LENGTH) { + // no, so expand by a full header block (FT_RECORD_SIZE = 32*80 = 2880) + header[0].datasize += FT_RECORD_SIZE; + REALLOCATE (header[0].buffer, char, header[0].datasize); + // re-find the "END" marker, in case new memory block is used + endptr = gfits_header_field (header, "END", 1); + if (endptr == NULL) return NULL; + memset (endptr + FT_LINE_LENGTH, ' ', FT_RECORD_SIZE); + } + + /* push END line back 1 */ + memmove ((endptr + FT_LINE_LENGTH), endptr, FT_LINE_LENGTH); + memset (endptr, ' ', FT_LINE_LENGTH); + + strncpy (endptr, newline, 80); + endptr += FT_LINE_LENGTH; + + return endptr; +} + +static char *rawkeywords[] = {"SIMPLE", "BITPIX", "NAXIS", "PCOUNT", "GCOUNT", "EXTEND", "EXTNAME", "BSCALE", "BZERO", "TFIELDS", "TFORM", "TZERO", "TSCAL", " ", NULL}; + // write a set of vectors to a FITS file (vectors names become fits column names) -int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) { +int WriteVectorTableFITS (char *filename, char *extname, Header *extraheader, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) { Header rawheader; @@ -191,5 +235,5 @@ rawtable.header = &rawheader; if (!WriteVectorTable (&rawtable, extname, vec, Nvec, format, (compress != NULL))) goto escape; - // NOTE : for compression, the table is constructed in native by order + // NOTE : for compression, the table is constructed in native byte order FTable *outtable = &rawtable; @@ -215,4 +259,5 @@ if (!append) { + // generate a blank PHU header Header header; Matrix matrix; @@ -228,4 +273,30 @@ } + if (extraheader) { + // copy keywords which are not the standard or table keywords + char *buf = extraheader->buffer; + char *endptr = NULL; + for (int i = 0; i < extraheader->datasize; i+= FT_LINE_LENGTH, buf += FT_LINE_LENGTH) { + + if (0) { + char temp1[32]; + memset (temp1, 0, 32); + memcpy (temp1, buf, 8); + fprintf (stderr, "buffer: %s\n", temp1); + } + + for (int j = 0; rawkeywords[j] != NULL; j++) { + if (!strncmp (buf, rawkeywords[j], strlen(rawkeywords[j]))) goto skip_insert; + } + endptr = gfits_insert_header_line (outheader, endptr, buf); + if (!endptr) { + gprint (GP_ERR, "failed to update FITS header with extra keywords\n"); + return (FALSE); + } + skip_insert: + continue; + } + } + // write the actual table data gfits_fwrite_Theader (f, outheader); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c (revision 41170) @@ -63,4 +63,6 @@ p = line; + int Nline = strlen(line); + done = FALSE; status = TRUE; @@ -74,82 +76,95 @@ tmpline = strncreate (p, q - p); stripwhite (tmpline); - if (*tmpline) { + myAssert (tmpline, "oops"); - if (bufferPending) { - // flush old messages - ExpectMessage (server, 0.2, &message); - bufferPending = FALSE; + // empty command, free and continue + if (*tmpline == 0) { + free (tmpline); + if (q == line + Nline) { + done = TRUE; + } else { + p = q + 1; + myAssert (p - line <= Nline, "oops"); } + continue; + } - status = command (tmpline, &outline, verbose); + if (bufferPending) { + // flush old messages + ExpectMessage (server, 0.2, &message); + bufferPending = FALSE; + } - if (status == -1) { - if (server) { - // send the command to the server instead - if (!SendMessage (server, "%s", outline)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (32); - default: - gprint (GP_ERR, "server is busy...32\n"); - bufferPending = TRUE; - goto escape; - } + // input tmpline is freed by command + status = command (tmpline, &outline, verbose); + + if (status == -1) { + if (server) { + // send the command to the server instead + if (!SendMessage (server, "%s", outline)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (32); + default: + gprint (GP_ERR, "server is busy...32\n"); + bufferPending = TRUE; + goto escape; } + } - // receive the command exit status - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (33); - default: - gprint (GP_ERR, "server is busy...33\n"); - bufferPending = TRUE; - goto escape; - } - } else { - sscanf (message.buffer, "STATUS %d", &status); - } - - // receive the resulting stderr - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (34); - default: - gprint (GP_ERR, "server is busy...34\n"); - bufferPending = TRUE; - goto escape; - } - } else { - gwrite (message.buffer, 1, message.Nbuffer, GP_ERR); - } - - // receive the resulting stdout - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (35); - default: - gprint (GP_ERR, "server is busy...35\n"); - bufferPending = TRUE; - goto escape; - } - } else { - gwrite (message.buffer, 1, message.Nbuffer, GP_LOG); + // receive the command exit status + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (33); + default: + gprint (GP_ERR, "server is busy...33\n"); + bufferPending = TRUE; + goto escape; } } else { - // if no server is defined, we treat an unknown command as an error - status = FALSE; - } - } - escape: - if (outline != NULL) free (outline); - if (!status && auto_break) done = TRUE; + sscanf (message.buffer, "STATUS %d", &status); + } + + // receive the resulting stderr + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (34); + default: + gprint (GP_ERR, "server is busy...34\n"); + bufferPending = TRUE; + goto escape; + } + } else { + gwrite (message.buffer, 1, message.Nbuffer, GP_ERR); + } + + // receive the resulting stdout + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (35); + default: + gprint (GP_ERR, "server is busy...35\n"); + bufferPending = TRUE; + goto escape; + } + } else { + gwrite (message.buffer, 1, message.Nbuffer, GP_LOG); + } + } else { + // if no server is defined, we treat an unknown command as an error + status = FALSE; + } } + escape: + if (outline != NULL) free (outline); + if (!status && auto_break) done = TRUE; + p = q + 1; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c (revision 41170) @@ -107,4 +107,16 @@ LOAD_RC = FALSE; } + if ((N = get_argument (*argc, argv, "--no-rc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } + if ((N = get_argument (*argc, argv, "-norc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } + if ((N = get_argument (*argc, argv, "-no-rc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } ONLY_INPUT = FALSE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile (revision 41170) @@ -26,8 +26,8 @@ $(SRC)/deimos_getobj.$(ARCH).o \ $(SRC)/deimos_fitobj.$(ARCH).o \ -$(SRC)/deimos_fitalt.$(ARCH).o \ -$(SRC)/deimos_mkalt.$(ARCH).o \ $(SRC)/deimos_mkmodel.$(ARCH).o \ $(SRC)/deimos_arclines.$(ARCH).o \ +$(SRC)/deimos_fitarc.$(ARCH).o \ +$(SRC)/deimos_fitprofile.$(ARCH).o \ $(SRC)/findrowpeaks.$(ARCH).o Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c (revision 41170) @@ -4,19 +4,23 @@ int deimos_fitslit (int argc, char **argv); int deimos_mkobj (int argc, char **argv); -int deimos_mkalt (int argc, char **argv); int deimos_getobj (int argc, char **argv); +// int deimos_getalt (int argc, char **argv); int deimos_fitobj (int argc, char **argv); -int deimos_fitalt (int argc, char **argv); +// int deimos_fitalt (int argc, char **argv); int deimos_arclines (int argc, char **argv); +int deimos_fitarc (int argc, char **argv); +int deimos_fitprofile (int argc, char **argv); static Command deimos_commands[] = { - {1, "fitobj", deimos_fitobj, "fit for object parameters"}, - {1, "fitalt", deimos_fitalt, "fit for object parameters using LMM"}, - {1, "getobj", deimos_getobj, "determine crude object parameters"}, - {1, "mkobj", deimos_mkobj, "make a full object image"}, - {1, "mkalt", deimos_mkalt, "make a full object image (uses deimos_make_object, for a test)"}, - {1, "mkslit", deimos_mkslit, "make a slit image"}, - {1, "fitslit", deimos_fitslit, "fit slit image to observed slit flux"}, - {1, "arclines", deimos_arclines, "detect arclines using LSF and STILT"}, + {1, "fitobj", deimos_fitobj, "fit for object parameters using LMM"}, + {1, "fitalt", deimos_fitobj, "fit for object parameters using LMM"}, + {1, "getobj", deimos_getobj, "determine crude object parameters"}, + {1, "getalt", deimos_getobj, "determine crude object parameters"}, + {1, "mkobj", deimos_mkobj, "make a full object image"}, + {1, "mkslit", deimos_mkslit, "make a slit image"}, + {1, "fitslit", deimos_fitslit, "fit slit image to observed slit flux"}, + {1, "arclines", deimos_arclines, "detect arclines using LSF and STILT"}, + {1, "fitarc", deimos_fitarc, "fit arclamp lines"}, + {1, "fitprofile", deimos_fitprofile, "fit slit profile"}, }; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c (revision 41170) @@ -53,7 +53,14 @@ deimos_make_LSF_kernel (LSF, stilt, Nx); + // FILE *Fout = NULL; + // loop over the rows for (int iy = 0; iy < Ny; iy++) { float Fsum = 0; + float Ksum = 0; + + // if (iy == 5802) Fout = fopen ("test.5802.dat", "w"); + // if (iy == 5815) Fout = fopen ("test.5815.dat", "w"); + // cross-correlation of buffer values and kernel values for (int ix = 0; ix < Nx; ix++) { @@ -62,7 +69,16 @@ if ((iy + ky) < 0) continue; if ((iy + ky) >= Ny) continue; + if (!isfinite(Fin[ix + (iy + ky)*Nx])) continue; + if (!isfinite(kernel[ix + ko*Nx])) continue; Fsum += Fin[ix + (iy + ky)*Nx] * kernel[ix + ko*Nx]; + Ksum += kernel[ix + ko*Nx]; + // if ((iy == 5815) || (iy == 5802)) { + // fprintf (Fout, "%d %d %d : %f %f\n", iy, ix, ky, Fin[ix + (iy + ky)*Nx], kernel[ix + ko*Nx]); + // } } } + + // if ((iy == 5815) || (iy == 5802)) fclose (Fout); + coord->elements.Flt[iy] = iy; flux->elements.Flt[iy] = Fsum; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitalt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitalt.c (revision 41144) +++ (revision ) @@ -1,447 +1,0 @@ -# include "data.h" -# include "deimos.h" - -# if (0) -int deimos_fitalt (int argc, char **argv) { - OHANA_UNUSED_PARAM(argc); - OHANA_UNUSED_PARAM(argv); - return FALSE; -} - -# else - -/* - this is starting to work OK. some improvements to make - * use Gaussdev to sample - * do not scale down range (or user-set scale-down) - */ - -// internal functions to fitobj -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); -double deimos_LMM_update (float *data, float *model_ref, float *model_obj, float *model_sky, float *model_bck, float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda); -static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue); -opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow); - -int deimos_fitalt (int argc, char **argv) { - - // input parameters: - // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * PSF : point-spread function vector (flux normalized, x-dir) - // * stilt : slit tilt response : 2D kernel? - - // in-out parameters: - // * obj : vector of object flux vs y-coord (starting guess and result) - // * sky : vector of local sky signal vs y-coord - // * bck : vector of extra-slit background flux vs y-coord - - int N; - - Spline *trace = NULL; - Vector *profile = NULL; - - Vector *obj = NULL; - Vector *sky = NULL; - Vector *bck = NULL; - - Vector *psf = NULL; - - Buffer *buffer = NULL; - - float stilt = 0.0; // angle of the slit - ohana_gaussdev_init (); - - int DO_PLUS = FALSE; - if ((N = get_argument (argc, argv, "-plus"))) { - remove_argument (N, &argc, argv); - DO_PLUS = TRUE; - } - - int VERBOSE = FALSE; - if ((N = get_argument (argc, argv, "-v"))) { - remove_argument (N, &argc, argv); - VERBOSE = TRUE; - } - - // a crude noise model valid for both the guess vectors and the - // data buffer: - float gain = 1.0; - if ((N = get_argument (argc, argv, "-gain"))) { - remove_argument (N, &argc, argv); - gain = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float noise = 0.0; - if ((N = get_argument (argc, argv, "-noise"))) { - remove_argument (N, &argc, argv); - noise = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float nsigma = 2.0; - if ((N = get_argument (argc, argv, "-nsigma"))) { - remove_argument (N, &argc, argv); - nsigma = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float lambda = 1.0; - if ((N = get_argument (argc, argv, "-lambda"))) { - remove_argument (N, &argc, argv); - lambda = atof(argv[N]); - remove_argument (N, &argc, argv); - } - - int Niter = 25; - if ((N = get_argument (argc, argv, "-iter"))) { - remove_argument (N, &argc, argv); - Niter = atoi(argv[N]); - remove_argument (N, &argc, argv); - } - int Nrow = 5; - if ((N = get_argument (argc, argv, "-row"))) { - remove_argument (N, &argc, argv); - Nrow = atoi(argv[N]); - remove_argument (N, &argc, argv); - } - - // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-profile"))) { - remove_argument (N, &argc, argv); - if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-psf"))) { - remove_argument (N, &argc, argv); - if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-stilt"))) { - remove_argument (N, &argc, argv); - stilt = atof (argv[N]); - remove_argument (N, &argc, argv); - } else { goto usage; } - - // In-Out parameters: - if ((N = get_argument (argc, argv, "-object"))) { - remove_argument (N, &argc, argv); - if ((obj = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-sky"))) { - remove_argument (N, &argc, argv); - if ((sky = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-backgnd"))) { - remove_argument (N, &argc, argv); - if ((bck = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - - if (argc != 3) goto usage; - - // buffer is the full image, Xref is reference coordinate of the profile in the buffer - if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - - // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) - if (profile->Nelements % 2 == 0) { - gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); - return FALSE; - } - if (psf->Nelements % 2 == 0) { - gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); - return FALSE; - } - - // observed buffer size - int Nx = buffer[0].matrix.Naxis[0]; - int Ny = buffer[0].matrix.Naxis[1]; - float *bufVal = (float *) buffer->matrix.buffer; - - // obj, sky, bck must be consistent with data - if (Ny != obj->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (object)\n"); - return FALSE; - } - if (Ny != sky->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (sky)\n"); - return FALSE; - } - if (Ny != bck->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (backgnd)\n"); - return FALSE; - } - - // for the cross-dispersion reference pixel, use the user value or set to Nx/2 if < 0 - deimos_set_cross_ref (atoi(argv[2]), Nx); - deimos_make_kernel (stilt, Nx); - - Vector *objMin = NULL; - Vector *skyMin = NULL; - Vector *bckMin = NULL; - - if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); - if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); - if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); - - // the functions below accept an opihi_flt array for object, sky, background - opihi_flt *objVal = objMin->elements.Flt; - opihi_flt *skyVal = skyMin->elements.Flt; - opihi_flt *bckVal = bckMin->elements.Flt; - - // we make a copy of the input guess vectors and minimize them - for (int iy = 0; iy < Ny; iy++) { - objVal[iy] = obj->elements.Flt[iy]; - skyVal[iy] = sky->elements.Flt[iy]; - bckVal[iy] = bck->elements.Flt[iy]; - } - - float noiseVar = SQ(noise); - ALLOCATE_PTR (objNoise, float, Ny); - ALLOCATE_PTR (skyNoise, float, Ny); - ALLOCATE_PTR (bckNoise, float, Ny); - - // generate noise vectors - for (int iy = 0; iy < Ny; iy++) { - objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy]) / gain, noiseVar)); - skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy]) / gain, noiseVar)); - bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy]) / gain, noiseVar)); - } - - // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. - - // XXX worry about last segment (< Nrow rows) - for (int iter = 0; iter < Niter; iter++) { - - int dRow = Nrow/2; - - // use scaled-noise to define test ranges - // we are going to try each row using a window Nrow wide to measure chisq: - for (int row = dRow; row < Ny - dRow - 1; row++) { - - // I am using a numerical equivalent of Levenberg-Marquardt, but only for a single - // p_m[i] = (obj,sky,bck) triplet [m is the index of obj,sky,bck; i is the row - // index] at a time. I start with a current best guess set of (obj,sky,bck). I - // need to calculate the model at this location, and then at offset locations p_m + - // dp_m. Note that I do not need to calculate chisq for each of these to generate - // the elements of the LMM equations: - - int row_0 = row - dRow; // start row of the subset test range - - // Nrow = 5, dRow = 2, Ny = 100, last start is Ny - 2 - 1 = 97 - // last row_0 is 95 - - // generate subset vectors for the range **centered** on row - opihi_flt *objCurr = deimos_subvector (objVal, row_0, Nrow); - opihi_flt *skyCurr = deimos_subvector (skyVal, row_0, Nrow); - opihi_flt *bckCurr = deimos_subvector (bckVal, row_0, Nrow); - - // current vector value: - // for deimos_make_model, row is the starting point of the subimage - float *model_ref = deimos_make_model (objCurr, skyCurr, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // in the test functions below, dRow is the central pixel of the subset, e.g., obj[row] => objCurr[dRow] - - float dObj = 0.0, dSky = 0.0, dBck = 0.0; - - // delta obj (save dObj) - opihi_flt *objtest = deimos_make_test (objCurr, objNoise[row], dRow, Nrow, &dObj); - float *model_obj = deimos_make_model (objtest, skyCurr, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // delta sky (save dSky) - opihi_flt *skytest = deimos_make_test (skyCurr, skyNoise[row], dRow, Nrow, &dSky); - float *model_sky = deimos_make_model (objCurr, skytest, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // delta bck (save dBck) - opihi_flt *bcktest = deimos_make_test (bckCurr, bckNoise[row], dRow, Nrow, &dBck); - float *model_bck = deimos_make_model (objCurr, skyCurr, bcktest, psf, profile, trace, Nx, Nrow, row_0); - - // dObj, dSky, dBck carry the delta used derivatives in and the delta applied to the values out - float chisq = deimos_LMM_update (bufVal, model_ref, model_obj, model_sky, model_bck, &dObj, &dSky, &dBck, Nx, Nrow, row_0, lambda); - - if (1) { - if (DO_PLUS) { - objVal[row] += dObj; - skyVal[row] += dSky; - bckVal[row] += dBck; - } else { - objVal[row] -= dObj; - skyVal[row] -= dSky; - bckVal[row] -= dBck; - } - } - - free (objCurr); - free (skyCurr); - free (bckCurr); - - free (model_ref); - - free (model_obj); - free (model_sky); - free (model_bck); - - free (objtest); - free (skytest); - free (bcktest); - - if (VERBOSE) fprintf (stderr, "chisq: %f (%f, %f, %f)\n", chisq, dObj, dSky, dBck); - } - - float *model_full = deimos_make_model (objVal, skyVal, bckVal, psf, profile, trace, Nx, Ny, 0); - - int Npts = 0; - float chisq_full = deimos_get_chisq (bufVal, model_full, Nx, Ny, 0, &Npts); - - fprintf (stderr, "** Full chisq: %f (%d) : %f)\n", chisq_full, Npts, chisq_full / (1.0 * Npts)); - } - - return TRUE; - - usage: - gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); - return FALSE; -} - -/****************** fitobj Support Functions *******************/ - -// we generate a new vector with a single element (row) modified based on sigma -opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow) { - - ALLOCATE_PTR (value, opihi_flt, Nrow); - - for (int i = 0; i < Nrow; i++) { - value[i] = fullVec[i + row]; - } - - return value; -} - - -// we generate a new vector with a single element (row) modified based on sigma -static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue) { - - // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) - // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) - // sigma is valid for the element at row - - ALLOCATE_PTR (value, opihi_flt, Nrow); - - for (int i = 0; i < Nrow; i++) { - value[i] = guess[i]; - } - - value[row] = guess[row] + 0.1*sigma; - - *dValue = 0.1*sigma; - return value; -} - - -double deimos_LMM_update (float *data, float *model_ref, - float *model_obj, float *model_sky, float *model_bck, - float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda) { - - // we are going to loop over the images calculating the following value: - - // W = 1 / sigma^2, leave as 1.0 for now - // chisq = sum W * (data - model_ref)^2 - // dF_X = sum W * (data - model_ref) * (model_ref - model_X) / dX - // d2F_XY = sum W * (model_ref - model_X) * (model_ref - model_Y) / (dX * dY) - - double chisq = 0.0; - double dF_obj = 0.0, dF_sky = 0.0, dF_bck = 0.0; - double d2F_obj_obj = 0.0, d2F_obj_sky = 0.0, d2F_obj_bck = 0.0, d2F_sky_sky = 0.0, d2F_sky_bck = 0.0, d2F_bck_bck = 0.0; - - float W = 1.0; - - for (int iy = 0; iy < Ny; iy++) { - for (int ix = 0; ix < Nx; ix++) { - int pix = ix + iy*Nx; - int pix_data = ix + (iy + row)*Nx; - - // if any of the pixel values are NAN, skip the point: - if (!isfinite(data[pix_data])) continue; - if (!isfinite(model_ref[pix])) continue; - if (!isfinite(model_obj[pix])) continue; - if (!isfinite(model_sky[pix])) continue; - if (!isfinite(model_bck[pix])) continue; - - float df = W * (data[pix_data] - model_ref[pix]); - - chisq += SQ(df); - - float df_obj = (model_ref[pix] - model_obj[pix]) / *dObj; - float df_sky = (model_ref[pix] - model_sky[pix]) / *dSky; - float df_bck = (model_ref[pix] - model_bck[pix]) / *dBck; - - dF_obj += df*df_obj; - dF_sky += df*df_sky; - dF_bck += df*df_bck; - - d2F_obj_obj += W * SQ(df_obj); - d2F_sky_sky += W * SQ(df_sky); - d2F_bck_bck += W * SQ(df_bck); - - d2F_obj_sky += W * df_obj * df_sky; - d2F_obj_bck += W * df_obj * df_bck; - d2F_sky_bck += W * df_sky * df_bck; - } - } - - // these are the elements of Ax = B - ALLOCATE_PTR (A, double *, 3); - ALLOCATE_PTR (B, double *, 3); - for (int i = 0; i < 3; i++) { - ALLOCATE (A[i], double, 3); - ALLOCATE (B[i], double, 1); - } - - A[0][0] = d2F_obj_obj * (1.0 + lambda); - A[1][1] = d2F_sky_sky * (1.0 + lambda); - A[2][2] = d2F_bck_bck * (1.0 + lambda); - - A[0][1] = A[1][0] = d2F_obj_sky; - A[1][2] = A[2][1] = d2F_sky_bck; - A[0][2] = A[2][0] = d2F_obj_bck; - - B[0][0] = dF_obj; - B[1][0] = dF_sky; - B[2][0] = dF_bck; - - dgaussjordan (A, B, 3, 1); - - *dObj = B[0][0]; - *dSky = B[1][0]; - *dBck = B[2][0]; - - return chisq; -} - -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { - - int npts = 0; - float chisq = 0; - - for (int iy = 0; iy < Ny; iy++) { - - for (int ix = 0; ix < Nx; ix++) { - - int pix_buffer = ix + (iy + row)*Nx; - int pix_model = ix + iy*Nx; - - if (!isfinite(buffer[pix_buffer])) continue; - if (!isfinite(model[pix_model])) continue; - chisq += SQ(buffer[pix_buffer] - model[pix_model]); - npts ++; - } - } - *Npts = npts; - return chisq; -} -# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c (revision 41170) @@ -0,0 +1,237 @@ +# include "data.h" +# define NPAR 4 + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); + +opihi_flt fgaussOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar); + +void gaussian_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar); +void gaussian_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC); + +double deimos_weight_cauchy (double x); + +// use these globals to carry into sigmoid_fit +static int VERBOSE = FALSE; +static int QUIET = FALSE; +static int MaxIterations = 10; + +/* fitarc buffer Xo Yo Nx Ny + + * for each col, fit the y-dir profile with a gauss + * fit the C0 values to a line to get the angle + */ + +int deimos_fitarc (int argc, char **argv) { + + // input parameters: + // * xprofile : extracted slit window coordinates (vector) + // * xprofile : extracted slit window profile (vector) + // * Xs, Xe : scale coordinates of window boundaries + + // output parameters: + // * sigmoid_left_center, etc : coordinates of window boundaries + + int N; + Buffer *buffer = NULL; + + QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + MaxIterations = 10; + if ((N = get_argument (argc, argv, "-max-iterations"))) { + remove_argument (N, &argc, argv); + MaxIterations = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + Buffer *residbuf = NULL; + if ((N = get_argument (argc, argv, "-resid"))) { + remove_argument (N, &argc, argv); + residbuf = SelectBuffer (argv[N], ANYBUFFER, TRUE); + if (!residbuf) goto usage; + remove_argument (N, &argc, argv); + } + + Buffer *varbuf = NULL; + if ((N = get_argument (argc, argv, "-varbuf"))) { + remove_argument (N, &argc, argv); + varbuf = SelectBuffer (argv[N], OLDBUFFER, TRUE); + if (!varbuf) goto usage; + remove_argument (N, &argc, argv); + } + + if (argc != 6) goto usage; + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + int nx = buffer[0].matrix.Naxis[0]; + int ny = buffer[0].matrix.Naxis[1]; + + // supplied guess for boundaries + int Xs = atof(argv[2]); + int Ys = atof(argv[3]); + int Nx = atof(argv[4]); + int Ny = atof(argv[5]); + + if (residbuf) { + ResetBuffer (residbuf, nx, ny, -32, 0.0, 1.0); + } + + // validate Xs, Ys, Nx, Ny + // Xs + Nx <= nx + + // loop over the cols + + ALLOCATE_PTR (xcol, double, Ny); + ALLOCATE_PTR (ycol, double, Ny); + ALLOCATE_PTR (ocol, double, Ny); // fitted value + + float *bufVal = (float *) buffer->matrix.buffer; + + for (int ix = Xs; ix < Xs + Nx; ix++) { + + // loop over the pixels in this column to get the x and y vectors + // find the peak at the same time + + double xmax = -HUGE_VAL; + double ymax = -HUGE_VAL; + int Nc = 0; + + for (int iy = 0; iy < Ny; iy++) { + + float yval = bufVal[ix + (iy + Ys)*Nx]; + if (!isfinite(yval)) continue; + + xcol[Nc] = iy + Ys; + ycol[Nc] = yval; + + if (yval > ymax) { + ymax = yval; + xmax = iy + Ys; + } + Nc ++; + } + + // now do a Gauss fit given C0 = xmax, C1 = 2, C2 = ymax, C3 = 0 + + // temporary storage vectors + opihi_flt par[NPAR]; + par[0] = xmax; + par[1] = 2; + par[2] = ymax; + par[3] = 0; + gaussian_fit_irls (xcol, ycol, NULL, ocol, Nc, par, NPAR); + + fprintf (stderr, "%d : %f %f : %f %f\n", ix, par[0], par[2], par[1], par[3]); + set_variable ("C0", par[0]); + set_variable ("C1", par[1]); + set_variable ("C2", par[2]); + set_variable ("C3", par[3]); + + if (residbuf) { + float *residVal = (float *) residbuf->matrix.buffer; + for (int iy = 0; iy < Ny; iy++) { + float yval = fgaussOD (iy + Ys, par, NPAR, NULL); + // residVal[ix + (iy + Ys)*Nx] = bufVal[ix + (iy + Ys)*Nx] - yval; + residVal[ix + (iy + Ys)*Nx] = yval; + } + } + } + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos fitarc buff Xo Yo Nx Ny\n"); + gprint (GP_ERR, " [-resid buffer] : output residual image\n"); + gprint (GP_ERR, " [-varbuf buffer] : input variance image\n"); + return FALSE; +} + +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 + +// xprofile : x-coordinate +// fprofile : flux (counts) +// wprofile : weight (true poisson error) +// oprofile : output profile (not used?) +void gaussian_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar) { + + // buffer to store result from each iteration + ALLOCATE_PTR (par_old, opihi_flt, Npar); + + // define initial weights as 1.0 + ALLOCATE_PTR (wt, opihi_flt, Npts); + ALLOCATE_PTR (wtemp, opihi_flt, Npts); + for (int i = 0; i < Npts; i++) { + wtemp[i] = wprofile ? (fabs(wprofile[i]) > 1e-5 ? wprofile[i] : 1e-5) : 1.0; + wt[i] = 1.0 / SQ(wtemp[i]); + } + + // initial fit with ordinary least-squares + // otemp is the fitted value of the function at xtemp + gaussian_fit (xprofile, fprofile, wt, NULL, Npts, par, Npar, fgaussOD); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < MaxIterations); iterations++) { + + for (int i = 0; i < Npts; i++) { + wt[i] = deimos_weight_cauchy ((fprofile[i] - oprofile[i]) / wtemp[i]) / SQ(wtemp[i]); + } + + gaussian_fit (xprofile, fprofile, wt, oprofile, Npts, par, Npar, fgaussOD); + + // save this solution + for (int i = 0; i < Npar; i++) { + par_old[i] = par[i]; + } + + converged = TRUE; + for (int i = 0; i < Npts; i++) { + if ((fabs(par[i] - par_old[i]) > FIT_TOLERANCE * fabs(par[i])) && + (fabs(par[i] - par_old[i]) > FLT_TOLERANCE)) + converged = FALSE; + } + } + + for (int i = 0; i < Npts; i++) { + oprofile[i] = fgaussOD (xprofile[i], par, Npar, NULL); + } + + FREE (par_old); + FREE (wt); +} + +void gaussian_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC) { + + double ochisq = mrqinit (xprofile, fprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + double dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + double chisq = mrqmin (xprofile, fprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + dchisq = ochisq - chisq; + ochisq = chisq; + if (VERBOSE) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (VERBOSE) gprint (GP_ERR, "%d iterations\n", Niter); + + if (!QUIET) gprint (GP_LOG, "gauss @ %f (%f) : top %f bottom %f\n", par[0], par[1], par[2], par[3]); + + if (oprofile) { + for (int i = 0; i < Npts; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + } + + mrqfree (Npar); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c (revision 41170) @@ -1,13 +1,4 @@ # include "data.h" # include "deimos.h" - -# if (0) -int deimos_fitobj (int argc, char **argv) { - OHANA_UNUSED_PARAM(argc); - OHANA_UNUSED_PARAM(argv); - return FALSE; -} - -# else /* @@ -18,19 +9,19 @@ // internal functions to fitobj -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); -int *deimos_sort_result (DeimosResult *result, int Nresult); -int deimos_get_random_sample (int Nresult); -static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput); - -int USE_GAUSS_DEV = FALSE; +static float deimos_get_chisq (float *buffer, float *vardata, float *model, int Nx, int Ny, int row, int *Npts); +double deimos_LMM_update (float *data, float *vardata, float *model_ref, float *model_obj, float *model_sky, float *model_bck, float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda); +static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue); +opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow); int deimos_fitobj (int argc, char **argv) { // input parameters: - // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * PSF : point-spread function vector (flux normalized, x-dir) - // * stilt : slit tilt response : 2D kernel? + // * buffer : 2D image of slit (full frame or cutout?) + // * slit_trace_red : spline fit of slit central x pos vs y-coord + // * slit_trace_blu : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of slit central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? // in-out parameters: @@ -41,17 +32,48 @@ int N; - Spline *trace = NULL; - Vector *profile = NULL; - - Vector *obj = NULL; - Vector *sky = NULL; - Vector *bck = NULL; - - Vector *psf = NULL; - - Buffer *buffer = NULL; - - float stilt = 0.0; // angle of the slit + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; + + Vector *profile = NULL; + Vector *psf = NULL; + + Vector *obj = NULL; + Vector *sky = NULL; + Vector *bck = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + ohana_gaussdev_init (); + + int DO_PLUS = FALSE; + if ((N = get_argument (argc, argv, "-plus"))) { + remove_argument (N, &argc, argv); + DO_PLUS = TRUE; + } + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + + int SKIPFIT = FALSE; + if ((N = get_argument (argc, argv, "-skipfit"))) { + remove_argument (N, &argc, argv); + SKIPFIT = TRUE; + } + + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } // a crude noise model valid for both the guess vectors and the @@ -75,11 +97,10 @@ remove_argument (N, &argc, argv); } - float scale = 0.75; - if ((N = get_argument (argc, argv, "-scale"))) { - remove_argument (N, &argc, argv); - scale = atof(argv[N]); - remove_argument (N, &argc, argv); - } - + float lambda = 1.0; + if ((N = get_argument (argc, argv, "-lambda"))) { + remove_argument (N, &argc, argv); + lambda = atof(argv[N]); + remove_argument (N, &argc, argv); + } int Niter = 25; @@ -89,10 +110,4 @@ remove_argument (N, &argc, argv); } - int Ntry = 5; - if ((N = get_argument (argc, argv, "-try"))) { - remove_argument (N, &argc, argv); - Ntry = atoi(argv[N]); - remove_argument (N, &argc, argv); - } int Nrow = 5; if ((N = get_argument (argc, argv, "-row"))) { @@ -102,26 +117,19 @@ } - USE_GAUSS_DEV = FALSE; - if ((N = get_argument (argc, argv, "-gaussdev"))) { - remove_argument (N, &argc, argv); - USE_GAUSS_DEV = TRUE; - } - - int SAVE_ALL_VECTORS = FALSE; - if ((N = get_argument (argc, argv, "-save-all-vectors"))) { - remove_argument (N, &argc, argv); - SAVE_ALL_VECTORS = TRUE; - } - - int SAVE_MIN_VECTORS = FALSE; - if ((N = get_argument (argc, argv, "-save-min-vectors"))) { - remove_argument (N, &argc, argv); - SAVE_MIN_VECTORS = TRUE; - } - // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } else { goto usage; } @@ -158,4 +166,20 @@ remove_argument (N, &argc, argv); } else { goto usage; } + + // save the model + Buffer *model = NULL; + if ((N = get_argument (argc, argv, "-model"))) { + remove_argument (N, &argc, argv); + if ((model = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + // save the model + Buffer *varim = NULL; + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((varim = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } if (argc != 3) goto usage; @@ -179,4 +203,10 @@ float *bufVal = (float *) buffer->matrix.buffer; + if (varim) { + if (Nx != varim[0].matrix.Naxis[0]) { gprint (GP_ERR, "inconsistent sizes for image and varim\n"); return FALSE; } + if (Ny != varim[0].matrix.Naxis[1]) { gprint (GP_ERR, "inconsistent sizes for image and varim\n"); return FALSE; } + } + float *varVal = varim ? (float *) varim->matrix.buffer : NULL; + // obj, sky, bck must be consistent with data if (Ny != obj->Nelements) { @@ -193,240 +223,141 @@ } - // for the cross-dispersion reference pixle, use the user value or set to Nx/2 if < 0 + if (model) { + if (!CreateBuffer (model, Nx, Ny, -32, 1.0, 0.0)) { gprint (GP_ERR, "error generating output model buffer\n"); return FALSE; } + } + + // for the cross-dispersion reference pixel, use the user value or set to Nx/2 if < 0 deimos_set_cross_ref (atoi(argv[2]), Nx); deimos_make_kernel (stilt, Nx); - // the functions below accept an opihi_flt array for object, sky, background - opihi_flt *objVal = obj->elements.Flt; - opihi_flt *skyVal = sky->elements.Flt; - opihi_flt *bckVal = bck->elements.Flt; - - int NRESULT = Niter + Ntry*Niter + 1; - ALLOCATE_PTR (result, DeimosResult, NRESULT); - - Vector **objOut = NULL; - Vector **skyOut = NULL; - Vector **bckOut = NULL; Vector *objMin = NULL; Vector *skyMin = NULL; Vector *bckMin = NULL; - // Save all result vectors for each of obj, sky, bck: - if (SAVE_ALL_VECTORS) { - ALLOCATE (objOut, Vector *, NRESULT); - ALLOCATE (skyOut, Vector *, NRESULT); - ALLOCATE (bckOut, Vector *, NRESULT); - for (int i = 0; i < NRESULT; i++) { - char name[64]; - snprintf (name, 64, "obj_%04d", i); if ((objOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objOut[i], OPIHI_FLT, Ny); - snprintf (name, 64, "sky_%04d", i); if ((skyOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyOut[i], OPIHI_FLT, Ny); - snprintf (name, 64, "bck_%04d", i); if ((bckOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckOut[i], OPIHI_FLT, Ny); - for (int iy = 0; iy < Ny; iy++) { - objOut[i]->elements.Flt[iy] = 0.0; - skyOut[i]->elements.Flt[iy] = 0.0; - bckOut[i]->elements.Flt[iy] = 0.0; + if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); + if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); + if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); + + // the functions below accept an opihi_flt array for object, sky, background + opihi_flt *objVal = objMin->elements.Flt; + opihi_flt *skyVal = skyMin->elements.Flt; + opihi_flt *bckVal = bckMin->elements.Flt; + + // we make a copy of the input guess vectors and minimize them + for (int iy = 0; iy < Ny; iy++) { + objVal[iy] = obj->elements.Flt[iy]; + skyVal[iy] = sky->elements.Flt[iy]; + bckVal[iy] = bck->elements.Flt[iy]; + } + + float noiseVar = SQ(noise); + ALLOCATE_PTR (objNoise, float, Ny); + ALLOCATE_PTR (skyNoise, float, Ny); + ALLOCATE_PTR (bckNoise, float, Ny); + + // generate noise vectors + for (int iy = 0; iy < Ny; iy++) { + objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy]) / gain, noiseVar)); + skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy]) / gain, noiseVar)); + bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy]) / gain, noiseVar)); + } + + // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. + + // XXX worry about last segment (< Nrow rows) + for (int iter = 0; iter < Niter; iter++) { + + int dRow = Nrow/2; + + // use scaled-noise to define test ranges + // we are going to try each row using a window Nrow wide to measure chisq: + for (int row = dRow; !SKIPFIT && (row < Ny - dRow - 1); row++) { + + // I am using a numerical equivalent of Levenberg-Marquardt, but only for a single + // p_m[i] = (obj,sky,bck) triplet [m is the index of obj,sky,bck; i is the row + // index] at a time. I start with a current best guess set of (obj,sky,bck). I + // need to calculate the model at this location, and then at offset locations p_m + + // dp_m. Note that I do not need to calculate chisq for each of these to generate + // the elements of the LMM equations: + + int row_0 = row - dRow; // start row of the subset test range + + // Nrow = 5, dRow = 2, Ny = 100, last start is Ny - 2 - 1 = 97 + // last row_0 is 95 + + // generate subset vectors for the range **centered** on row + opihi_flt *objCurr = deimos_subvector (objVal, row_0, Nrow); + opihi_flt *skyCurr = deimos_subvector (skyVal, row_0, Nrow); + opihi_flt *bckCurr = deimos_subvector (bckVal, row_0, Nrow); + + // current vector value: + // for deimos_make_model, row is the starting point of the subimage + float *model_ref = deimos_make_model (objCurr, skyCurr, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // in the test functions below, dRow is the central pixel of the subset, e.g., obj[row] => objCurr[dRow] + + float dObj = 0.0, dSky = 0.0, dBck = 0.0; + + // delta obj (save dObj) + opihi_flt *objtest = deimos_make_test (objCurr, objNoise[row], dRow, Nrow, &dObj); + float *model_obj = deimos_make_model (objtest, skyCurr, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // delta sky (save dSky) + opihi_flt *skytest = deimos_make_test (skyCurr, skyNoise[row], dRow, Nrow, &dSky); + float *model_sky = deimos_make_model (objCurr, skytest, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // delta bck (save dBck) + opihi_flt *bcktest = deimos_make_test (bckCurr, bckNoise[row], dRow, Nrow, &dBck); + float *model_bck = deimos_make_model (objCurr, skyCurr, bcktest, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // dObj, dSky, dBck carry the delta used derivatives in and the delta applied to the values out + float chisq = deimos_LMM_update (bufVal, varVal, model_ref, model_obj, model_sky, model_bck, &dObj, &dSky, &dBck, Nx, Nrow, row_0, lambda); + + if (FALSE && (row > 10) && (row < 20)) { + fprintf (stderr, "%d %d : %f %f %f\n", iter, row, dObj, dSky, dBck); } - } - } - if (SAVE_MIN_VECTORS) { - if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); - if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); - if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); - for (int iy = 0; iy < Ny; iy++) { - objMin->elements.Flt[iy] = 0.0; - skyMin->elements.Flt[iy] = 0.0; - bckMin->elements.Flt[iy] = 0.0; - } - } - Vector *chisqVect; - if ((chisqVect = SelectVector ("chisqOut", ANYVECTOR, TRUE)) == NULL) { return FALSE; } - ResetVector (chisqVect, OPIHI_FLT, NRESULT); - for (int i = 0; i < NRESULT; i++) { chisqVect->elements.Flt[i] = 0.0; } - - // XXX temp hack: save chisq subset vectors for each row pass - int Npass = Ny / Nrow; - ALLOCATE_PTR (chiPass, Vector *, Npass); - for (int pass = 0; pass < Npass; pass++) { - char name[64]; - snprintf (name, 64, "chiPass_%03d", pass); - if ((chiPass[pass] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return FALSE; } ResetVector (chiPass[pass], OPIHI_FLT, NRESULT); - } - - float noiseVar = SQ(noise); - ALLOCATE_PTR (objNoise, float, Nrow); - ALLOCATE_PTR (skyNoise, float, Nrow); - ALLOCATE_PTR (bckNoise, float, Nrow); - - // work on segments of Nrow at a time - // XXX worry about last segment (< Nrow rows) - int pass = 0; - for (int row = 0; row < Ny - Nrow; row += Nrow, pass++) { - - int Nresult = 0; - - // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. - - // If I just use the value in a given bin to define the range, random noise for the - // faint end could have bad results (range ~ 0.0). For that matter, if I am in a - // region with large dynamic range (e.g., on an emission or absorption line), then we - // likely have large errors in how the guess follows the truth. - - /** old concept: - float objRange = 0, skyRange = 0, bckRange = 0; - for (int iy = 0; iy < Nrow; iy++) { - objRange = MAX(objVal[iy + row], objRange); - skyRange = MAX(skyVal[iy + row], skyRange); - bckRange = MAX(bckVal[iy + row], bckRange); - } - **/ - - // new concept: start with 2-sigma range, then ramp down below - for (int iy = 0; iy < Nrow; iy++) { - objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy + row]) / gain, noiseVar)); - skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy + row]) / gain, noiseVar)); - bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy + row]) / gain, noiseVar)); - } - - // first test is the input guess - if (1) { - opihi_flt *objtest = deimos_make_test (objVal, NULL, row, Nrow, TRUE); - opihi_flt *skytest = deimos_make_test (skyVal, NULL, row, Nrow, TRUE); - opihi_flt *bcktest = deimos_make_test (bckVal, NULL, row, Nrow, TRUE); - - // generate the model based on the test values - // objtest, skytest, bcktest are segments of the full parameter vectors - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - // XXX need to add in the per-pixel error or variance - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - fprintf (stderr, "guess: chisq: %f, Npts: %d\n", chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; - } - - // save the results - result[Nresult].chisq = chisq / (float) Npts; - result[Nresult].obj = objtest; - result[Nresult].sky = skytest; - result[Nresult].bck = bcktest; - Nresult ++; - } - - // run Niter iterations - for (int iter = 0; iter < Niter; iter ++) { - - // generate a set of test values for obj, sky, backgnd based on the current guess - opihi_flt *objtest = deimos_make_test (objVal, objNoise, row, Nrow, TRUE); - opihi_flt *skytest = deimos_make_test (skyVal, skyNoise, row, Nrow, TRUE); - opihi_flt *bcktest = deimos_make_test (bckVal, bckNoise, row, Nrow, TRUE); - - for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { - fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", - objVal[itmp + row], skyVal[itmp + row], bckVal[itmp + row], - objtest[itmp], skytest[itmp], bcktest[itmp]); - } - - // generate the model based on the test values - // objtest, skytest, bcktest are segments of the full parameter vectors - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - // XXX need to add in the per-pixel error or variance - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - // fprintf (stderr, "iter: %d, chisq: %f, Npts: %d\n", iter, chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; - } - - // save the results - result[Nresult].chisq = chisq / (float) Npts; - result[Nresult].obj = objtest; - result[Nresult].sky = skytest; - result[Nresult].bck = bcktest; - Nresult ++; - } - - for (int try = 0; try < Ntry; try ++) { - // generate an index of results sorted by chisq - int *IDX = deimos_sort_result (result, Nresult); - - int Nnewset = 0; - - // XXX not sure this scaling factor concept is good - for (int iy = 0; iy < Nrow; iy++) { - objNoise[iy] = scale*objNoise[iy]; - skyNoise[iy] = scale*skyNoise[iy]; - bckNoise[iy] = scale*bckNoise[iy]; - } - - // run Niter iterations, selecting guess values from the result set - for (int iter = 0; iter < Niter; iter ++) { - int bin = deimos_get_random_sample (Nresult); - int entry = IDX[bin]; - // fprintf (stderr, "try: %d : %d, %f\n", bin, entry, result[entry].chisq); - - // generate a set of test values for obj, sky, backgnd based on the current guess - opihi_flt *objtest = deimos_make_test (result[entry].obj, objNoise, row, Nrow, FALSE); - opihi_flt *skytest = deimos_make_test (result[entry].sky, skyNoise, row, Nrow, FALSE); - opihi_flt *bcktest = deimos_make_test (result[entry].bck, bckNoise, row, Nrow, FALSE); - - for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { - fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", - result[entry].obj[itmp], result[entry].sky[itmp], result[entry].bck[itmp], - objtest[itmp], skytest[itmp], bcktest[itmp]); - } - - // generate the model based on the test values - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - // fprintf (stderr, "try: %d, iter: %d, chisq: %f, Npts: %d\n", try, iter, chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult + Nnewset] = chisq / (float) Npts; - } - - // save the results - result[Nresult + Nnewset].chisq = chisq / (float) Npts; - result[Nresult + Nnewset].obj = objtest; - result[Nresult + Nnewset].sky = skytest; - result[Nresult + Nnewset].bck = bcktest; - Nnewset ++; - } - Nresult += Nnewset; - } - - int *IDX = deimos_sort_result (result, Nresult); - - // ENDING: save the best result and info about the allowed chisq region - // int *index = sort_result (result, Nresult); - // copy result[index[0]].obj,sky,bck to obj,sky,bck - for (int i = 0; i < Nresult; i++) { - for (int iy = 0; iy < Nrow; iy++) { - if (SAVE_ALL_VECTORS) { - objOut[i]->elements.Flt[iy + row] = result[IDX[i]].obj[iy]; - skyOut[i]->elements.Flt[iy + row] = result[IDX[i]].sky[iy]; - bckOut[i]->elements.Flt[iy + row] = result[IDX[i]].bck[iy]; + + if (1) { + if (DO_PLUS) { + objVal[row] += dObj; + skyVal[row] += dSky; + bckVal[row] += dBck; + } else { + objVal[row] -= dObj; + skyVal[row] -= dSky; + bckVal[row] -= dBck; } } - chisqVect->elements.Flt[i] += result[IDX[i]].chisq; + + free (objCurr); + free (skyCurr); + free (bckCurr); + + free (model_ref); + + free (model_obj); + free (model_sky); + free (model_bck); + + free (objtest); + free (skytest); + free (bcktest); + + if (VERBOSE) fprintf (stderr, "chisq: %f (%f, %f, %f)\n", chisq, dObj, dSky, dBck); } - if (SAVE_MIN_VECTORS) { - for (int iy = 0; iy < Nrow; iy++) { - objMin->elements.Flt[iy + row] = result[IDX[0]].obj[iy]; - skyMin->elements.Flt[iy + row] = result[IDX[0]].sky[iy]; - bckMin->elements.Flt[iy + row] = result[IDX[0]].bck[iy]; - } + + float *model_full = deimos_make_model (objVal, skyVal, bckVal, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Ny, 0); + + int Npts = 0; + float chisq_full = deimos_get_chisq (bufVal, varVal, model_full, Nx, Ny, 0, &Npts); + + if (model) { + free (model[0].matrix.buffer); + model[0].matrix.buffer = (char *) model_full; + } else { + free (model_full); } + + fprintf (stderr, "** Full chisq: %f (%d) : %f)\n", chisq_full, Npts, chisq_full / (1.0 * Npts)); } @@ -434,5 +365,6 @@ usage: - gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -slit-trace-red (spline) -slit-trace-blu (spline) -psf-trace (spline) -profile vector -psf vector -stilt (angle)\n"); + gprint (GP_ERR, "OPTIONS: [-model image] [-variance image] [-gain value] [-noise value] [-nsigma value] [-lambda value] [-iter N] [-row N] [-redlimit Npixel] [-skipfit] [-v] [-plus]\n"); return FALSE; } @@ -440,38 +372,120 @@ /****************** fitobj Support Functions *******************/ -static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput) { - - // we have a vector, guess, with some length >= row + Nrow. - // generate a set of random values in the vicinty of guess[row] to guess[row+Nrow] - // return vector is of length Nrow +// we generate a new vector with a single element (row) modified based on sigma +opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow) { + + ALLOCATE_PTR (value, opihi_flt, Nrow); + + for (int i = 0; i < Nrow; i++) { + value[i] = fullVec[i + row]; + } + + return value; +} + + +// we generate a new vector with a single element (row) modified based on sigma +static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue) { // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) - // sigma is always only Nrow long + // sigma is valid for the element at row ALLOCATE_PTR (value, opihi_flt, Nrow); - int start_pix = fullInput ? row : 0; - int stop_pix = fullInput ? row + Nrow : Nrow; - - int iv = 0; - for (int i = start_pix; i < stop_pix; i++, iv++) { - if (!sigma) { - value[iv] = guess[i]; - continue; - } - if (USE_GAUSS_DEV) { - value[iv] = ohana_gaussdev_rnd(guess[i], sigma[iv]); - continue; - } - - // uniform distribution: - value[iv] = sigma[iv]*(drand48() - 0.5) + guess[i]; - } + for (int i = 0; i < Nrow; i++) { + value[i] = guess[i]; + } + + value[row] = guess[row] + 0.1*sigma; + + *dValue = 0.1*sigma; return value; } -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { +double deimos_LMM_update (float *data, float *vardata, float *model_ref, + float *model_obj, float *model_sky, float *model_bck, + float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda) { + + // we are going to loop over the images calculating the following value: + + // W = 1 / sigma^2, leave as 1.0 for now + // chisq = sum W * (data - model_ref)^2 + // dF_X = sum W * (data - model_ref) * (model_ref - model_X) / dX + // d2F_XY = sum W * (model_ref - model_X) * (model_ref - model_Y) / (dX * dY) + + double chisq = 0.0; + double dF_obj = 0.0, dF_sky = 0.0, dF_bck = 0.0; + double d2F_obj_obj = 0.0, d2F_obj_sky = 0.0, d2F_obj_bck = 0.0, d2F_sky_sky = 0.0, d2F_sky_bck = 0.0, d2F_bck_bck = 0.0; + + for (int iy = 0; iy < Ny; iy++) { + for (int ix = 0; ix < Nx; ix++) { + int pix = ix + iy*Nx; + int pix_data = ix + (iy + row)*Nx; + + // if any of the pixel values are NAN, skip the point: + if (!isfinite(data[pix_data])) continue; + if (!isfinite(model_ref[pix])) continue; + if (!isfinite(model_obj[pix])) continue; + if (!isfinite(model_sky[pix])) continue; + if (!isfinite(model_bck[pix])) continue; + + if (vardata && !isfinite(vardata[pix_data])) continue; + float Var = vardata ? vardata[pix_data] : 1.0; + float W = (fabs(Var) < 0.01) ? 100.0 : 1.0 / Var; + + float df = W * (data[pix_data] - model_ref[pix]); + + chisq += SQ(df); + + float df_obj = (model_ref[pix] - model_obj[pix]) / *dObj; + float df_sky = (model_ref[pix] - model_sky[pix]) / *dSky; + float df_bck = (model_ref[pix] - model_bck[pix]) / *dBck; + + dF_obj += df*df_obj; + dF_sky += df*df_sky; + dF_bck += df*df_bck; + + d2F_obj_obj += W * SQ(df_obj); + d2F_sky_sky += W * SQ(df_sky); + d2F_bck_bck += W * SQ(df_bck); + + d2F_obj_sky += W * df_obj * df_sky; + d2F_obj_bck += W * df_obj * df_bck; + d2F_sky_bck += W * df_sky * df_bck; + } + } + + // these are the elements of Ax = B + ALLOCATE_PTR (A, double *, 3); + ALLOCATE_PTR (B, double *, 3); + for (int i = 0; i < 3; i++) { + ALLOCATE (A[i], double, 3); + ALLOCATE (B[i], double, 1); + } + + A[0][0] = d2F_obj_obj * (1.0 + lambda); + A[1][1] = d2F_sky_sky * (1.0 + lambda); + A[2][2] = d2F_bck_bck * (1.0 + lambda); + + A[0][1] = A[1][0] = d2F_obj_sky; + A[1][2] = A[2][1] = d2F_sky_bck; + A[0][2] = A[2][0] = d2F_obj_bck; + + B[0][0] = dF_obj; + B[1][0] = dF_sky; + B[2][0] = dF_bck; + + dgaussjordan (A, B, 3, 1); + + *dObj = B[0][0]; + *dSky = B[1][0]; + *dBck = B[2][0]; + + return chisq; +} + +static float deimos_get_chisq (float *buffer, float *vardata, float *model, int Nx, int Ny, int row, int *Npts) { int npts = 0; @@ -487,5 +501,12 @@ if (!isfinite(buffer[pix_buffer])) continue; if (!isfinite(model[pix_model])) continue; - chisq += SQ(buffer[pix_buffer] - model[pix_model]); + + if (vardata && !isfinite(vardata[pix_buffer])) continue; + float Var = vardata ? vardata[pix_buffer] : 1.0; + float W = (fabs(Var) < 0.01) ? 100.0 : 1.0 / Var; + + float dchisq = SQ(buffer[pix_buffer] - model[pix_model]) * W; + if (!isfinite(dchisq)) continue; + chisq += dchisq; npts ++; } @@ -494,42 +515,2 @@ return chisq; } - -int *deimos_sort_result (DeimosResult *result, int Nresult) { - - ALLOCATE_PTR (IDX, int, Nresult); - ALLOCATE_PTR (chisq, float, Nresult); - - for (int i = 0; i < Nresult; i++) { - IDX[i] = i; - chisq[i] = result[i].chisq; - } - - sort_float_index (chisq, IDX, Nresult); - free (chisq); - - return IDX; -} - -static float DEIMOS_A = 2.0; // value for random sample - -int deimos_get_random_sample (int Nresult) { - - // choose a bin from the range 0 - Nresult, with front-loaded weighting. - - // bin = Nresult * (exp(q) - exp(-A)) / (exp(A) - exp(-A)) - // q = 2A(x - 1/2) : random number between -A and +A - // x : random number between 0 and 1 - - // f(q;A) = (exp(q) - exp(-A)) / (exp(A) - exp(-A)) - // f(q;A) ranges from 0 to +1 as x ranges from 0 to +1 - - float x = drand48(); - float q = 2*DEIMOS_A*(x - 0.5); - float f = (exp(q) - exp(-DEIMOS_A)) / (exp(DEIMOS_A) - exp(-DEIMOS_A)); - // XXX optimization : precalculate and save the elements of this function - - int bin = floor(Nresult * f); - - return bin; -} -# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c (revision 41170) @@ -0,0 +1,538 @@ +# include "data.h" +# include "deimos.h" + +# if (0) +int deimos_fitobj (int argc, char **argv) { + OHANA_UNUSED_PARAM(argc); + OHANA_UNUSED_PARAM(argv); + return FALSE; +} + +# else + +/* + this is starting to work OK. some improvements to make + * use Gaussdev to sample + * do not scale down range (or user-set scale-down) + */ + +// internal functions to fitobj +static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); +int *deimos_sort_result (DeimosResult *result, int Nresult); +int deimos_get_random_sample (int Nresult); +static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput); + +int USE_GAUSS_DEV = FALSE; + +int deimos_fitobj (int argc, char **argv) { + + // input parameters: + // * buffer : 2D image of slit (full frame or cutout?) + // * trace : spline fit of slit central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? + + // in-out parameters: + // * obj : vector of object flux vs y-coord (starting guess and result) + // * sky : vector of local sky signal vs y-coord + // * bck : vector of extra-slit background flux vs y-coord + + int N; + + Spline *trace = NULL; + Vector *profile = NULL; + + Vector *obj = NULL; + Vector *sky = NULL; + Vector *bck = NULL; + + Vector *psf = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + ohana_gaussdev_init (); + + fprintf (stderr, "use deimos fitalt instead\n"); + return FALSE; + + // a crude noise model valid for both the guess vectors and the + // data buffer: + float gain = 1.0; + if ((N = get_argument (argc, argv, "-gain"))) { + remove_argument (N, &argc, argv); + gain = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float noise = 0.0; + if ((N = get_argument (argc, argv, "-noise"))) { + remove_argument (N, &argc, argv); + noise = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float nsigma = 2.0; + if ((N = get_argument (argc, argv, "-nsigma"))) { + remove_argument (N, &argc, argv); + nsigma = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float scale = 0.75; + if ((N = get_argument (argc, argv, "-scale"))) { + remove_argument (N, &argc, argv); + scale = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + + int Niter = 25; + if ((N = get_argument (argc, argv, "-iter"))) { + remove_argument (N, &argc, argv); + Niter = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int Ntry = 5; + if ((N = get_argument (argc, argv, "-try"))) { + remove_argument (N, &argc, argv); + Ntry = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int Nrow = 5; + if ((N = get_argument (argc, argv, "-row"))) { + remove_argument (N, &argc, argv); + Nrow = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + + USE_GAUSS_DEV = FALSE; + if ((N = get_argument (argc, argv, "-gaussdev"))) { + remove_argument (N, &argc, argv); + USE_GAUSS_DEV = TRUE; + } + + int SAVE_ALL_VECTORS = FALSE; + if ((N = get_argument (argc, argv, "-save-all-vectors"))) { + remove_argument (N, &argc, argv); + SAVE_ALL_VECTORS = TRUE; + } + + int SAVE_MIN_VECTORS = FALSE; + if ((N = get_argument (argc, argv, "-save-min-vectors"))) { + remove_argument (N, &argc, argv); + SAVE_MIN_VECTORS = TRUE; + } + + // Input parameters: + if ((N = get_argument (argc, argv, "-trace"))) { + remove_argument (N, &argc, argv); + if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-profile"))) { + remove_argument (N, &argc, argv); + if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf"))) { + remove_argument (N, &argc, argv); + if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-stilt"))) { + remove_argument (N, &argc, argv); + stilt = atof (argv[N]); + remove_argument (N, &argc, argv); + } else { goto usage; } + + // In-Out parameters: + if ((N = get_argument (argc, argv, "-object"))) { + remove_argument (N, &argc, argv); + if ((obj = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-sky"))) { + remove_argument (N, &argc, argv); + if ((sky = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-backgnd"))) { + remove_argument (N, &argc, argv); + if ((bck = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if (argc != 3) goto usage; + + // buffer is the full image, Xref is reference coordinate of the profile in the buffer + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) + if (profile->Nelements % 2 == 0) { + gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); + return FALSE; + } + if (psf->Nelements % 2 == 0) { + gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); + return FALSE; + } + + // observed buffer size + int Nx = buffer[0].matrix.Naxis[0]; + int Ny = buffer[0].matrix.Naxis[1]; + float *bufVal = (float *) buffer->matrix.buffer; + + // obj, sky, bck must be consistent with data + if (Ny != obj->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (object)\n"); + return FALSE; + } + if (Ny != sky->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (sky)\n"); + return FALSE; + } + if (Ny != bck->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (backgnd)\n"); + return FALSE; + } + + // for the cross-dispersion reference pixle, use the user value or set to Nx/2 if < 0 + deimos_set_cross_ref (atoi(argv[2]), Nx); + deimos_make_kernel (stilt, Nx); + + // the functions below accept an opihi_flt array for object, sky, background + opihi_flt *objVal = obj->elements.Flt; + opihi_flt *skyVal = sky->elements.Flt; + opihi_flt *bckVal = bck->elements.Flt; + + int NRESULT = Niter + Ntry*Niter + 1; + ALLOCATE_PTR (result, DeimosResult, NRESULT); + + Vector **objOut = NULL; + Vector **skyOut = NULL; + Vector **bckOut = NULL; + Vector *objMin = NULL; + Vector *skyMin = NULL; + Vector *bckMin = NULL; + + // Save all result vectors for each of obj, sky, bck: + if (SAVE_ALL_VECTORS) { + ALLOCATE (objOut, Vector *, NRESULT); + ALLOCATE (skyOut, Vector *, NRESULT); + ALLOCATE (bckOut, Vector *, NRESULT); + for (int i = 0; i < NRESULT; i++) { + char name[64]; + snprintf (name, 64, "obj_%04d", i); if ((objOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objOut[i], OPIHI_FLT, Ny); + snprintf (name, 64, "sky_%04d", i); if ((skyOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyOut[i], OPIHI_FLT, Ny); + snprintf (name, 64, "bck_%04d", i); if ((bckOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckOut[i], OPIHI_FLT, Ny); + for (int iy = 0; iy < Ny; iy++) { + objOut[i]->elements.Flt[iy] = 0.0; + skyOut[i]->elements.Flt[iy] = 0.0; + bckOut[i]->elements.Flt[iy] = 0.0; + } + } + } + if (SAVE_MIN_VECTORS) { + if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); + if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); + if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); + for (int iy = 0; iy < Ny; iy++) { + objMin->elements.Flt[iy] = 0.0; + skyMin->elements.Flt[iy] = 0.0; + bckMin->elements.Flt[iy] = 0.0; + } + } + Vector *chisqVect; + if ((chisqVect = SelectVector ("chisqOut", ANYVECTOR, TRUE)) == NULL) { return FALSE; } + ResetVector (chisqVect, OPIHI_FLT, NRESULT); + for (int i = 0; i < NRESULT; i++) { chisqVect->elements.Flt[i] = 0.0; } + + // XXX temp hack: save chisq subset vectors for each row pass + int Npass = Ny / Nrow; + ALLOCATE_PTR (chiPass, Vector *, Npass); + for (int pass = 0; pass < Npass; pass++) { + char name[64]; + snprintf (name, 64, "chiPass_%03d", pass); + if ((chiPass[pass] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return FALSE; } ResetVector (chiPass[pass], OPIHI_FLT, NRESULT); + } + + float noiseVar = SQ(noise); + ALLOCATE_PTR (objNoise, float, Nrow); + ALLOCATE_PTR (skyNoise, float, Nrow); + ALLOCATE_PTR (bckNoise, float, Nrow); + + // work on segments of Nrow at a time + // XXX worry about last segment (< Nrow rows) + int pass = 0; + for (int row = 0; row < Ny - Nrow; row += Nrow, pass++) { + + int Nresult = 0; + + // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. + + // If I just use the value in a given bin to define the range, random noise for the + // faint end could have bad results (range ~ 0.0). For that matter, if I am in a + // region with large dynamic range (e.g., on an emission or absorption line), then we + // likely have large errors in how the guess follows the truth. + + /** old concept: + float objRange = 0, skyRange = 0, bckRange = 0; + for (int iy = 0; iy < Nrow; iy++) { + objRange = MAX(objVal[iy + row], objRange); + skyRange = MAX(skyVal[iy + row], skyRange); + bckRange = MAX(bckVal[iy + row], bckRange); + } + **/ + + // new concept: start with 2-sigma range, then ramp down below + for (int iy = 0; iy < Nrow; iy++) { + objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy + row]) / gain, noiseVar)); + skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy + row]) / gain, noiseVar)); + bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy + row]) / gain, noiseVar)); + } + + // first test is the input guess + if (1) { + opihi_flt *objtest = deimos_make_test (objVal, NULL, row, Nrow, TRUE); + opihi_flt *skytest = deimos_make_test (skyVal, NULL, row, Nrow, TRUE); + opihi_flt *bcktest = deimos_make_test (bckVal, NULL, row, Nrow, TRUE); + + // generate the model based on the test values + // objtest, skytest, bcktest are segments of the full parameter vectors + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + // XXX need to add in the per-pixel error or variance + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + fprintf (stderr, "guess: chisq: %f, Npts: %d\n", chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; + } + + // save the results + result[Nresult].chisq = chisq / (float) Npts; + result[Nresult].obj = objtest; + result[Nresult].sky = skytest; + result[Nresult].bck = bcktest; + Nresult ++; + } + + // run Niter iterations + for (int iter = 0; iter < Niter; iter ++) { + + // generate a set of test values for obj, sky, backgnd based on the current guess + opihi_flt *objtest = deimos_make_test (objVal, objNoise, row, Nrow, TRUE); + opihi_flt *skytest = deimos_make_test (skyVal, skyNoise, row, Nrow, TRUE); + opihi_flt *bcktest = deimos_make_test (bckVal, bckNoise, row, Nrow, TRUE); + + for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { + fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", + objVal[itmp + row], skyVal[itmp + row], bckVal[itmp + row], + objtest[itmp], skytest[itmp], bcktest[itmp]); + } + + // generate the model based on the test values + // objtest, skytest, bcktest are segments of the full parameter vectors + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + // XXX need to add in the per-pixel error or variance + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + // fprintf (stderr, "iter: %d, chisq: %f, Npts: %d\n", iter, chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; + } + + // save the results + result[Nresult].chisq = chisq / (float) Npts; + result[Nresult].obj = objtest; + result[Nresult].sky = skytest; + result[Nresult].bck = bcktest; + Nresult ++; + } + + for (int try = 0; try < Ntry; try ++) { + // generate an index of results sorted by chisq + int *IDX = deimos_sort_result (result, Nresult); + + int Nnewset = 0; + + // XXX not sure this scaling factor concept is good + for (int iy = 0; iy < Nrow; iy++) { + objNoise[iy] = scale*objNoise[iy]; + skyNoise[iy] = scale*skyNoise[iy]; + bckNoise[iy] = scale*bckNoise[iy]; + } + + // run Niter iterations, selecting guess values from the result set + for (int iter = 0; iter < Niter; iter ++) { + int bin = deimos_get_random_sample (Nresult); + int entry = IDX[bin]; + // fprintf (stderr, "try: %d : %d, %f\n", bin, entry, result[entry].chisq); + + // generate a set of test values for obj, sky, backgnd based on the current guess + opihi_flt *objtest = deimos_make_test (result[entry].obj, objNoise, row, Nrow, FALSE); + opihi_flt *skytest = deimos_make_test (result[entry].sky, skyNoise, row, Nrow, FALSE); + opihi_flt *bcktest = deimos_make_test (result[entry].bck, bckNoise, row, Nrow, FALSE); + + for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { + fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", + result[entry].obj[itmp], result[entry].sky[itmp], result[entry].bck[itmp], + objtest[itmp], skytest[itmp], bcktest[itmp]); + } + + // generate the model based on the test values + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + // fprintf (stderr, "try: %d, iter: %d, chisq: %f, Npts: %d\n", try, iter, chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult + Nnewset] = chisq / (float) Npts; + } + + // save the results + result[Nresult + Nnewset].chisq = chisq / (float) Npts; + result[Nresult + Nnewset].obj = objtest; + result[Nresult + Nnewset].sky = skytest; + result[Nresult + Nnewset].bck = bcktest; + Nnewset ++; + } + Nresult += Nnewset; + } + + int *IDX = deimos_sort_result (result, Nresult); + + // ENDING: save the best result and info about the allowed chisq region + // int *index = sort_result (result, Nresult); + // copy result[index[0]].obj,sky,bck to obj,sky,bck + for (int i = 0; i < Nresult; i++) { + for (int iy = 0; iy < Nrow; iy++) { + if (SAVE_ALL_VECTORS) { + objOut[i]->elements.Flt[iy + row] = result[IDX[i]].obj[iy]; + skyOut[i]->elements.Flt[iy + row] = result[IDX[i]].sky[iy]; + bckOut[i]->elements.Flt[iy + row] = result[IDX[i]].bck[iy]; + } + } + chisqVect->elements.Flt[i] += result[IDX[i]].chisq; + } + if (SAVE_MIN_VECTORS) { + for (int iy = 0; iy < Nrow; iy++) { + objMin->elements.Flt[iy + row] = result[IDX[0]].obj[iy]; + skyMin->elements.Flt[iy + row] = result[IDX[0]].sky[iy]; + bckMin->elements.Flt[iy + row] = result[IDX[0]].bck[iy]; + } + } + } + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + return FALSE; +} + +/****************** fitobj Support Functions *******************/ + +static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput) { + + // we have a vector, guess, with some length >= row + Nrow. + // generate a set of random values in the vicinty of guess[row] to guess[row+Nrow] + // return vector is of length Nrow + + // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) + // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) + // sigma is always only Nrow long + + ALLOCATE_PTR (value, opihi_flt, Nrow); + + int start_pix = fullInput ? row : 0; + int stop_pix = fullInput ? row + Nrow : Nrow; + + int iv = 0; + for (int i = start_pix; i < stop_pix; i++, iv++) { + if (!sigma) { + value[iv] = guess[i]; + continue; + } + if (USE_GAUSS_DEV) { + value[iv] = ohana_gaussdev_rnd(guess[i], sigma[iv]); + continue; + } + + // uniform distribution: + value[iv] = sigma[iv]*(drand48() - 0.5) + guess[i]; + } + return value; +} + + +static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { + + int npts = 0; + float chisq = 0; + + for (int iy = 0; iy < Ny; iy++) { + + for (int ix = 0; ix < Nx; ix++) { + + int pix_buffer = ix + (iy + row)*Nx; + int pix_model = ix + iy*Nx; + + if (!isfinite(buffer[pix_buffer])) continue; + if (!isfinite(model[pix_model])) continue; + chisq += SQ(buffer[pix_buffer] - model[pix_model]); + npts ++; + } + } + *Npts = npts; + return chisq; +} + +int *deimos_sort_result (DeimosResult *result, int Nresult) { + + ALLOCATE_PTR (IDX, int, Nresult); + ALLOCATE_PTR (chisq, float, Nresult); + + for (int i = 0; i < Nresult; i++) { + IDX[i] = i; + chisq[i] = result[i].chisq; + } + + sort_float_index (chisq, IDX, Nresult); + free (chisq); + + return IDX; +} + +static float DEIMOS_A = 2.0; // value for random sample + +int deimos_get_random_sample (int Nresult) { + + // choose a bin from the range 0 - Nresult, with front-loaded weighting. + + // bin = Nresult * (exp(q) - exp(-A)) / (exp(A) - exp(-A)) + // q = 2A(x - 1/2) : random number between -A and +A + // x : random number between 0 and 1 + + // f(q;A) = (exp(q) - exp(-A)) / (exp(A) - exp(-A)) + // f(q;A) ranges from 0 to +1 as x ranges from 0 to +1 + + float x = drand48(); + float q = 2*DEIMOS_A*(x - 0.5); + float f = (exp(q) - exp(-DEIMOS_A)) / (exp(DEIMOS_A) - exp(-DEIMOS_A)); + // XXX optimization : precalculate and save the elements of this function + + int bin = floor(Nresult * f); + + return bin; +} +# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c (revision 41170) @@ -0,0 +1,299 @@ +# include "data.h" + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); +typedef enum {MODE_UPPER, MODE_LOWER} SigmoidMode; + +void sigmoid_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC); +void sigmoid_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int NptsAll, opihi_flt *par, int Npar, SigmoidMode mode, double limit); + +opihi_flt fsigmoidDeimos (opihi_flt, opihi_flt *, int, opihi_flt *); +opihi_flt rsigmoidDeimos (opihi_flt, opihi_flt *, int, opihi_flt *); + +double deimos_weight_cauchy (double x); + +// use these globals to carry into sigmoid_fit +static int VERBOSE = FALSE; +static int QUIET = FALSE; +static int MaxIterations = 10; + +int deimos_fitprofile (int argc, char **argv) { + + // fit simple slit window profile model to vector + // I am fitting a sigmoid at both edges + + // input parameters: + // * xprofile : extracted slit window coordinates (vector) + // * xprofile : extracted slit window profile (vector) + // * Xs, Xe : scale coordinates of window boundaries + + // output parameters: + // * sigmoid_left_center, etc : coordinates of window boundaries + + int N; + Vector *xprofile = NULL; + Vector *fprofile = NULL; + Vector *wprofile = NULL; + + QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + MaxIterations = 10; + if ((N = get_argument (argc, argv, "-max-iterations"))) { + remove_argument (N, &argc, argv); + MaxIterations = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 6) goto usage; + if ((xprofile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((fprofile = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((wprofile = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if (xprofile->Nelements != fprofile->Nelements) goto usage; + if (xprofile->Nelements != wprofile->Nelements) goto usage; + + // supplied guess for boundaries + double Xs = atof(argv[4]); + double Xe = atof(argv[5]); + + // output vectors of fits + Vector *Lprofile = SelectVector ("sigmoid_left", ANYVECTOR, TRUE); + if (Lprofile == NULL) return (FALSE); + MatchVector (Lprofile, xprofile, OPIHI_FLT); + Vector *Rprofile = SelectVector ("sigmoid_right", ANYVECTOR, TRUE); + if (Rprofile == NULL) return (FALSE); + MatchVector (Rprofile, xprofile, OPIHI_FLT); + + // first, use supplied Xs & Xe guess to make a guess for BckL, Sky, BckR + + float Sky_V = 0.0, BckL_V = 0.0, BckR_V = 0.0; + float Sky_N = 0.0, BckL_N = 0.0, BckR_N = 0.0; + for (int i = 0; i < xprofile->Nelements; i++) { + if (!isfinite(wprofile->elements.Flt[i])) continue; + if (!isfinite(fprofile->elements.Flt[i])) continue; + float wt = 1 / SQ(wprofile->elements.Flt[i]); + if (xprofile->elements.Flt[i] < Xs) { + BckL_V += fprofile->elements.Flt[i]*wt; + BckL_N += wt; + continue; + } + if (xprofile->elements.Flt[i] > Xe) { + BckR_V += fprofile->elements.Flt[i]*wt; + BckR_N += wt; + continue; + } + Sky_V += fprofile->elements.Flt[i]*wt; + Sky_N += wt; + } + float Sky = Sky_V / Sky_N; + float BckL = BckL_V / BckL_N; + float BckR = BckR_V / BckR_N; + + // temporary storage vectors + opihi_flt par[4]; + + float Xmid = 0.5*(Xs + Xe); + + int Npar = 4; + par[0] = Xs; + par[1] = 1; + par[2] = Sky; + par[3] = BckL; + sigmoid_fit_irls (xprofile->elements.Flt, fprofile->elements.Flt, wprofile->elements.Flt, Lprofile->elements.Flt, xprofile->Nelements, par, Npar, MODE_LOWER, Xmid); + set_variable ("sigmoid_left_center", par[0]); + set_variable ("sigmoid_left_sigma", par[1]); + set_variable ("sigmoid_left_top", par[2]); + set_variable ("sigmoid_left_bottom", par[3]); + + par[0] = Xe; + par[1] = 1; + par[2] = Sky; + par[3] = BckR; + sigmoid_fit_irls (xprofile->elements.Flt, fprofile->elements.Flt, wprofile->elements.Flt, Rprofile->elements.Flt, xprofile->Nelements, par, Npar, MODE_UPPER, Xmid); + set_variable ("sigmoid_right_center", par[0]); + set_variable ("sigmoid_right_sigma", par[1]); + set_variable ("sigmoid_right_top", par[2]); + set_variable ("sigmoid_right_bottom", par[3]); + + return TRUE; + +usage: + gprint (GP_ERR, "USAGE: deimos fitprofile xprof fprof wprof Xs Xe\n"); + return FALSE; +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt rsigmoidDeimos (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + if (dpar) { + dpar[0] = par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + } + + return (f); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt fsigmoidDeimos (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (-z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = -r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = -par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = -par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + if (dpar) { + dpar[0] = -par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + } + + return (f); +} + +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 + +// wprofile is the true poisson error +void sigmoid_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int NptsAll, opihi_flt *par, int Npar, SigmoidMode mode, double limit) { + + FitFunc *FUNC = (mode == MODE_LOWER) ? fsigmoidDeimos : rsigmoidDeimos; + + ALLOCATE_PTR (par_old, opihi_flt, Npar); + + // allocate these as global static in this file? + ALLOCATE_PTR (xtemp, opihi_flt, NptsAll); + ALLOCATE_PTR (ytemp, opihi_flt, NptsAll); + ALLOCATE_PTR (wtemp, opihi_flt, NptsAll); + ALLOCATE_PTR (otemp, opihi_flt, NptsAll); + + // extract the points on the right side and fit + int Npts = 0; + for (int i = 0; i < NptsAll; i++) { + if (!isfinite(wprofile[i])) continue; + if (!isfinite(fprofile[i])) continue; + int keep = (mode == MODE_UPPER) ^ (xprofile[i] < limit); + if (keep) { + xtemp[Npts] = xprofile[i]; + ytemp[Npts] = fprofile[i]; + wtemp[Npts] = (wprofile[i] == 0.0) ? 1.0 : wprofile[i]; // crude + Npts ++; + } + } + + // define initial weights as 1.0 + ALLOCATE_PTR (wt, opihi_flt, Npts); + for (int i = 0; i < Npts; i++) { + wt[i] = 1.0 / SQ(wtemp[i]); + } + + // initial fit with ordinary least-squares + // otemp is the fitted value of the function at xtemp + sigmoid_fit (xtemp, ytemp, wt, otemp, Npts, par, Npar, FUNC); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < MaxIterations); iterations++) { + + for (int i = 0; i < Npts; i++) { + wt[i] = deimos_weight_cauchy ((ytemp[i] - otemp[i]) / wtemp[i]) / SQ(wtemp[i]); + } + + sigmoid_fit (xtemp, ytemp, wt, otemp, Npts, par, Npar, FUNC); + + // save this solution + for (int i = 0; i < Npar; i++) { + par_old[i] = par[i]; + } + + converged = TRUE; + for (int i = 0; i < Npts; i++) { + if ((fabs(par[i] - par_old[i]) > FIT_TOLERANCE * fabs(par[i])) && + (fabs(par[i] - par_old[i]) > FLT_TOLERANCE)) + converged = FALSE; + } + } + + for (int i = 0; i < NptsAll; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + + FREE (xtemp); + FREE (ytemp); + FREE (wtemp); + FREE (otemp); + + FREE (par_old); + FREE (wt); +} + +void sigmoid_fit (opihi_flt *xprofile, opihi_flt *yprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC) { + + double ochisq = mrqinit (xprofile, yprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + double dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + double chisq = mrqmin (xprofile, yprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + dchisq = ochisq - chisq; + ochisq = chisq; + if (VERBOSE) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (VERBOSE) gprint (GP_ERR, "%d iterations\n", Niter); + + if (!QUIET) gprint (GP_LOG, "sigmoid @ %f (%f) : top %f bottom %f\n", par[0], par[1], par[2], par[3]); + + for (int i = 0; i < Npts; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + + mrqfree (Npar); +} + +double deimos_weight_cauchy (double x) { + double r = x / 2.385; + return (1.0 / (1.0 + SQ(r))); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c (revision 41170) @@ -1,9 +1,11 @@ # include "data.h" +# define IRLS_TOLERANCE 1e-4 + +float weight_cauchy_square_flt (float x2); +static void fitflux (float *Fwind, float *Fprof, float *weight, int Nx, double *flux, double *sky); int deimos_fitslit (int argc, char **argv) { - // fitslit (window) (slit) (flux) (sky) - - // int N; + // fitslit (window) (vari) (slit) (flux) (sky) Vector *flux = NULL; @@ -11,9 +13,25 @@ Buffer *wind = NULL; + Buffer *vari = NULL; Buffer *slit = NULL; - if (argc != 5) { - gprint (GP_ERR, "USAGE: deimos fitslit (window) (slit) (flux) (sky)\n"); - gprint (GP_ERR, " inputs: window (observed 2D flux), slit (model 2D flux)\n"); + int N; + int Niter = 10; + if ((N = get_argument (argc, argv, "-irls-iter"))) { + remove_argument (N, &argc, argv); + Niter = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + Buffer *model = NULL; + if ((N = get_argument (argc, argv, "-model"))) { + remove_argument (N, &argc, argv); + if ((model = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 6) { + gprint (GP_ERR, "USAGE: deimos fitslit (window) (variance) (slit) (flux) (sky)\n"); + gprint (GP_ERR, " inputs: window (observed 2D flux), variance (on 2D flux), slit (model 2D flux)\n"); gprint (GP_ERR, " outputs: flux (best-fit 1D flux), sky (best-fit 1D background)\n"); return FALSE; @@ -22,9 +40,8 @@ // XXX I probably should rename FindSpline as SelectSpline and give it the same behavior if ((wind = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - if ((slit = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); - if ((flux = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); - if ((sky = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); - - // XXX confirm slit and wind have same dimensions + if ((vari = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((slit = SelectBuffer (argv[3], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((flux = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((sky = SelectVector (argv[5], ANYVECTOR, TRUE)) == NULL) return (FALSE); // define the output window @@ -32,42 +49,117 @@ int Ny = wind[0].matrix.Naxis[1]; + // confirm wind, vari, slit have same dimensions if (Nx != slit[0].matrix.Naxis[0]) { gprint (GP_ERR, "size mismatch in fitslit for window and slit\n"); return FALSE; } if (Ny != slit[0].matrix.Naxis[1]) { gprint (GP_ERR, "size mismatch in fitslit for window and slit\n"); return FALSE; } + if (Nx != vari[0].matrix.Naxis[0]) { gprint (GP_ERR, "size mismatch in fitslit for window and variance\n"); return FALSE; } + if (Ny != vari[0].matrix.Naxis[1]) { gprint (GP_ERR, "size mismatch in fitslit for window and variance\n"); return FALSE; } ResetVector (flux, OPIHI_FLT, Ny); ResetVector (sky, OPIHI_FLT, Ny); + if (model) { + if (!CreateBuffer (model, Nx, Ny, -32, 1.0, 0.0)) { gprint (GP_ERR, "error generating output model buffer\n"); return FALSE; } + } + float *Fwind = (float *) wind[0].matrix.buffer; + float *Fvari = (float *) vari[0].matrix.buffer; float *Fprof = (float *) slit[0].matrix.buffer; + float *FmodOut = model ? (float *) model[0].matrix.buffer : NULL; + + ALLOCATE_PTR (weight, float, Nx); + ALLOCATE_PTR (rawwgt, float, Nx); // loop over the rows for (int iy = 0; iy < Ny; iy++) { - // calculate elements of the chi-square for this row: + double F = 0.0, S = 0.0; - double R = 0.0, P = 0.0, P2 = 0.0, f = 0.0, fp = 0.0; - double wt = 1.0; - + // set weight based on the variance for (int ix = 0; ix < Nx; ix++) { - - float fxy = Fwind[ix + iy*Nx]; - float pxy = Fprof[ix + iy*Nx]; - - R += wt; - P += pxy*wt; - P2 += pxy*pxy*wt; - f += fxy*wt; - fp += fxy*pxy*wt; + float vari = Fvari[ix + iy*Nx]; + weight[ix] = !isfinite(vari) || (fabs(vari) < 1e-6) ? 1.0 : 1.0 / vari; + rawwgt[ix] = weight[ix]; } - double det = R*P2 - P*P; - double S = (P2*f - P*fp) / det; - double F = (R*fp - P*f) / det; + // calculate elements of the chi-square for this row: + fitflux (&Fwind[iy*Nx], &Fprof[iy*Nx], weight, Nx, &F, &S); + + // do an IRLS loop here to reject outliers + int converged = FALSE; + for (int iter = 0; (iter < Niter) && !converged; iter++) { + + double Flast = F; + double Slast = S; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int ix = 0; ix < Nx; ix++) { + + // F & S are model parameters; compare the model and observed values for each pixel + float Fmodel = S + F*Fprof[ix + iy*Nx]; + float dV = (Fwind[ix + iy*Nx] - Fmodel); + float d2 = SQ(dV) * rawwgt[ix]; + + float Mod = weight_cauchy_square_flt (d2); + weight[ix] = Mod * rawwgt[ix]; + } + fitflux (&Fwind[iy*Nx], &Fprof[iy*Nx], weight, Nx, &F, &S); + + float dF = fabs(F - Flast); + float dS = fabs(S - Slast); + + if ((dF < F * IRLS_TOLERANCE) && (dS < S * IRLS_TOLERANCE)) converged = TRUE; + } + + if (FmodOut) { + for (int ix = 0; ix < Nx; ix++) { + // F & S are model parameters; compare the model and observed values for each pixel + float Fmodel = S + F*Fprof[ix + iy*Nx]; + float dV = (Fwind[ix + iy*Nx] - Fmodel); + float d2 = SQ(dV) * rawwgt[ix]; + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) { + FmodOut[ix + iy*Nx] = NAN; + } else { + FmodOut[ix + iy*Nx] = Fmodel; + } + } + } flux->elements.Flt[iy] = F; sky->elements.Flt[iy] = S; } - return TRUE; } +// weight is 1 / variance +// Fwind, Fprof are pointers to the start of a single row +// Fwind is the observed flux in the window +// Fprof is the slit profile +void fitflux (float *Fwind, float *Fprof, float *weight, int Nx, double *flux, double *sky) { + + // calculate elements of the chi-square fit for this row: + + double R = 0.0, P = 0.0, P2 = 0.0, f = 0.0, fp = 0.0; + + for (int ix = 0; ix < Nx; ix++) { + + double fxy = Fwind[ix]; + double pxy = Fprof[ix]; + + // skip NAN / Inf pixels in either window or slit + if (!isfinite(fxy)) continue; + if (!isfinite(pxy)) continue; + + double wt = (weight == NULL) ? 1.0 : weight[ix]; + R += wt; + P += pxy*wt; + P2 += pxy*pxy*wt; + f += fxy*wt; + fp += fxy*pxy*wt; + } + + double det = R*P2 - P*P; + *sky = (P2*f - P*fp) / det; + *flux = (R*fp - P*f) / det; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c (revision 41170) @@ -3,29 +3,17 @@ // temporary storage used in local functions: -float *tmpv = NULL; -float *tmpx = NULL; - -// pass in a pointer to the row (&buffV[iy*Nx]) -float get_segment_median (float *buffV, float Xs, float Xe, float *Xref) { - - // get the pixels in the left portion below profile: - // XXX: worry about the effect of a fractional offset - for (int ix = Xs + 0.5; ix < Xe + 0.5; ix++) { - int i = ix - Xs; - tmpv[i] = buffV[ix]; - tmpx[i] = ix; - } - int Nv = Xe - Xs; - - // generate a statistic from these pixels: mean, median, etc? - fsort (tmpv, Nv); - int Nv2 = Nv/2; // Nv == 1, Nv2 = 0, Nv == 2, Nv2 = 1, Nv == 3, Nv2 = 1, Nv == 4, Nv == 2 - float V = (Nv % 2) ? tmpv[Nv2] : 0.5*(tmpv[Nv2-1] + tmpv[Nv2]); - - // calculate average coordinate (not flux-weighted...) - // XXX : if we allow (and exclude) NAN-valued pixels, we can calculate this more explicitly - *Xref = 0.5*(Xe + Xs); - - return V; +static float *tmpv = NULL; +static float *tmpx = NULL; + +float get_median_flt (float *val, int Nval) { + + if (Nval <= 0) return NAN; + + int Nval2 = Nval / 2; + + fsort (val, Nval); + float value = (Nval % 2) ? val[Nval2] : 0.5*(val[Nval2-1] + val[Nval2]); + + return value; } @@ -37,7 +25,12 @@ // not used : LSF -- not needed (measurement per disp pixel) + // This version uses the median of the pixels in the image which are + // in regions where the slit profile is < 15% (as opposed to the + // getobj version which attempts to fit both background and sky regions + // input parameters: // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord + // * slit_trace : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of psf central x pos vs y-coord // * profile : slit window profile (vector) // * PSF : point-spread function vector (flux normalized, x-dir) @@ -51,5 +44,7 @@ int N; - Spline *trace = NULL; + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; Vector *profile = NULL; @@ -64,14 +59,28 @@ float stilt = 0.0; // angle of the slit - int SaveSegments = FALSE; - if ((N = get_argument (argc, argv, "-save-segments"))) { - remove_argument (N, &argc, argv); - SaveSegments = TRUE; - } + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } else { goto usage; } @@ -137,47 +146,4 @@ ResetVector (backgnd, OPIHI_FLT, Ny); - // I need to generate a sky-left and sky-right vector in a region on the slit portion of - // the profile, but off the object. Then each of these vectors can be shifted up and down - // based on stilt. Use these sky vectors to predict the sky under the PSF - - // *** scan the profile to determine the background and sky cut points *** - - // values below in the profile coordinate system need to be shifted by (Xref - Nprofile/2) - // to match the image coordinate system - - int B_s_l = 0, B_e_l = -1; // left background window - int B_s_r = -1, B_e_r = Nprofile - 1; // right background window - int S_left = -1, S_right = -1; // top portion of profile - for (int i = 0; i < Nprofile; i++) { - if ((B_e_l < 0) && (profile->elements.Flt[i] > 0.10)) B_e_l = i - 1; - if ((B_e_l >= 0) && (S_left < 0) && (profile->elements.Flt[i] > 0.95)) S_left = i; - if ((S_left >= 0) && (S_right < 0) && (profile->elements.Flt[i] < 0.95)) S_right = i - 1; - if ((S_right >= 0) && (profile->elements.Flt[i] < 0.10)) { B_s_r = i; break; } - } - if (B_e_l == B_s_l) B_e_l ++; - if (B_e_r == B_s_r) B_s_r --; - - // S_left to S_right defines are window at the top, choose 0.05 - 0.15, 0.85 - 0.95 - int N_top = S_right - S_left + 1; - - // if N_top < 20, these jumps might be too small - // if N_top < 20, these jumps might be too small - int S_s_l = S_left + 0.05*N_top; - int S_e_l = S_left + 0.15*N_top; - if (S_e_l == S_s_l) S_e_l ++; - int S_s_r = S_left + 0.85*N_top; - int S_e_r = S_left + 0.95*N_top; - if (S_e_r == S_s_r) S_s_r --; - - gprint (GP_ERR, "windows: %.1f - %.1f, %.1f - %.1f, %.1f - %.1f, %.1f - %.1f\n", - B_s_l + Xref - Xprofile, B_e_l + Xref - Xprofile, - S_s_l + Xref - Xprofile, S_e_l + Xref - Xprofile, - S_s_r + Xref - Xprofile, S_e_r + Xref - Xprofile, - B_s_r + Xref - Xprofile, B_e_r + Xref - Xprofile); - - // OPEN QUESTIONS: - // NOTE: if the range of pixels in the windows is too large, then the tilt will wash out the signal. - // this pass is a guess, so perhaps that does not matter, but there is clearly a trade-off here. - // generate two buffers to store the flux and coordinate for the background, sky, object pixels: ALLOCATE (tmpv, float, Nx); @@ -186,75 +152,9 @@ float *buffV = (float *) buffer[0].matrix.buffer; - // use these to store the flux vectors - ALLOCATE_PTR (bckF_lf, float, Ny); - ALLOCATE_PTR (bckF_rt, float, Ny); - ALLOCATE_PTR (skyF_lf, float, Ny); - ALLOCATE_PTR (skyF_rt, float, Ny); - - // use these to store the x-coord vectors - ALLOCATE_PTR (bckX_lf, float, Ny); - ALLOCATE_PTR (bckX_rt, float, Ny); - ALLOCATE_PTR (skyX_lf, float, Ny); - ALLOCATE_PTR (skyX_rt, float, Ny); - float dxMax = 0; // maximum distance between sky regions - Vector *skyLsV = NULL, *skyLeV = NULL, *skyLfV = NULL, *skyRsV = NULL, *skyReV = NULL, *skyRfV = NULL; - Vector *bckLsV = NULL, *bckLeV = NULL, *bckLfV = NULL, *bckRsV = NULL, *bckReV = NULL, *bckRfV = NULL; - if (SaveSegments) { - skyLsV = SelectVector ("skyLs", ANYVECTOR, TRUE); ResetVector (skyLsV, OPIHI_FLT, Ny); - skyLeV = SelectVector ("skyLe", ANYVECTOR, TRUE); ResetVector (skyLeV, OPIHI_FLT, Ny); - skyLfV = SelectVector ("skyLf", ANYVECTOR, TRUE); ResetVector (skyLfV, OPIHI_FLT, Ny); - skyRsV = SelectVector ("skyRs", ANYVECTOR, TRUE); ResetVector (skyRsV, OPIHI_FLT, Ny); - skyReV = SelectVector ("skyRe", ANYVECTOR, TRUE); ResetVector (skyReV, OPIHI_FLT, Ny); - skyRfV = SelectVector ("skyRf", ANYVECTOR, TRUE); ResetVector (skyRfV, OPIHI_FLT, Ny); - bckLsV = SelectVector ("bckLs", ANYVECTOR, TRUE); ResetVector (bckLsV, OPIHI_FLT, Ny); - bckLeV = SelectVector ("bckLe", ANYVECTOR, TRUE); ResetVector (bckLeV, OPIHI_FLT, Ny); - bckLfV = SelectVector ("bckLf", ANYVECTOR, TRUE); ResetVector (bckLfV, OPIHI_FLT, Ny); - bckRsV = SelectVector ("bckRs", ANYVECTOR, TRUE); ResetVector (bckRsV, OPIHI_FLT, Ny); - bckReV = SelectVector ("bckRe", ANYVECTOR, TRUE); ResetVector (bckReV, OPIHI_FLT, Ny); - bckRfV = SelectVector ("bckRf", ANYVECTOR, TRUE); ResetVector (bckRfV, OPIHI_FLT, Ny); - } - - // loop over the y positions - for (int iy = 0; iy < Ny; iy++) { - - // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dXtrace = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); - - // Xref is the reference coordinate of the profile on the image - // Xprofile is the center of the profile. - // dXtrace follows the x-dir variations of the slit profile - float Xtrace = Xref - Xprofile + dXtrace; - - bckF_lf[iy] = get_segment_median(&buffV[iy*Nx], B_s_l + Xtrace, B_e_l + Xtrace, &bckX_lf[iy]); - bckF_rt[iy] = get_segment_median(&buffV[iy*Nx], B_s_r + Xtrace, B_e_r + Xtrace, &bckX_rt[iy]); - skyF_lf[iy] = get_segment_median(&buffV[iy*Nx], S_s_l + Xtrace, S_e_l + Xtrace, &skyX_lf[iy]); - skyF_rt[iy] = get_segment_median(&buffV[iy*Nx], S_s_r + Xtrace, S_e_r + Xtrace, &skyX_rt[iy]); - dxMax = MAX (dxMax, skyX_rt[iy] - skyX_lf[iy]); // this is a constant value unless we ignore some of the pixels - // assert skyX_rt > skyX_lf ? - - if (SaveSegments) { - skyLsV->elements.Flt[iy] = S_s_l + Xtrace; - skyLeV->elements.Flt[iy] = S_e_l + Xtrace; - skyLfV->elements.Flt[iy] = skyF_lf[iy]; - skyRsV->elements.Flt[iy] = S_s_r + Xtrace; - skyReV->elements.Flt[iy] = S_e_r + Xtrace; - skyRfV->elements.Flt[iy] = skyF_rt[iy]; - bckLsV->elements.Flt[iy] = B_s_l + Xtrace; - bckLeV->elements.Flt[iy] = B_e_l + Xtrace; - bckLfV->elements.Flt[iy] = bckF_lf[iy]; - bckRsV->elements.Flt[iy] = B_s_r + Xtrace; - bckReV->elements.Flt[iy] = B_e_r + Xtrace; - bckRfV->elements.Flt[iy] = bckF_rt[iy]; - } - } - // extract sky-subtracted values for the psf fit, using the two sky vectors to predict the sky - float dyMax = dxMax*sin(stilt*RAD_DEG); // dxMax is based on the profile width, calculated above - - // find nominal dy_lf, dy_rt offsets for the sky vectors. - int dy_lf = -dyMax / 2; - int dy_rt = +dyMax / 2; + float sin_stilt = sin(stilt*RAD_DEG); + float dyMax = dxMax*sin_stilt; // dxMax is based on the profile width, calculated above // we are fitting this PSF to the data: @@ -273,32 +173,99 @@ } - - // Ys, Ye based on the dyMax value above (skip the first few lines to avoid running out of bounds on sky) for (int iy = fabs(dyMax); iy < Ny - fabs(dyMax); iy++) { - // X_lf & X_rt are calculated in image x-coords - float S_lf = skyF_lf[iy + dy_lf]; - float S_rt = skyF_rt[iy + dy_rt]; - float X_lf = skyX_lf[iy + dy_lf]; - float X_rt = skyX_rt[iy + dy_rt]; - // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dXtrace = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dXtraceSLIT = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + float dXtracePSF = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); // Xref is the reference coordinate of the profile on the image // Xpsf is the center of the PSF. // dXtrace follows the x-dir variations of the slit profile (or PSF?) - float Xtrace = Xref - Xpsf + dXtrace; - - // include the PSF trace and restrict the fit below to portions where the profile is > 0.95 + + // XtracePSF is the offset between the buffer ix and the psf elements at this location + float XtracePSF = Xref - Xpsf + dXtraceSLIT + dXtracePSF; + + // XtraceSLIT is the offset between the buffer ix and the slit profile elements at this location + float XtraceSLIT = Xref - Xprofile + dXtraceSLIT; + + // measure the background outside the slit + + // select pixels from the image which are in the background regions (profile < 0.15) + int Nbck = 0; + int Nsky = 0; + for (int ix = 0; ix < Nx; ix ++) { + int islit = ix - XtraceSLIT - 0.5; // slit window coordinate + int ipsf = ix - XtracePSF - 0.5; // psf model coordinate + + if (islit < 0) continue; + if (islit >= Nprofile) continue; + + // coordinate of the PSF center: + int dxpsf = ipsf - Xpsf; + int dypsf = dxpsf * sin_stilt; + + int iY = iy + dypsf; + + if (iy == 6000) { + fprintf (stderr, "bck : %d %d %d : %f : %f\n", ix, islit, ipsf, buffV[ix + iY*Nx], profile->elements.Flt[islit]); + } + + // pixels in the off-slit background: + if (profile->elements.Flt[islit] < 0.15) { + tmpv[Nbck] = buffV[ix + iY*Nx]; + if (!isfinite(tmpv[Nbck])) continue; + Nbck ++; + continue; + } + + // the psf window can be smaller than the slit window + if (ipsf < 0) continue; + if (ipsf >= Npsf) continue; + + // pixels in the on-slit sky: + if ((profile->elements.Flt[islit] > 0.95) && (psfV[ipsf] < 0.15)) { + tmpx[Nsky] = buffV[ix + iY*Nx]; + if (!isfinite(tmpx[Nsky])) continue; + Nsky ++; + } + } + backgnd->elements.Flt[iy] = get_median_flt (tmpv, Nbck); + float So = get_median_flt (tmpx, Nsky); + sky->elements.Flt[iy] = So; // now I need to find the limits of the PSF in the buffer at this iy location - for (int ipsf = 0; ipsf < Npsf; ipsf ++) { - - int ix = ipsf + Xtrace + 0.5; // image coordinate (nearest pixel center) - float So = S_lf + (ix - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak - - tmpv[ipsf] = buffV[ix + iy*Nx] - So; - tmpx[ipsf] = ipsf; + int Ntmp = 0; + for (int ix = 0; ix < Nx; ix ++) { + int islit = ix - XtraceSLIT - 0.5; // slit window coordinate + int ipsf = ix - XtracePSF - 0.5; // psf model coordinate + + if (islit < 0) continue; + if (islit >= Nprofile) continue; + + // the psf window can be smaller than the slit window + if (ipsf < 0) continue; + if (ipsf >= Npsf) continue; + + // coordinate of the PSF center: + int dxpsf = ipsf - Xpsf; + int dypsf = dxpsf * sin_stilt; + + int iY = iy + dypsf; + + // filter on the PSF value? + if (profile->elements.Flt[islit] < 0.95) continue; + + if (iy == 6000) { + fprintf (stderr, "%d %d %d : %f %f : %f : %f\n", ix, islit, ipsf, buffV[ix + iY*Nx] - So, psfV[ipsf], (buffV[ix + iY*Nx] - So)*psfV[ipsf], profile->elements.Flt[islit]); + } + + tmpv[Ntmp] = buffV[ix + iY*Nx] - So; + tmpx[Ntmp] = psfV[ipsf]; + + if (!isfinite(tmpv[Ntmp])) continue; + if (!isfinite(tmpx[Ntmp])) continue; + + Ntmp ++; } @@ -306,31 +273,16 @@ // now fit the PSF to tmpv by calculating a normalization float S1 = 0.0, S2 = 0.0; - for (int ipsf = 0; ipsf < Npsf; ipsf++) { + for (int ipsf = 0; ipsf < Ntmp; ipsf++) { if (iy == 6000) { - fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], psfV[ipsf], tmpv[ipsf]*psfV[ipsf]); - } - S1 += tmpv[ipsf]*psfV[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) - S2 += psfV[ipsf]*psfV[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], tmpx[ipsf], tmpv[ipsf]*tmpx[ipsf]); + } + S1 += tmpv[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + S2 += tmpx[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) } - - int Xo = Xref - dXtrace; // center of PSF in image x-coordinates - float So = S_lf + (Xo - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak object->elements.Flt[iy] = S1 / S2; - sky->elements.Flt[iy] = So; - backgnd->elements.Flt[iy] = 0.5*(bckF_lf[iy] + bckF_rt[iy]); } free (tmpv); free (tmpx); - - free (bckF_lf); - free (bckF_rt); - free (skyF_lf); - free (skyF_rt); - - free (bckX_lf); - free (bckX_rt); - free (skyX_lf); - free (skyX_rt); return TRUE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c (revision 41170) @@ -0,0 +1,383 @@ +# include "data.h" + +// temporary storage used in local functions: + +static float *tmpv = NULL; +static float *tmpx = NULL; + +// pass in a pointer to the row (&buffV[iy*Nx]) +float get_segment_median (float *buffV, float Xs, float Xe, float *Xref) { + + // get the pixels in the left portion below profile: + // XXX: worry about the effect of a fractional offset + for (int ix = Xs + 0.5; ix < Xe + 0.5; ix++) { + int i = ix - Xs; + tmpv[i] = buffV[ix]; + tmpx[i] = ix; + } + int Nv = Xe - Xs; + + // generate a statistic from these pixels: mean, median, etc? + fsort (tmpv, Nv); + int Nv2 = Nv/2; // Nv == 1, Nv2 = 0, Nv == 2, Nv2 = 1, Nv == 3, Nv2 = 1, Nv == 4, Nv == 2 + float V = (Nv % 2) ? tmpv[Nv2] : 0.5*(tmpv[Nv2-1] + tmpv[Nv2]); + + // calculate average coordinate (not flux-weighted...) + // XXX : if we allow (and exclude) NAN-valued pixels, we can calculate this more explicitly + *Xref = 0.5*(Xe + Xs); + + return V; +} + +int deimos_getobj (int argc, char **argv) { + + // determine rough object model parameters based on quick stats + // input : buffer PSF trace profile stilt + // output : object sky background + // not used : LSF -- not needed (measurement per disp pixel) + + // input parameters: + // * buffer : 2D image of slit (full frame or cutout?) + // * slit_trace : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of psf central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? + + // output parameters: + // * object : vector of object flux vs y-coord + // * sky : vector of local sky signal vs y-coord + // * background : vector of extra-slit background flux vs y-coord + + int N; + + Spline *slit_trace = NULL; + Spline *psf_trace = NULL; + Vector *profile = NULL; + + Vector *object = NULL; + Vector *sky = NULL; + Vector *backgnd = NULL; + + Vector *psf = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + + int SaveSegments = FALSE; + if ((N = get_argument (argc, argv, "-save-segments"))) { + remove_argument (N, &argc, argv); + SaveSegments = TRUE; + } + + // Input parameters: + if ((N = get_argument (argc, argv, "-slit-trace"))) { + remove_argument (N, &argc, argv); + if ((slit_trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-profile"))) { + remove_argument (N, &argc, argv); + if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf"))) { + remove_argument (N, &argc, argv); + if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-stilt"))) { + remove_argument (N, &argc, argv); + stilt = atof (argv[N]); + remove_argument (N, &argc, argv); + } else { goto usage; } + + // Output parameters: + if ((N = get_argument (argc, argv, "-object"))) { + remove_argument (N, &argc, argv); + if ((object = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-sky"))) { + remove_argument (N, &argc, argv); + if ((sky = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-backgnd"))) { + remove_argument (N, &argc, argv); + if ((backgnd = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if (argc != 3) goto usage; + + // buffer is the full image, Xref is reference coordinate of the profile in the buffer + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + float Xref = atof(argv[2]); + + // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) + int Nprofile = profile->Nelements; + if (Nprofile % 2 == 0) { + gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); + return FALSE; + } + int Xprofile = Nprofile / 2; + + int Npsf = psf->Nelements; + if (Npsf % 2 == 0) { + gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); + return FALSE; + } + int Xpsf = Npsf / 2; + + // we will generate output vectors (object, sky, background) of length Ny + int Nx = buffer[0].matrix.Naxis[0]; + int Ny = buffer[0].matrix.Naxis[1]; + ResetVector (object, OPIHI_FLT, Ny); + ResetVector (sky, OPIHI_FLT, Ny); + ResetVector (backgnd, OPIHI_FLT, Ny); + + // I need to generate a sky-left and sky-right vector in a region on the slit portion of + // the profile, but off the object. Then each of these vectors can be shifted up and down + // based on stilt. Use these sky vectors to predict the sky under the PSF + + // *** scan the profile to determine the background and sky cut points *** + + // values below in the profile coordinate system need to be shifted by (Xref - Nprofile/2) + // to match the image coordinate system + + int B_s_l = 0, B_e_l = -1; // left background window + int B_s_r = -1, B_e_r = Nprofile - 1; // right background window + int S_left = -1, S_right = -1; // top portion of profile + for (int i = 0; i < Nprofile; i++) { + if ((B_e_l < 0) && (profile->elements.Flt[i] > 0.10)) B_e_l = i - 1; + if ((B_e_l >= 0) && (S_left < 0) && (profile->elements.Flt[i] > 0.95)) S_left = i; + if ((S_left >= 0) && (S_right < 0) && (profile->elements.Flt[i] < 0.95)) S_right = i - 1; + if ((S_right >= 0) && (profile->elements.Flt[i] < 0.10)) { B_s_r = i; break; } + } + if (B_e_l == B_s_l) B_e_l ++; + if (B_e_r == B_s_r) B_s_r --; + + // S_left to S_right defines are window at the top, choose 0.05 - 0.15, 0.85 - 0.95 + int N_top = S_right - S_left + 1; + + // if N_top < 20, these jumps might be too small + // if N_top < 20, these jumps might be too small + int S_s_l = S_left + 0.05*N_top; + int S_e_l = S_left + 0.15*N_top; + if (S_e_l == S_s_l) S_e_l ++; + int S_s_r = S_left + 0.85*N_top; + int S_e_r = S_left + 0.95*N_top; + if (S_e_r == S_s_r) S_s_r --; + + gprint (GP_ERR, "windows: %.1f - %.1f, %.1f - %.1f, %.1f - %.1f, %.1f - %.1f\n", + B_s_l + Xref - Xprofile, B_e_l + Xref - Xprofile, + S_s_l + Xref - Xprofile, S_e_l + Xref - Xprofile, + S_s_r + Xref - Xprofile, S_e_r + Xref - Xprofile, + B_s_r + Xref - Xprofile, B_e_r + Xref - Xprofile); + + // OPEN QUESTIONS: + // NOTE: if the range of pixels in the windows is too large, then the tilt will wash out the signal. + // this pass is a guess, so perhaps that does not matter, but there is clearly a trade-off here. + + // generate two buffers to store the flux and coordinate for the background, sky, object pixels: + ALLOCATE (tmpv, float, Nx); + ALLOCATE (tmpx, float, Nx); + + float *buffV = (float *) buffer[0].matrix.buffer; + + // use these to store the flux vectors + ALLOCATE_PTR (bckF_lf, float, Ny); + ALLOCATE_PTR (bckF_rt, float, Ny); + ALLOCATE_PTR (skyF_lf, float, Ny); + ALLOCATE_PTR (skyF_rt, float, Ny); + + // use these to store the x-coord vectors + ALLOCATE_PTR (bckX_lf, float, Ny); + ALLOCATE_PTR (bckX_rt, float, Ny); + ALLOCATE_PTR (skyX_lf, float, Ny); + ALLOCATE_PTR (skyX_rt, float, Ny); + + float dxMax = 0; // maximum distance between sky regions + + Vector *skyLsV = NULL, *skyLeV = NULL, *skyLfV = NULL, *skyRsV = NULL, *skyReV = NULL, *skyRfV = NULL; + Vector *bckLsV = NULL, *bckLeV = NULL, *bckLfV = NULL, *bckRsV = NULL, *bckReV = NULL, *bckRfV = NULL; + if (SaveSegments) { + skyLsV = SelectVector ("skyLs", ANYVECTOR, TRUE); ResetVector (skyLsV, OPIHI_FLT, Ny); + skyLeV = SelectVector ("skyLe", ANYVECTOR, TRUE); ResetVector (skyLeV, OPIHI_FLT, Ny); + skyLfV = SelectVector ("skyLf", ANYVECTOR, TRUE); ResetVector (skyLfV, OPIHI_FLT, Ny); + skyRsV = SelectVector ("skyRs", ANYVECTOR, TRUE); ResetVector (skyRsV, OPIHI_FLT, Ny); + skyReV = SelectVector ("skyRe", ANYVECTOR, TRUE); ResetVector (skyReV, OPIHI_FLT, Ny); + skyRfV = SelectVector ("skyRf", ANYVECTOR, TRUE); ResetVector (skyRfV, OPIHI_FLT, Ny); + bckLsV = SelectVector ("bckLs", ANYVECTOR, TRUE); ResetVector (bckLsV, OPIHI_FLT, Ny); + bckLeV = SelectVector ("bckLe", ANYVECTOR, TRUE); ResetVector (bckLeV, OPIHI_FLT, Ny); + bckLfV = SelectVector ("bckLf", ANYVECTOR, TRUE); ResetVector (bckLfV, OPIHI_FLT, Ny); + bckRsV = SelectVector ("bckRs", ANYVECTOR, TRUE); ResetVector (bckRsV, OPIHI_FLT, Ny); + bckReV = SelectVector ("bckRe", ANYVECTOR, TRUE); ResetVector (bckReV, OPIHI_FLT, Ny); + bckRfV = SelectVector ("bckRf", ANYVECTOR, TRUE); ResetVector (bckRfV, OPIHI_FLT, Ny); + } + + // loop over the y positions + // examine the slit window profile to measure background and slit sky + for (int iy = 0; iy < Ny; iy++) { + + // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + float dXtrace = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + + // Xref is the reference coordinate of the profile on the image + // Xprofile is the center of the profile. + // dXtrace follows the x-dir variations of the slit profile + float Xtrace = Xref - Xprofile + dXtrace; + + bckF_lf[iy] = get_segment_median(&buffV[iy*Nx], B_s_l + Xtrace, B_e_l + Xtrace, &bckX_lf[iy]); + bckF_rt[iy] = get_segment_median(&buffV[iy*Nx], B_s_r + Xtrace, B_e_r + Xtrace, &bckX_rt[iy]); + skyF_lf[iy] = get_segment_median(&buffV[iy*Nx], S_s_l + Xtrace, S_e_l + Xtrace, &skyX_lf[iy]); + skyF_rt[iy] = get_segment_median(&buffV[iy*Nx], S_s_r + Xtrace, S_e_r + Xtrace, &skyX_rt[iy]); + dxMax = MAX (dxMax, skyX_rt[iy] - skyX_lf[iy]); // this is a constant value unless we ignore some of the pixels + // assert skyX_rt > skyX_lf ? + + if (SaveSegments) { + skyLsV->elements.Flt[iy] = S_s_l + Xtrace; + skyLeV->elements.Flt[iy] = S_e_l + Xtrace; + skyLfV->elements.Flt[iy] = skyF_lf[iy]; + skyRsV->elements.Flt[iy] = S_s_r + Xtrace; + skyReV->elements.Flt[iy] = S_e_r + Xtrace; + skyRfV->elements.Flt[iy] = skyF_rt[iy]; + bckLsV->elements.Flt[iy] = B_s_l + Xtrace; + bckLeV->elements.Flt[iy] = B_e_l + Xtrace; + bckLfV->elements.Flt[iy] = bckF_lf[iy]; + bckRsV->elements.Flt[iy] = B_s_r + Xtrace; + bckReV->elements.Flt[iy] = B_e_r + Xtrace; + bckRfV->elements.Flt[iy] = bckF_rt[iy]; + } + } + + // extract sky-subtracted values for the psf fit, using the two sky vectors to predict the sky + float dyMax = dxMax*sin(stilt*RAD_DEG); // dxMax is based on the profile width, calculated above + + // find nominal dy_lf, dy_rt offsets for the sky vectors. + int dy_lf = -dyMax / 2; + int dy_rt = +dyMax / 2; + + // we are fitting this PSF to the data: + opihi_flt *psfV = psf->elements.Flt; + + // init the regions which are outside the valid region + for (int iy = 0; iy < fabs(dyMax); iy++) { + object->elements.Flt[iy] = 0; + sky->elements.Flt[iy] = 0; + backgnd->elements.Flt[iy] = 0; + } + for (int iy = Ny - fabs(dyMax); iy < Ny; iy++) { + object->elements.Flt[iy] = 0; + sky->elements.Flt[iy] = 0; + backgnd->elements.Flt[iy] = 0; + } + + + // Ys, Ye based on the dyMax value above (skip the first few lines to avoid running out of bounds on sky) + for (int iy = fabs(dyMax); iy < Ny - fabs(dyMax); iy++) { + + // X_lf & X_rt are calculated in image x-coords + float S_lf = skyF_lf[iy + dy_lf]; + float S_rt = skyF_rt[iy + dy_rt]; + float X_lf = skyX_lf[iy + dy_lf]; + float X_rt = skyX_rt[iy + dy_rt]; + + // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + float dXtraceSLIT = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + float dXtracePSF = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); + + // Xref is the reference coordinate of the profile on the image + // Xpsf is the center of the PSF. + // dXtrace follows the x-dir variations of the slit profile (or PSF?) + float XtracePSF = Xref - Xpsf + dXtraceSLIT + dXtracePSF; + float XtraceSLIT = Xref - Xprofile + dXtraceSLIT; + + // include the PSF trace and restrict the fit below to portions where the profile is > 0.95 + + // now I need to find the limits of the PSF in the buffer at this iy location + int Ntmp = 0; + for (int ipsf = 0; ipsf < Npsf; ipsf ++) { + + int ix = ipsf + XtracePSF + 0.5; // image coordinate (nearest pixel center) + int islit = ix - XtraceSLIT - 0.5; // image coordinate (nearest pixel center) + + float So = S_lf + (ix - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak + + if (islit < 0) continue; + if (islit >= profile->Nelements) continue; + + if (iy == 6000) { + fprintf (stderr, "%d : %f %f : %f : %f\n", ipsf, buffV[ix + iy*Nx] - So, psfV[ipsf], (buffV[ix + iy*Nx] - So)*psfV[ipsf], profile->elements.Flt[islit]); + } + + if (profile->elements.Flt[islit] < 0.95) continue; + + tmpv[Ntmp] = buffV[ix + iy*Nx] - So; + tmpx[Ntmp] = psfV[ipsf]; + Ntmp ++; + } + + // I now have a pair of vectors (tmpx, tmpv) which are aligned to the PSF at this location + // now fit the PSF to tmpv by calculating a normalization + float S1 = 0.0, S2 = 0.0; + for (int ipsf = 0; ipsf < Ntmp; ipsf++) { + if (iy == 6000) { + fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], tmpx[ipsf], tmpv[ipsf]*tmpx[ipsf]); + } + S1 += tmpv[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + S2 += tmpx[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + } + + int Xo = Xref - dXtraceSLIT - dXtracePSF; // center of PSF in image x-coordinates + float So = S_lf + (Xo - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak + object->elements.Flt[iy] = S1 / S2; + sky->elements.Flt[iy] = So; + backgnd->elements.Flt[iy] = 0.5*(bckF_lf[iy] + bckF_rt[iy]); + } + + free (tmpv); + free (tmpx); + + free (bckF_lf); + free (bckF_rt); + free (skyF_lf); + free (skyF_rt); + + free (bckX_lf); + free (bckX_rt); + free (skyX_lf); + free (skyX_rt); + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos getobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + // can I make default values for trace (0.0), profile (?), psf (fraction of profile?), stilt (0.0) + return FALSE; +} + + /* + // convolve the object flux at this y-coord with PSF and add sky + for (int ix = 0; i < Nx; i++) { + opihi_flt g = s = 0; + for (int n = -Npsf; n <= Npsf; n++) { + if (ix + n < 0) continue; + if (ix + n >= Nx) continue; + opihi_flt value = objV ? objV[ix+n] : 1.0; + s += psfV[n]*value; + g += psfV[n]; + } + out[ix + iy*Nx] = Npsf ? s / g : (objV ? objV[ix+n] : 1.0); + if (skyV) { + out[ix + iy*Nx] += skyV[iy] + } + */ + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c (revision 41170) @@ -1,4 +1,6 @@ # include "data.h" # include "deimos.h" + +XXX: deprecated // Use this to test the deimos_make_object function used by deimos_fitobj @@ -92,5 +94,5 @@ // generate the model based on the test values - float *model = deimos_make_model (objV, skyV, bckV, psf, profile, trace, Nx, Ny, 0); + float *model = deimos_make_model (objV, skyV, bckV, psf, profile, trace, NULL, Nx, Ny, 0); ResetBuffer (output, Nx, Ny, -32, 0.0, 1.0); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c (revision 41170) @@ -86,11 +86,12 @@ // generate a subimage of the full model for the range row - row + Ny // obj,sky,bck are subset vectors corresponding to the range row to row+Ny -float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *trace, int Nx, int Ny, int row) { - - float *raw = deimos_make_straight_image (obj, sky, psf, Nx, Ny, row); +// the slit_trace_* and psf_trace refer to the full image starting at row +float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, Spline *psf_trace, int Nx, int Ny, int row) { + + float *raw = deimos_make_straight_image (obj, sky, psf, psf_trace, Nx, Ny, row); float *new = deimos_apply_tilt (raw, Nx, Ny); deimos_apply_profile (profile, new, Nx, Ny); deimos_add_background (bck, new, Nx, Ny, row); - float *out = deimos_apply_trace (trace, new, Nx, Ny, row); + float *out = deimos_apply_trace (slit_trace_red, slit_trace_blu, redlimit, new, Nx, Ny, row); free (raw); @@ -104,5 +105,5 @@ // *** generate a raw image with no slit tilt and no trace offsets. apply the PSF to the // object flux, add in the sky flux -float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, int Nx, int Ny, int row) { +float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, Spline *psf_trace, int Nx, int Ny, int row) { OHANA_UNUSED_PARAM(row); @@ -113,5 +114,5 @@ // Xref is a global supplied by the user (defaults to Nx/2 if < 0) - int Xoff = (int)(Xref - Npsf); + int XoffRef = (int)(Xref - Npsf); // Nx : width of output window @@ -123,7 +124,26 @@ // we are generating the image for just the row range row to row+Ny + // note that obj & sky are subset vectors for just this range of pixels opihi_flt objVy = obj[iy]; // if we shift to using the row offset: obj[iy + row] opihi_flt skyVy = sky[iy]; // if we shift to using the row offset: sky[iy + row] + // integral and fractional pixel offsets of PSF (none, by default) + int dxi = 0; + float dxf = 0.0, dxr = 1.0; + int Xoff = XoffRef; + + if (psf_trace) { + // the psf_trace is referenced against the full image, so we need to add row to iy: + float dx = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy + row); + dxi = floor(dx); + dxf = dx - dxi; // -1.7 -> +0.3, -0.5 -> +0.5, +0.5 ->+0.5, +1.7 -> +0.7 + dxr = 1 - dxf; + Xoff += dxi; + // fprintf (stderr, "%d: %f : %d\n", iy + row, dx, Xoff); + } + + // if fractional offset is small, do not interpolate + int doInterp = fabs(dxf) < 1e-5 ? FALSE : TRUE; + // flux = obj * PSF + sky for (int ix = 0; ix < Nx; ix++) { @@ -136,5 +156,20 @@ // only add in the flux if we are in range of the PSF if ((n >= 0) && (n < NpsfFull)) { - value += objVy * psfV[n]; + if (doInterp) { + if ((n > 0) && (n < NpsfFull)) { + if (isfinite(psfV[n]) && isfinite(psfV[n-1])) { + value += objVy*psfV[n]*dxr + objVy*psfV[n-1]*dxf; + } + } + if (n == 0) { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]*dxr; + } + } + } else { + if (isfinite(psfV[n])) { + value += objVy * psfV[n]; + } + } } out[ix + iy*Nx] = value; @@ -219,5 +254,5 @@ // shift pixels in x based on the trace -float *deimos_apply_trace (Spline *trace, float *input, int Nx, int Ny, int row) { +float *deimos_apply_trace (Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, float *input, int Nx, int Ny, int row) { ALLOCATE_PTR (output, float, Nx*Ny); @@ -227,5 +262,7 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center // NOTE: spline is evaluated at full dispersion coordinate - float dx = -spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy + row); + int iy_full = iy + row; + Spline *slit_trace = (iy_full < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = -spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy_full); // extract the integer pixel offset and the fractional offset Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c (revision 41170) @@ -6,33 +6,60 @@ // input parameters: - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * object : vector of object flux vs y-coord - // * sky : vector of local sky signal vs y-coord - // * background : vector of extra-slit background flux vs y-coord - // * PSF : point-spread function vector (flux normalized, x-dir) - // * LSF : sline-spread function vector (flux normalized, y-dir) - // * stilt : slit tilt response : 2D kernel? + // * slit_trace_red : spline fit of slit central x pos vs y-coord for red chip + // * slit_trace_blu : spline fit of slit central x pos vs y-coord for blu chip + // * psf_trace : spline fit of psf central x pos vs y-coord + // * profile : slit window profile (vector) + // * object : vector of object flux vs y-coord + // * sky : vector of local sky signal vs y-coord + // * backgnd : vector of extra-slit background flux vs y-coord + // * PSF : point-spread function vector (flux normalized, x-dir) + // * LSF : sline-spread function vector (flux normalized, y-dir) + // * stilt : slit tilt response : 2D kernel? + + // * redlimit : pixel break between red and blu chips + // * Nwave : number of wavelength pixels in output image int N; // if any of these are not defined, they will have assumed identity values - Spline *trace = NULL; - Vector *profile = NULL; - Vector *object = NULL; - Vector *sky = NULL; - Vector *backgnd = NULL; - - Vector *psf = NULL; - Vector *lsf = NULL; + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; + Vector *profile = NULL; + Vector *object = NULL; + Vector *sky = NULL; + Vector *backgnd = NULL; + + Vector *psf = NULL; + Vector *lsf = NULL; // add tilt later - Buffer *output = NULL; + Buffer *output = NULL; float stilt = 0.0; // angle of the slit - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } @@ -81,6 +108,24 @@ } + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + if (argc != 3) { - gprint (GP_ERR, "USAGE: deimos mkobj (buffer) (Ncross) [-Nwave N] [-object vector] [-sky vector] [-backgnd vector] [-trace spline] [-profile vector]\n"); + gprint (GP_ERR, "USAGE: deimos mkobj (buffer) (Ncross) [-object vector] [-sky vector] [-backgnd vector] [-slit-trace-red spline] [-slit-trace-blu spline] [-psf-trace spline] [-profile vector] [-psf vector] [-lsf vector] [-stilt angle] [-Nwave Npixel] [-redlimit pixel]\n"); + gprint (GP_ERR, " slit_trace_red : spline fit of slit central x pos vs y-coord for red chip\n"); + gprint (GP_ERR, " slit_trace_blu : spline fit of slit central x pos vs y-coord for blu chip\n"); + gprint (GP_ERR, " psf_trace : spline fit of psf central x pos vs y-coord\n"); + gprint (GP_ERR, " profile : slit window profile (vector)\n"); + gprint (GP_ERR, " object : vector of object flux vs y-coord \n"); + gprint (GP_ERR, " sky : vector of local sky signal vs y-coord\n"); + gprint (GP_ERR, " backgnd : vector of extra-slit background flux vs y-coord\n"); + gprint (GP_ERR, " PSF : point-spread function vector (flux normalized, x-dir)\n"); + gprint (GP_ERR, " LSF : sline-spread function vector (flux normalized, y-dir)\n"); + gprint (GP_ERR, " stilt : slit tilt response : 2D kernel? \n"); + gprint (GP_ERR, " redlimit : pixel break between red and blu chips\n"); + gprint (GP_ERR, " Nwave : number of wavelength pixels in output image\n"); return FALSE; } @@ -123,10 +168,11 @@ // if we are appying a trace offset spline, we need to generate an output window which // is Nx + the full swing of the trace, then window back down - if (trace) { + if (slit_trace_red && slit_trace_blu) { float dXmin = +1000; float dXmax = -1000; for (int iy = 0; iy < Ny; iy++) { // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dx = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); dXmin = MIN (dx, dXmin); dXmax = MAX (dx, dXmax); @@ -181,4 +227,21 @@ if (psf) Npof -= Npsf; + // integral and fractional pixel offsets of PSF (none, by default) + int dxi = 0; + float dxf = 0.0, dxr = 1.0; + + if (psf_trace) { + float dx = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); + dxi = floor(dx); + dxf = dx - dxi; // -1.7 -> +0.3, -0.5 -> +0.5, +0.5 ->+0.5, +1.7 -> +0.7 + dxr = 1 - dxf; + Npof += dxi; + // fprintf (stderr, "%d: %f : %d\n", iy, dx, Npof); + if (VERBOSE && (iy % 200 == 0)) { gprint (GP_ERR, "%d : %f : %d : %f %f\n", iy, dx, dxi, dxf, dxr); } + } + + // if fractional offset is small, do not interpolate + int doInterp = fabs(dxf) < 1e-5 ? FALSE : TRUE; + // flux = obj * PSF + sky for (int ix = 0; ix < Nx; ix++) { @@ -191,5 +254,20 @@ // only add in the flux if we are in range of the PSF if ((n >= 0) && (n < NpsfFull)) { - value += objVy * psfV[n]; + if (doInterp) { + if ((n > 0) && (n < NpsfFull)) { + if (isfinite(psfV[n]) && isfinite(psfV[n-1])) { + value += objVy*psfV[n]*dxr + objVy*psfV[n-1]*dxf; + } + } + if (n == 0) { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]*dxr; + } + } + } else { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]; + } + } } } else { @@ -329,5 +407,5 @@ // shift pixels in x based on the trace - if (trace) { + if (slit_trace_red && slit_trace_blu) { ALLOCATE_PTR (outTraceBuffer, char, NxBase*Ny*sizeof(float)); float *outTrace = (float *) outTraceBuffer; @@ -337,5 +415,6 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dx = -spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = -spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); // extract the integer pixel offset and the fractional offset Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c (revision 41170) @@ -7,14 +7,21 @@ // input parameters: // slit profile response : vector of fractional flux vs x-coord - // trace : spline fit of slit central x pos vs y-coord - // flux : vector of input signal vs y-coord + // trace_ref,trace_blu : spline fit of slit central x pos vs y-coord + // flux (optional) : vector of input signal vs y-coord + // sky (optional) : vector of input signal vs y-coord + // fluxbins (optional) : size of output image in y-direction + // NOTE: one of flux, sky, or fluxbins must be defined + + // output : buff (an image with Nx defined by profile and Ny defined + // by flux, sky, or fluxbins) int N; - Vector *profile = NULL; - Vector *flux = NULL; - Vector *sky = NULL; - Spline *trace = NULL; - Buffer *buff = NULL; + Vector *profile = NULL; + Vector *flux = NULL; + Vector *sky = NULL; + Spline *trace_red = NULL; + Spline *trace_blu = NULL; + Buffer *buff = NULL; // user-specified flux value (otherwise assumed to be 1) @@ -40,6 +47,24 @@ } - if (argc != 4) { - gprint (GP_ERR, "USAGE: deimos mkslit (profile) (trace) (buffer) [-flux vector] [-fluxbins N] [-sky vector]\n"); + // either flux or fluxbins must be specified to define output size + int profilebins = 0; + if ((N = get_argument (argc, argv, "-profilebins"))) { + remove_argument (N, &argc, argv); + profilebins = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: deimos mkslit (profile) (trace_red) (trace_blu) (buffer) [-flux vector] [-fluxbins N] [-sky vector]\n"); return FALSE; } @@ -51,10 +76,12 @@ // XXX I probably should rename FindSpline as SelectSpline and give it the same behavior - if ((profile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); - if ((trace = FindSpline (argv[2])) == NULL) return (FALSE); - if ((buff = SelectBuffer (argv[3], ANYBUFFER, TRUE)) == NULL) return (FALSE); + if ((profile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((trace_red = FindSpline (argv[2])) == NULL) return (FALSE); + if ((trace_blu = FindSpline (argv[3])) == NULL) return (FALSE); + if ((buff = SelectBuffer (argv[4], ANYBUFFER, TRUE)) == NULL) return (FALSE); // define the output window - int Nx = 2*profile[0].Nelements + 1; + // if no profile half-width is specified, use supplied profile + int Nx = profilebins ? 2*profilebins + 1 : 2*profile[0].Nelements + 1; int Ny = flux ? flux[0].Nelements : fluxbins; @@ -73,4 +100,5 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + Spline *trace = (iy < redlimit) ? trace_red : trace_blu; float dx = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); @@ -90,5 +118,5 @@ // set the sky in the entire window - out[ix + iy*Nx] = sky ? sky->elements.Flt[iy] : 0.0; + out[ix + iy*Nx] = NAN; // equivalent coord in the profile: @@ -98,4 +126,5 @@ // a default value: + float vsky = sky ? sky->elements.Flt[iy] : 0.0; float vout = NAN; @@ -121,5 +150,5 @@ if (flux) vout *= flux->elements.Flt[iy]; - out[ix + iy*Nx] += vout; + out[ix + iy*Nx] = vsky + vout; } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh (revision 41170) @@ -0,0 +1,91 @@ + +macro test_01 + if ($0 != 2) + echo "USAGE: test_01 (offset)" + break + end + + # make a gaussian profile + create x 0 31 + set yo = 105.1*exp(-0.5*(x - 12.2)^2/2.2^2) + gaussdev dg yo[] 0 1 + # set dg = zero(yo) + set y = yo + dg*sqrt(yo) + + # fitarc version + dimenup y im 1 31 + deimos fitarc im 0 0 1 31 -resid imf -max-iterations 0 + dimendown imf yf1 + + echo $C0 $C2 $C1 $C3 + + $C0 = $C0 + $1 + vgauss -apply x yf2 + + # dy1 : model from fitarc + # dy2 : rebuilt model + # NOTE: vgauss and fitarc output results match exactly + # asymmetry is a result of the fitted values + + set dy1 = y - yf1 + set dy2 = y - yf2 + + lim -n 1 x dy2; clear; box; + + plot x dy1 -c red -x hist + plot x dy2 -c blue -x hist + + lim -n 2 x y; clear; box; + + plot x y -c black -x hist -lw 4 + + plot x yf1 -c red -x hist -lw 3 + plot x yf2 -c blue -x hist -lw 2 +end + +macro test_02 + if ($0 != 2) + echo "USAGE: test_02 (offset)" + break + end + + # make a gaussian profile + create x 0 31 + set yo = 105.1*exp(-0.5*(x - 12.2)^2/2.2^2) + gaussdev dg yo[] 0 1 + set dg = zero(yo) + set y = yo + dg*sqrt(yo) + + # vgauss version + $C0 = 15 + $C1 = 2 + $C2 = 100 + $C3 = 0 + vgauss x y con yf1 -q + echo $C0 $C2 $C1 $C3 + $C0 = $C0 + $1 + vgauss -apply x yf3 + + set dy1 = y - yf1 + set dy3 = y - yf3 + + # fitarc version + dimenup y im 1 31 + deimos fitarc im 0 0 1 31 -resid dim -max-iterations 0 + dimendown dim yf2 + set dy2 = y - yf2 + + lim -n 1 x dy2; clear; box; + + plot x dy1 -c red -x hist + plot x dy2 -c blue -x hist + plot x dy3 -c black -x hist + + lim -n 2 x y; clear; box; + + plot x y -c black -x hist -lw 3 + + plot x yf2 -c blue -x hist -lw 3 + plot x yf1 -c red -x hist -lw 2 + plot x yf3 -c blue60 -x hist +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh (revision 41170) @@ -0,0 +1,28 @@ + +macro memtest1 + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set fprofile = profile_left + profile_mid + profile_right + set wprofile = 0.02*fprofile + + $i = 0 + deimos fitprofile -q xprof fprofile wprofile $Xpmin $Xpmax + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos fitprofile -q xprof fprofile wprofile $Xpmin $Xpmax + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh (revision 41170) @@ -0,0 +1,31 @@ + +macro memtest1 + # make a delta-function profile + create x 0 11 + set yg = exp(-0.25*(x - 5)^2) + + # flux vs wavelength is flat & short + create w 0 100 + set f = zero(w) + 1 + + delete -q Xn Yn + vlist Xn 0 100 + vlist Yn 0 2 + + spline create trace Xn Yn + spline apply trace w dX + $i = 0 + + # run once to ensure the output is create + deimos mkslit yg trace trace outbuff -flux f + mgaussdev noise outbuff[][0] outbuff[0][] 0.0 0.2 + + set obs = 2*outbuff + noise + deimos fitslit obs outbuff outf outs + + memory check + for i 0 1000 + deimos mkslit yg trace trace outbuff -flux f + end + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh (revision 41170) @@ -0,0 +1,55 @@ + +macro memtest1 + # generate LSF: + create x 0 7 + set lsf = exp(-0.5*(x-3)^2) + + # generate PSF: + create xp 0 9 + set psf = exp(-0.5*(xp-5)^2/1.5^2) + + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set profile = profile_left + profile_mid + profile_right + + # generate object flux: + create wave 0 100 + set flux = 100 + 0.5*wave + + # generate sky flux: + set sky = 200 + zero(wave) + + # off-slit background: + set backgnd = 300 + zero(wave) + + # slit trace + vlist Xn 0 100 + vlist Yn 0 4 + spline create slit_trace_in Xn Yn + + # psf trace + vlist Xn 0 25 50 75 100 + vlist Yn 0 2 4 2 0 + spline create psf_trace_in Xn Yn + + $i = 0 + deimos mkalt out 31 -psf psf -object flux -sky sky -stilt 20.0 -profile profile -backgnd backgnd -trace slit_trace_in + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos mkalt out 31 -psf psf -object flux -sky sky -stilt 20.0 -profile profile -backgnd backgnd -trace slit_trace_in + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh (revision 41170) @@ -0,0 +1,55 @@ + +macro memtest1 + # generate LSF: + create x 0 7 + set lsf = exp(-0.5*(x-3)^2) + + # generate PSF: + create xp 0 9 + set psf = exp(-0.5*(xp-5)^2/1.5^2) + + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set profile = profile_left + profile_mid + profile_right + + # generate object flux: + create wave 0 100 + set flux = 100 + 0.5*wave + + # generate sky flux: + set sky = 200 + zero(wave) + + # off-slit background: + set backgnd = 300 + zero(wave) + + # slit trace + vlist Xn 0 100 + vlist Yn 0 4 + spline create slit_trace_in Xn Yn + + # psf trace + vlist Xn 0 25 50 75 100 + vlist Yn 0 2 4 2 0 + spline create psf_trace_in Xn Yn + + $i = 0 + deimos mkobj out 31 -psf psf -object flux -sky sky -stilt 20.0 -lsf lsf -profile profile -backgnd backgnd -slit-trace slit_trace_in -psf-trace psf_trace_in + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos mkobj out 31 -psf psf -object flux -sky sky -stilt 20.0 -lsf lsf -profile profile -backgnd backgnd -slit-trace slit_trace_in -psf-trace psf_trace_in + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh (revision 41170) @@ -0,0 +1,28 @@ + + +macro memtest1 + # make a delta-function profile + create x 0 11 + set yg = exp(-0.25*(x - 5)^2) + + # flux vs wavelength is flat & short + create w 0 100 + set f = zero(w) + 1 + + delete -q Xn Yn + vlist Xn 0 100 + vlist Yn 0 2 + + spline create trace Xn Yn + spline apply trace w dX + $i = 0 + + # run once to ensure the output is create + deimos mkslit yg trace trace outbuff -flux f + + memory check + for i 0 1000 + deimos mkslit yg trace trace outbuff -flux f + end + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c (revision 41170) @@ -62,4 +62,8 @@ catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } catalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog Index: branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c (revision 41144) +++ branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c (revision 41170) @@ -73,4 +73,8 @@ catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -162,5 +166,11 @@ // we can optionally convert output format here // but it would be better to define a dvo crawler program to do this - // catalog.catformat = DVO_FORMAT_PS1_V1; + // catalog.catformat = DVO_FORMAT_PS1_V1; + + // modify the output format as desired (ignore current format on disk) + if (UPDATE_CATFORMAT) { + catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } + SetProtect (TRUE); if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }