Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/Makefile (revision 41170) @@ -27,4 +27,5 @@ loadgaia : $(BIN)/loadgaia.$(ARCH) loadgaia_dr2 : $(BIN)/loadgaia_dr2.$(ARCH) +loadukirt_uhs : $(BIN)/loadukirt_uhs.$(ARCH) loadstarpar : $(BIN)/loadstarpar.$(ARCH) loadstarpar_client : $(BIN)/loadstarpar_client.$(ARCH) @@ -44,5 +45,5 @@ # programs in 'SERVER' use the client-server concept and are out of date -INSTALL = addstar addstar_client sedstar loadgalphot loadgaia loadgaia_dr2 loadstarpar loadstarpar_client setobjflags setobjflags_client loadICRF loadICRF_client skycells mkcmf dumpskycells findskycell load2mass loadwise loadtycho loadbsc loadsupercos +INSTALL = addstar addstar_client sedstar loadgalphot loadgaia loadgaia_dr2 loadukirt_uhs loadstarpar loadstarpar_client setobjflags setobjflags_client loadICRF loadICRF_client skycells mkcmf dumpskycells findskycell load2mass loadwise loadtycho loadbsc loadsupercos SERVER = addstarc addstard addstart @@ -305,4 +306,26 @@ $(SRC)/psps_ids.$(ARCH).o +LOAD-UKIRT_UHS = \ +$(SRC)/loadukirt_uhs.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/args_loadukirt_uhs.$(ARCH).o \ +$(SRC)/find_matches_ukirt_uhs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_catalog.$(ARCH).o \ +$(SRC)/loadukirt_uhs_make_subset.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_uhs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ugcs.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ugps.$(ARCH).o \ +$(SRC)/loadukirt_uhs_readstars_ulas.$(ARCH).o \ +$(SRC)/loadukirt_uhs_table.$(ARCH).o \ +$(SRC)/parse_csv.$(ARCH).o \ +$(SRC)/resort_catalog.$(ARCH).o \ +$(SRC)/build_links.$(ARCH).o \ +$(SRC)/strhash.$(ARCH).o \ +$(SRC)/sortIDs.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o + LOAD-STARPAR = \ $(SRC)/loadstarpar.$(ARCH).o \ @@ -528,4 +551,5 @@ $(LOAD-GAIA) : $(INC)/addstar.h $(INC)/gaia.h $(LOAD-GAIA_DR2) : $(INC)/addstar.h $(INC)/gaia_dr2.h +$(LOAD-UKIRT_UHS) : $(INC)/addstar.h $(INC)/ukirt_uhs.h $(LOAD-STARPAR) : $(INC)/addstar.h $(INC)/loadstarpar.h $(LOAD-STARPAR-CLIENT) : $(INC)/addstar.h $(INC)/loadstarpar.h @@ -551,4 +575,5 @@ $(BIN)/loadgaia.$(ARCH) : $(LOAD-GAIA) $(BIN)/loadgaia_dr2.$(ARCH) : $(LOAD-GAIA_DR2) +$(BIN)/loadukirt_uhs.$(ARCH) : $(LOAD-UKIRT_UHS) $(BIN)/loadstarpar.$(ARCH) : $(LOAD-STARPAR) $(BIN)/loadstarpar_client.$(ARCH) : $(LOAD-STARPAR-CLIENT) Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/include/ukirt_uhs.h (revision 41170) @@ -0,0 +1,65 @@ + +typedef enum {UKIRT_MODE_NONE, UKIRT_MODE_UHS, UKIRT_MODE_UGCS, UKIRT_MODE_UGPS, UKIRT_MODE_ULAS} UkirtMode; + +# define NSTARS_MAX 10000000 +// # define NSTARS_MAX 10 + +# define BUFFER_SIZE 0x100000 +// # define BUFFER_SIZE 900 + +typedef struct { + Average average; + Measure *measure; + int flag; // in a subset? + int found; // assigned to an object? +} UKIRT_Stars; + +int HOST_ID; +char *HOSTDIR; +char *CPT_FILE; +char *INPUT; +int UKIRT_NFILTER; + +UkirtMode UKIRT_MODE; + +AddstarClientOptions args_loadukirt_uhs (int *argc, char **argv, AddstarClientOptions options); +// AddstarClientOptions args_loadukirt_uhs_client (int *argc, char **argv, AddstarClientOptions options); + +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options); + +UKIRT_Stars *loadukirt_uhs_make_subset (UKIRT_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset); + +// int loadukirt_uhs_save_remote (UKIRT_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region, char *fullname, AddstarClientOptions *options); +// int save_remote_host (HostInfo *host); + +// int init_remote_hosts (void); +// void free_remote_hosts (void); +// int find_empty_slot (void); +// int harvest_all (void); +// int harvest_host (void); + +int loadukirt_uhs_catalog (UKIRT_Stars *stars, int Nstars, SkyRegion *region, char *filename, AddstarClientOptions *options); + +// int galactic_to_celestial (double *R, double *D, double l, double b); + +int find_matches_ukirt_uhs (SkyRegion *region, UKIRT_Stars *stars, int Nstars, Catalog *catalog, AddstarClientOptions *options); + +int loadukirt_uhs_save_stars (char *filename, UKIRT_Stars *stars, int Nstars); +// UKIRT_Stars *loadukirt_uhs_load_stars (char *filename, int *nstars); + +UKIRT_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); + +UKIRT_Stars *loadukirt_uhs_readstars_ugps (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_ugcs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_ulas (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); +UKIRT_Stars *loadukirt_uhs_readstars_uhs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars); + + +int loadukirt_uhs_sortStars (UKIRT_Stars *stars, int Nstars); + +// int loadukirt_uhs_tmpdir (void); + +// roll these into a function with a structure to carry the Nlast, etc? +char *dparse_csv_rpt (double *X, int Nwant, int Nlast, char *line, int *status); +char *iparse_csv_rpt (int *X, int Nwant, int Nlast, char *line, int *status); +char *jparse_csv_rpt (uint64_t *X, int Nwant, int Nlast, char *line, int *status); Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/ReadImageHeader.c (revision 41170) @@ -118,5 +118,5 @@ if (photcode == 0) { if (!gfits_scan (header, "PHOTCODE", "%s", 1, photname)) { - fprintf (stderr, "photcode not supplied in header\n"); + fprintf (stderr, "failure: photcode not supplied in header\n"); return (FALSE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/args_loadukirt_uhs.c (revision 41170) @@ -0,0 +1,165 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +static void help (void); + +AddstarClientOptions args_loadukirt_uhs (int *argc, char **argv, AddstarClientOptions options) { + + int N; + + /* check for help request */ + if (get_argument (*argc, argv, "-help") || + get_argument (*argc, argv, "-h")) { + help (); + } + + // UKIRT_MODE is global + UKIRT_MODE = UKIRT_MODE_NONE; + if ((N = get_argument (*argc, argv, "-uhs"))) { + UKIRT_MODE = UKIRT_MODE_UHS; + UKIRT_NFILTER = 1; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ugcs"))) { + UKIRT_MODE = UKIRT_MODE_UGCS; + UKIRT_NFILTER = 6; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ugps"))) { + UKIRT_MODE = UKIRT_MODE_UGPS; + UKIRT_NFILTER = 5; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-ulas"))) { + UKIRT_MODE = UKIRT_MODE_ULAS; + UKIRT_NFILTER = 5; + remove_argument (N, argc, argv); + } + if (UKIRT_MODE == UKIRT_MODE_NONE) { + fprintf (stderr, "missing or invalid ukirt mode\n"); + help (); + } + + /* basic mode: image, list, refcat */ + options.mode = ADDSTAR_MODE_REFCAT; + + /* we do not allow a subset to be extracted -- all or nothing, babe */ + UserPatch.Rmin = 0; + UserPatch.Rmax= 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + + // XXX for the moment, make this selection manual. it needs to be automatic + // based on the state of the SkyTable + HOST_ID = 0; + PARALLEL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, argc, argv); + } + + /* only add to existing regions */ + options.existing_regions = FALSE; + if ((N = get_argument (*argc, argv, "-existing-regions"))) { + options.existing_regions = TRUE; + remove_argument (N, argc, argv); + } + /* only add to existing objects */ + options.only_match = FALSE; + if ((N = get_argument (*argc, argv, "-only-match"))) { + options.only_match = TRUE; + remove_argument (N, argc, argv); + } + /* replace measurement, don't duplicate (ref/cat only) */ + options.replace = FALSE; + if ((N = get_argument (*argc, argv, "-replace"))) { + options.replace = TRUE; + remove_argument (N, argc, argv); + } + /* override any header PHOTCODE values */ + options.photcode = 0; + + /* accept proper-motion (& parallax) data from reference */ + ACCEPT_MOTION = FALSE; + if ((N = get_argument (*argc, argv, "-accept-motion"))) { + ACCEPT_MOTION = TRUE; + remove_argument (N, argc, argv); + } + + /* provide a time for dataset */ + options.timeref = 0; + if ((N = get_argument (*argc, argv, "-time"))) { + time_t tmp; + remove_argument (N, argc, argv); + if (!ohana_str_to_time (argv[N], &tmp)) { + fprintf (stderr, "syntax error in time\n"); + exit (1); + } + options.timeref = tmp; + remove_argument (N, argc, argv); + } + + /* extra error messages */ + VERBOSE = FALSE; + if ((N = get_argument (*argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, argc, argv); + } + + /* other addstar options which cannot be used in loadukirt_uhs */ + // options.timeref = 0; + options.mosaic = FALSE; + options.skip_missed = FALSE; + options.closest = FALSE; + options.nosort = FALSE; + options.update = FALSE; + options.only_images = FALSE; + options.calibrate = FALSE; + options.quality_airmass = FALSE; + ACCEPT_ASTROM = FALSE; + FORCE_READ = FALSE; + TEXTMODE = FALSE; + SUBPIX = FALSE; + DUMP = NULL; + + if (*argc != 2) { + fprintf (stderr, "USAGE: loadukirt_uhs [options] (fitsfile)\n"); + exit (2); + } + return (options); +} + +static void help () { + + fprintf (stderr, "USAGE: loadukirt_uhs [options] (file) [-uhs | -ugcs | -ulas | -ugps] [..more files]\n"); + fprintf (stderr, " add data from UKIRT CSV file to DVO\n"); + fprintf (stderr, " mode is required : -uhs | -ugcs | -ulas | -ugps\n\n"); + + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -only-match : only add measurements to existing objects\n"); + fprintf (stderr, " -replace : replace time/photcode measurements (no duplication)\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/find_matches_ukirt_uhs.c (revision 41170) @@ -0,0 +1,346 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +int find_matches_ukirt_uhs (SkyRegion *region, UKIRT_Stars *stars, int NstarsIn, Catalog *tgtcat, AddstarClientOptions *options) { + + int Nsecfilt = GetPhotcodeNsecfilt (); + int Nsec = GetPhotcodeNsec (options->photcode); + + /** allocate local arrays (stars) **/ + ALLOCATE_PTR (X1, double, NstarsIn); + ALLOCATE_PTR (Y1, double, NstarsIn); + ALLOCATE_PTR (N1, off_t, NstarsIn); + + /** allocate local arrays (tgtcat) **/ + off_t NAVE = tgtcat[0].Naverage; + off_t Nave = tgtcat[0].Naverage; + ALLOCATE_PTR (X2, double, NAVE); + ALLOCATE_PTR (Y2, double, NAVE); + ALLOCATE_PTR (N2, off_t, NAVE); + + /* for secfilt j and star i, secfilt[i*Nsecfilt+j] */ + + /* internal counters */ + off_t Nmatch = 0; + off_t NMEAS = tgtcat[0].Nmeasure; + off_t Nmeas = tgtcat[0].Nmeasure; + + off_t *next_meas = NULL; + + // current max obj ID for this tgtcat + unsigned int objID = tgtcat[0].objID; + unsigned int catID = tgtcat[0].catID; + + /* project onto rectilinear grid with 1 arcsec pixels. the choice of ARC projection has + * the advantage that every point in R,D has a mapping to a unique X,Y. However, note + * that not all possible X,Y points map back to R,D and the local plate scale changes + * substantially far from the projection pole. We use the center of the region (tgtcat) + * for crval1,2. + */ + + Coords tcoords; + InitCoords (&tcoords, "DEC--ARC"); + tcoords.crval1 = 0.5*(region[0].Rmin + region[0].Rmax); + if ((region[0].Dmax < 90) && (region[0].Dmin > -90)) { + tcoords.crval2 = 0.5*(region[0].Dmin + region[0].Dmax); + } else { + tcoords.crval2 = (region[0].Dmax >= 90) ? 90.0 : -90.0; + } + tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; + + /* build spatial index (RA sort) referencing input array sequence */ + off_t Nstars = 0; + for (off_t i = 0; i < NstarsIn; i++) { + int status = RD_to_XY (&X1[Nstars], &Y1[Nstars], stars[i].average.R, stars[i].average.D, &tcoords); + if (!status) continue; + N1[Nstars] = i; + Nstars ++; + } + if (Nstars < 1) { + if (VERBOSE) fprintf (stderr, "skipping %s, no overlapping stars\n", tgtcat[0].filename); + free (X1); + free (Y1); + free (N1); + free (X2); + free (Y2); + free (N2); + return 0; + } + if (Nstars > 1) sort_coords_index (X1, Y1, N1, Nstars); + + /* build spatial index (RA sort) */ + for (off_t i = 0; i < Nave; i++) { + RD_to_XY (&X2[i], &Y2[i], tgtcat[0].average[i].R, tgtcat[0].average[i].D, &tcoords); + N2[i] = i; + } + if (Nave > 1) sort_coords_index (X2, Y2, N2, Nave); + + /* set up pointers for linked list of measure */ + if (tgtcat[0].sorted && (tgtcat[0].Nmeasure == tgtcat[0].Nmeasure_disk)) { + // this version is only valid if we have done a full tgtcat load, and if the tgtcat + // is sorted while processed + next_meas = init_measure_links (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas); + } else { + next_meas = build_measure_links (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas); + } + + /* choose a radius for matches */ + double RADIUS = (options->radius == 0) ? 2.0 : options->radius; /* provided by config */ + double RADIUS2 = RADIUS*RADIUS; + + /****************** find matched stars ********************/ + + for (off_t i = 0, j = 0; (i < Nstars) && (j < Nave); ) { + if (!finite(X1[i]) || !finite(Y1[i])) { + i++; + continue; + } + if (!finite(X2[j]) || !finite(Y2[j])) { + j++; + continue; + } + + /* negative dX: j is too large */ + double dX = X1[i] - X2[j]; + if (dX <= -1.02*RADIUS) { + i++; + continue; + } + /* positive dX, i is too large */ + if (dX >= 1.02*RADIUS) { + j++; + continue; + } + + // skip this star if already assigned to an object + if (stars[N1[i]].found) { + i++; + continue; + } + + /* this block will match a given detection to the closest object within range of that detection. + XXX note that this matches ALL detections within range of the single object to that same object + this is bad, but I cannot just go in linear order (ie, mark off each object as they are + used). I should make a list of all Nobj * Ndet pairs in range and choose the matches + based on their separations. UGH + */ + + /* within match range; look for matches */ + off_t Jmin = -1; + double Rmin = RADIUS2; + for (off_t J = j; (dX > -1.02*RADIUS) && (J < Nave); J++) { + /* find closest match for this detection */ + dX = X1[i] - X2[J]; + double dY = Y1[i] - Y2[J]; + double dR = dX*dX + dY*dY; + if (dR > RADIUS2) continue; + if (dR < Rmin) { + Rmin = dR; + Jmin = J; + } + } + + /* no match, try next detection */ + if (Jmin == -1) { + i++; + continue; + } + + /*** a match is found, add to average, measure ***/ + Nmatch ++; + off_t n = N2[Jmin]; + off_t N = N1[i]; + + /* make sure there is space for next entry */ + if (Nmeas >= NMEAS - stars[N].average.Nmeasure) { + NMEAS = Nmeas + 1000 + stars[N].average.Nmeasure; + REALLOCATE (next_meas, off_t, NMEAS); + REALLOCATE (tgtcat[0].measure, Measure, NMEAS); + } + + /** add measurements for this star **/ + for (int mSrc = 0; mSrc < stars[N].average.Nmeasure; mSrc++) { + + /* add to end of measurement list */ + add_meas_link (&tgtcat[0].average[n], next_meas, Nmeas, NMEAS); + + /** add measurements for this star **/ + + // set the new measurements + tgtcat[0].measure[Nmeas] = stars[N].measure[mSrc]; + + // measure now carries R,D (not dR,dD) + tgtcat[0].measure[Nmeas].dbFlags = 0; + tgtcat[0].measure[Nmeas].averef = n; + tgtcat[0].measure[Nmeas].objID = tgtcat[0].average[n].objID; + tgtcat[0].measure[Nmeas].catID = tgtcat[0].catID; + + float dRoff = dvoOffsetR(&tgtcat[0].measure[Nmeas], &tgtcat[0].average[n]); + + // rationalize dR + if (dRoff > +180.0*3600.0) { + // average on high end of boundary, move star up + tgtcat[0].measure[Nmeas].R += 360.0; + dRoff -= 360.0*3600.0; + } + if (dRoff < -180.0*3600.0) { + // average on low end of boundary, move star down + tgtcat[0].measure[Nmeas].R -= 360.0; + dRoff += 360.0*3600.0; + } + if (fabs(dRoff) > 10*RADIUS) { + // take declination into account and check again. + double cosD = cos(RAD_DEG*tgtcat[0].average[n].D); + if (fabs(dRoff*cosD) > 10*RADIUS) { + fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", + tgtcat[0].average[n].R, tgtcat[0].average[n].D, + stars[N].average.R, stars[N].average.D, + X1[i], X2[Jmin], + Y1[i], Y2[Jmin]); + // XXX abort on this? -- this is a bad failure... + } + } + + /* Nm is updated, but not written out in -update mode (for existing entries) + Nm is recalculated in build_meas_links if loaded table is not sorted */ + tgtcat[0].average[n].Nmeasure ++; + Nmeas ++; + + /* if we choose to flag close encounters, see find_matches.c */ + /* if we choose to calculate RA,DEC averages, see update_coords.c */ + } + + if (ACCEPT_MOTION) { + tgtcat[0].average[n].uR = stars[N].average.uR; + tgtcat[0].average[n].uD = stars[N].average.uD; + tgtcat[0].average[n].duR = stars[N].average.duR; + tgtcat[0].average[n].duD = stars[N].average.duD; + tgtcat[0].average[n].P = stars[N].average.P; + tgtcat[0].average[n].dP = stars[N].average.dP; + tgtcat[0].average[n].Tmean = stars[N].average.Tmean; + } + + stars[N].found = TRUE; + i++; + } + + /*************** add unmatched stars *************************/ + + /** incorporate unmatched image stars, if this star is in field of this tgtcat **/ + /* these new entries are all written out in UPDATE mode */ + for (off_t i = 0; (i < Nstars) && !options->only_match; i++) { + /* make sure there is space for next entry */ + if (Nmeas >= NMEAS - stars[i].average.Nmeasure) { + NMEAS = Nmeas + 1000 + stars[i].average.Nmeasure; + REALLOCATE (next_meas, off_t, NMEAS); + REALLOCATE (tgtcat[0].measure, Measure, NMEAS); + } + if (Nave >= NAVE) { + NAVE = Nave + 1000; + REALLOCATE (tgtcat[0].average, Average, NAVE); + if (tgtcat[0].secfilt) { + // we only update the secfilt table if it has been allocated for output + REALLOCATE (tgtcat[0].secfilt, SecFilt, NAVE*tgtcat[0].Nsecfilt); + } + } + + if (stars[i].found) continue; + if (!IN_REGION (stars[i].average.R, stars[i].average.D)) continue; + + dvo_average_init (&tgtcat[0].average[Nave]); + tgtcat[0].average[Nave].R = stars[i].average.R; + tgtcat[0].average[Nave].D = stars[i].average.D; + tgtcat[0].average[Nave].dR = stars[i].average.dR; + tgtcat[0].average[Nave].dD = stars[i].average.dD; + + tgtcat[0].average[Nave].measureOffset = Nmeas; + tgtcat[0].average[Nave].objID = objID; + tgtcat[0].average[Nave].catID = catID; + + if (PSPS_ID) { + tgtcat[0].average[Nave].extID = CreatePSPSObjectID(tgtcat[0].average[Nave].R, tgtcat[0].average[Nave].D); + } + + if (ACCEPT_MOTION) { + tgtcat[0].average[Nave].uR = stars[i].average.uR; + tgtcat[0].average[Nave].uD = stars[i].average.uD; + tgtcat[0].average[Nave].duR = stars[i].average.duR; + tgtcat[0].average[Nave].duD = stars[i].average.duD; + tgtcat[0].average[Nave].P = stars[i].average.P; + tgtcat[0].average[Nave].dP = stars[i].average.dP; + tgtcat[0].average[Nave].Tmean = stars[i].average.Tmean; + } + + objID ++; + + // we only update the secfilt table if it has been allocated for output + for (int j = 0; tgtcat[0].secfilt && (j < Nsecfilt); j++) { + dvo_secfilt_init (&tgtcat[0].secfilt[Nave*Nsecfilt+j], SECFILT_RESET_ALL); + } + + // supply the measurements from this detection + for (int mSrc = 0; mSrc < stars[i].average.Nmeasure; mSrc++) { + dvo_measure_init (&tgtcat[0].measure[Nmeas]); + tgtcat[0].measure[Nmeas] = stars[i].measure[mSrc]; + + // the following measure elements cannot be set until here: + tgtcat[0].measure[Nmeas].dbFlags = 0; + tgtcat[0].measure[Nmeas].averef = Nave; + tgtcat[0].measure[Nmeas].objID = tgtcat[0].average[Nave].objID; + tgtcat[0].measure[Nmeas].catID = tgtcat[0].catID; + + /* set the average magnitude if not already set and the photcode.equiv is not 0 */ + /* in UPDATE mode, this value is not saved; use relphot to recalculate */ + if (Nsec > -1) { + tgtcat[0].secfilt[Nave*Nsecfilt+Nsec].MpsfChp = PhotCat (&tgtcat[0].measure[Nmeas], MAG_CLASS_PSF); + } + + tgtcat[0].average[Nave].Nmeasure ++; + Nmeas ++; + + // update the next_meas pointer for this entry (last one for this star is -1) + next_meas[Nmeas-1] = (mSrc < stars[i].average.Nmeasure - 1) ? Nmeas : -1; + } + stars[i].found = TRUE; + Nave ++; + } + + REALLOCATE (tgtcat[0].average, Average, Nave); + REALLOCATE (tgtcat[0].measure, Measure, Nmeas); + + if (options->nosort) { + tgtcat[0].sorted = FALSE; + } else { + tgtcat[0].sorted = TRUE; + tgtcat[0].measure = sort_measure (tgtcat[0].average, Nave, tgtcat[0].measure, Nmeas, next_meas); + } + + /* check if the tgtcat has changed? if no change, no need to write */ + tgtcat[0].objID = objID; // new max value, save on tgtcat close + tgtcat[0].Naverage = Nave; + tgtcat[0].Nmeasure = Nmeas; + tgtcat[0].Nsecfilt_mem = tgtcat[0].secfilt ? Nave*Nsecfilt : 0; + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); + + free (X1); + free (Y1); + free (N1); + free (X2); + free (Y2); + free (N2); + free (next_meas); + + return (Nmatch); +} + +/* + notes: + + for finding if a tgtcat star is in an image or an image star is in the tgtcat: + + tgtcats have boundaries defined by RA and DEC, but they may curve in projection + images have boundaries which are lines in pixels coords, but curve in RA and DEC + + tgtcat[0].found[Ncat] but stars[Nstar].found + +*/ Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs.c (revision 41170) @@ -0,0 +1,40 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* This is the DVO program to upload UKIRT_UHS detections from cvs files supplied by Mike + Read into a DVO database. It is modeled on the loadgaia_dr2 program with column modifications + determined by Zhoujian Zhang + + USAGE: loadukirt_uhs -D CATDIR (catdir) (ukirt_uhsfile) [...more files] +*/ + +int main (int argc, char **argv) { + + SkyTable *sky; + SkyList *skylist = NULL; + AddstarClientOptions options; + + // need to construct these options with args_loadtycho... + SetSignals (); + options = ConfigInit (&argc, argv); + options = args_loadukirt_uhs (&argc, argv, options); + + // load the full sky description table: + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // generate the subset matching the user-selected region + skylist = SkyListByPatch (sky, -1, &UserPatch); + + loadukirt_uhs_table (skylist, argv[1], &options); + + SkyTableFree (sky); + SkyListFree(skylist); + FreePhotcodeTable(); + free (CATDIR); + + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); + exit (0); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_catalog.c (revision 41170) @@ -0,0 +1,32 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +int loadukirt_uhs_catalog (UKIRT_Stars *stars, int Nstars, SkyRegion *region, char *filename, AddstarClientOptions *options) { + + Catalog catalog; + + // now we have all of the loaded stars in this catalog + dvo_catalog_init (&catalog, TRUE); + catalog.filename = filename; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, region, VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename); + exit (2); + } + + find_matches_ukirt_uhs (region, stars, Nstars, &catalog, options); + + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + + dvo_catalog_free (&catalog); + + return (TRUE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_make_subset.c (revision 41170) @@ -0,0 +1,58 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +// assign stars in the given region to the subset (NOTE: stars are sorted by RA, start is +// first entry in stars array in this region) + +UKIRT_Stars *loadukirt_uhs_make_subset (UKIRT_Stars *stars, int Nstars, int start, SkyRegion *region, int *nsubset) { + + int i; + + // collect array of (Stars *) stars in a new output catalog + int Nsubset = 0; + int NSUBSET = 3000; + + ALLOCATE_PTR (subset, UKIRT_Stars, NSUBSET); + // for (int i = 0; i < NSUBSET; i++) { + // ALLOCATE (subset[i].measure, Measure, UKIRT_NFILTER); + // } + + // find the rest of the stars in this output region + for (i = start; i < Nstars; i++) { + if (stars[i].flag) continue; + + // check if in skyregion + if (stars[i].average.R < region[0].Rmin) continue; + if (stars[i].average.R >= region[0].Rmax) break; + if (stars[i].average.D < region[0].Dmin) continue; + if (stars[i].average.D >= region[0].Dmax) continue; + + // check if in UserPatch (a GLOBAL) + if (stars[i].average.R < UserPatch.Rmin) continue; + if (stars[i].average.R > UserPatch.Rmax) break; + if (stars[i].average.D < UserPatch.Dmin) continue; + if (stars[i].average.D > UserPatch.Dmax) continue; + + // the *measure value is a copy of the version allocated to stars[i] + // do NOT free subset[i].measure + subset[Nsubset] = stars[i]; + Nsubset ++; + + stars[i].flag = TRUE; + + CHECK_REALLOCATE (subset, UKIRT_Stars, NSUBSET, Nsubset, 10000); + + /* + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + for (int j = Nstars; j < NSTARS; j++) { + ALLOCATE (stars[j].measure, Measure, NFILTER); + } + } + */ + } + + *nsubset = Nsubset; + return subset; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars.c (revision 41170) @@ -0,0 +1,46 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +UKIRT_Stars *loadukirt_uhs_readstars (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + switch (UKIRT_MODE) { + case UKIRT_MODE_UHS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_uhs (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_UGCS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ugcs (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_UGPS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ugps (f, buffer, nstart, options, nstars); + return stars; + } + + case UKIRT_MODE_ULAS: { + UKIRT_Stars *stars = loadukirt_uhs_readstars_ulas (f, buffer, nstart, options, nstars); + return stars; + } + + default: + fprintf (stderr, "programming error: invalid mode\n"); + exit (1); + } + return NULL; +} + +int loadukirt_uhs_sortStars (UKIRT_Stars *stars, int Nstars) { + +# define SWAPFUNC(A,B){ UKIRT_Stars temp = stars[A]; stars[A] = stars[B]; stars[B] = temp; } +# define COMPARE(A,B)(stars[A].average.R < stars[B].average.R) + + OHANA_SORT (Nstars, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + + return TRUE; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugcs.c (revision 41170) @@ -0,0 +1,802 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + + 6 sigRa : + 7 sigDec : + 8 epoch : t ???? -- how is there a single epoch? + 9 muRa : + 10 muDec : + 11 sigMuRa : + 12 sigMuDec : + 13 chi2 : + 14 nFrames : + + 15 cx : + 16 cy : + 17 cz : + 18 htmID : + 19 l : + 20 b : + 21 lambda : + 22 eta : + 23 priOrSec : primary bit in dbFlags + 24 zmyPnt : + 25 zmyPntErr : + 26 ymjPnt : + 27 ymjPntErr : + 28 jmhPnt : + 29 jmhPntErr : + 30 hmk_1Pnt : + 31 hmk_1PntErr : + 32 zmyExt : + 33 zmyExtErr : + 34 ymjExt : + 35 ymjExtErr : + 36 jmhExt : + 37 jmhExtErr : + 38 hmk_1Ext : + 39 hmk_1ExtErr : + + 40 mergedClassStat : + 41 mergedClass : psfQFperf or photFlags2 + 42 pStar : psfChisq + 43 pGalaxy : extNsigma + 44 pNoise : + 45 pSaturated : + + 46 zHallMag : M + 47 zHallMagErr : dM + 48 zPetroMag : + 49 zPetroMagErr : + 50 zAperMag3 : Map + 51 zAperMag3Err : dMap + 52 zAperMag4 : Mkron + 53 zAperMag4Err : dMron + 54 zAperMag6 : + 55 zAperMag6Err : + 56 zGausig : + 57 zEll : + 58 zPA : posangle + 59 zErrBits : + 60 zDeblend : + 61 zClass : psfQF + 62 zClassStat : + 63 zppErrBits : photFlags + 64 zSeqNum : detID + 65 zObjID : + 66 zXi : + 67 zEta : + + 68 yHallMag : M + 69 yHallMagErr : dM + 70 yPetroMag : + 71 yPetroMagErr : + 72 yAperMag3 : Map + 73 yAperMag3Err : dMap + 74 yAperMag4 : Mkron + 75 yAperMag4Err : dMron + 76 yAperMag6 : + 77 yAperMag6Err : + 78 yGausig : + 79 yEll : + 80 yPA : posangle + 81 yErrBits : + 82 yDeblend : + 83 yClass : psfQF + 84 yClassStat : + 85 yppErrBits : photFlags + 86 ySeqNum : detID + 87 yObjID : + 88 yXi : + 89 yEta : + + 90 jHallMag : M + 91 jHallMagErr : dM + 92 jPetroMag : + 93 jPetroMagErr : + 94 jAperMag3 : Map + 95 jAperMag3Err : dMap + 96 jAperMag4 : Mkron + 97 jAperMag4Err : dMron + 98 jAperMag6 : + 99 jAperMag6Err : +100 jGausig : +101 jEll : +102 jPA : posangle +103 jErrBits : +104 jDeblend : +105 jClass : psfQF +106 jClassStat : +107 jppErrBits : photFlags +108 jSeqNum : detID +109 jObjID : +110 jXi : +111 jEta : + +112 hHallMag : M +113 hHallMagErr : dM +114 hPetroMag : +115 hPetroMagErr : +116 hAperMag3 : Map +117 hAperMag3Err : dMap +118 hAperMag4 : Mkron +119 hAperMag4Err : dMron +120 hAperMag6 : +121 hAperMag6Err : +122 hGausig : +123 hEll : +124 hPA : posangle +125 hErrBits : +126 hDeblend : +127 hClass : psfQF +128 hClassStat : +129 hppErrBits : photFlags +130 hSeqNum : detID +131 hObjID : +132 hXi : +133 hEta : + +134 k_1HallMag : M +135 k_1HallMagErr : dM +136 k_1PetroMag : +137 k_1PetroMagErr : +138 k_1AperMag3 : Map +139 k_1AperMag3Err : dMap +140 k_1AperMag4 : Mkron +141 k_1AperMag4Err : dMron +142 k_1AperMag6 : +143 k_1AperMag6Err : +144 k_1Gausig : +145 k_1Ell : +146 k_1PA : posangle +147 k_1ErrBits : +148 k_1Deblend : +149 k_1Class : psfQF +150 k_1ClassStat : +151 k_1ppErrBits : photFlags +152 k_1SeqNum : detID +153 k_1ObjID : +154 k_1Xi : +155 k_1Eta : + +156 k_2HallMag : M +157 k_2HallMagErr : dM +158 k_2PetroMag : +159 k_2PetroMagErr : +160 k_2AperMag3 : Map +161 k_2AperMag3Err : dMap +162 k_2AperMag4 : Mkron +163 k_2AperMag4Err : dMron +164 k_2AperMag6 : +165 k_2AperMag6Err : +166 k_2Gausig : +167 k_2Ell : +168 k_2PA : posangle +169 k_2ErrBits : +170 k_2Deblend : +171 k_2Class : psfQF +172 k_2ClassStat : +173 k_2ppErrBits : photFlags +174 k_2SeqNum : detID +175 k_2ObjID : +176 k_2Xi : +177 k_2Eta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ugcs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeZ = GetPhotcodeCodebyName ("UKIRT_Z"); if (!codeZ) Shutdown ("missing photcode UKIRT_Z"); + int codeY = GetPhotcodeCodebyName ("UKIRT_Y"); if (!codeY) Shutdown ("missing photcode UKIRT_Y"); + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (zHallMag, double, NELEM); + ALLOCATE_PTR (zHallMagErr, double, NELEM); + ALLOCATE_PTR (zAperMag3, double, NELEM); + ALLOCATE_PTR (zAperMag3Err, double, NELEM); + ALLOCATE_PTR (zAperMag4, double, NELEM); + ALLOCATE_PTR (zAperMag4Err, double, NELEM); + ALLOCATE_PTR (zPA, double, NELEM); + ALLOCATE_PTR (zClass, int, NELEM); + ALLOCATE_PTR (zppErrBits, int, NELEM); + ALLOCATE_PTR (zSeqNum, int, NELEM); + + ALLOCATE_PTR (yHallMag, double, NELEM); + ALLOCATE_PTR (yHallMagErr, double, NELEM); + ALLOCATE_PTR (yAperMag3, double, NELEM); + ALLOCATE_PTR (yAperMag3Err, double, NELEM); + ALLOCATE_PTR (yAperMag4, double, NELEM); + ALLOCATE_PTR (yAperMag4Err, double, NELEM); + ALLOCATE_PTR (yPA, double, NELEM); + ALLOCATE_PTR (yClass, int, NELEM); + ALLOCATE_PTR (yppErrBits, int, NELEM); + ALLOCATE_PTR (ySeqNum, int, NELEM); + + ALLOCATE_PTR (jHallMag, double, NELEM); + ALLOCATE_PTR (jHallMagErr, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + ALLOCATE_PTR (hHallMag, double, NELEM); + ALLOCATE_PTR (hHallMagErr, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (k1HallMag, double, NELEM); + ALLOCATE_PTR (k1HallMagErr, double, NELEM); + ALLOCATE_PTR (k1AperMag3, double, NELEM); + ALLOCATE_PTR (k1AperMag3Err, double, NELEM); + ALLOCATE_PTR (k1AperMag4, double, NELEM); + ALLOCATE_PTR (k1AperMag4Err, double, NELEM); + ALLOCATE_PTR (k1PA, double, NELEM); + ALLOCATE_PTR (k1Class, int, NELEM); + ALLOCATE_PTR (k1ppErrBits, int, NELEM); + ALLOCATE_PTR (k1SeqNum, int, NELEM); + + ALLOCATE_PTR (k2HallMag, double, NELEM); + ALLOCATE_PTR (k2HallMagErr, double, NELEM); + ALLOCATE_PTR (k2AperMag3, double, NELEM); + ALLOCATE_PTR (k2AperMag3Err, double, NELEM); + ALLOCATE_PTR (k2AperMag4, double, NELEM); + ALLOCATE_PTR (k2AperMag4Err, double, NELEM); + ALLOCATE_PTR (k2PA, double, NELEM); + ALLOCATE_PTR (k2Class, int, NELEM); + ALLOCATE_PTR (k2ppErrBits, int, NELEM); + ALLOCATE_PTR (k2SeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + dPARSE ( 8, 5, epoch, "epoch"); + jPARSE ( 23, 8, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 41, 23, mergedClass, "mergedClass"); + dPARSE ( 42, 41, pStar, "pStar"); + dPARSE ( 43, 42, pGalaxy, "pGalaxy"); + dPARSE ( 46, 43, zHallMag, "zHallMag"); + dPARSE ( 47, 46, zHallMagErr, "zHallMagErr"); + dPARSE ( 50, 47, zAperMag3, "zAperMag3"); + dPARSE ( 51, 50, zAperMag3Err, "zAperMag3Err"); + dPARSE ( 52, 51, zAperMag4, "zAperMag4"); + dPARSE ( 53, 52, zAperMag4Err, "zAperMag4Err"); + dPARSE ( 58, 53, zPA, "zPA"); + iPARSE ( 61, 58, zClass, "zClass"); + iPARSE ( 63, 61, zppErrBits, "zppErrBits"); + iPARSE ( 64, 63, zSeqNum, "zSeqNum"); + dPARSE ( 68, 64, yHallMag, "yHallMag"); + dPARSE ( 69, 68, yHallMagErr, "yHallMagErr"); + dPARSE ( 72, 69, yAperMag3, "yAperMag3"); + dPARSE ( 73, 72, yAperMag3Err, "yAperMag3Err"); + dPARSE ( 74, 73, yAperMag4, "yAperMag4"); + dPARSE ( 75, 74, yAperMag4Err, "yAperMag4Err"); + dPARSE ( 80, 75, yPA, "yPA"); + iPARSE ( 83, 80, yClass, "yClass"); + iPARSE ( 85, 83, yppErrBits, "yppErrBits"); + iPARSE ( 86, 85, ySeqNum, "ySeqNum"); + dPARSE ( 90, 86, jHallMag, "jHallMag"); + dPARSE ( 91, 90, jHallMagErr, "jHallMagErr"); + dPARSE ( 94, 91, jAperMag3, "jAperMag3"); + dPARSE ( 95, 94, jAperMag3Err, "jAperMag3Err"); + dPARSE ( 96, 95, jAperMag4, "jAperMag4"); + dPARSE ( 97, 96, jAperMag4Err, "jAperMag4Err"); + dPARSE (102, 97, jPA, "jPA"); + iPARSE (105, 102, jClass, "jClass"); + iPARSE (107, 105, jppErrBits, "jppErrBits"); + iPARSE (108, 107, jSeqNum, "jSeqNum"); + dPARSE (112, 108, hHallMag, "hHallMag"); + dPARSE (113, 112, hHallMagErr, "hHallMagErr"); + dPARSE (116, 113, hAperMag3, "hAperMag3"); + dPARSE (117, 116, hAperMag3Err, "hAperMag3Err"); + dPARSE (118, 117, hAperMag4, "hAperMag4"); + dPARSE (119, 118, hAperMag4Err, "hAperMag4Err"); + dPARSE (124, 119, hPA, "hPA"); + iPARSE (127, 124, hClass, "hClass"); + iPARSE (129, 127, hppErrBits, "hppErrBits"); + iPARSE (130, 129, hSeqNum, "hSeqNum"); + dPARSE (134, 130, k1HallMag, "k1HallMag"); + dPARSE (135, 134, k1HallMagErr, "k1HallMagErr"); + dPARSE (138, 135, k1AperMag3, "k1AperMag3"); + dPARSE (139, 138, k1AperMag3Err, "k1AperMag3Err"); + dPARSE (140, 139, k1AperMag4, "k1AperMag4"); + dPARSE (141, 140, k1AperMag4Err, "k1AperMag4Err"); + dPARSE (146, 141, k1PA, "k1PA"); + iPARSE (149, 146, k1Class, "k1Class"); + iPARSE (151, 149, k1ppErrBits, "k1ppErrBits"); + iPARSE (152, 151, k1SeqNum, "k1SeqNum"); + dPARSE (156, 152, k2HallMag, "k2HallMag"); + dPARSE (157, 156, k2HallMagErr, "k2HallMagErr"); + dPARSE (160, 157, k2AperMag3, "k2AperMag3"); + dPARSE (161, 160, k2AperMag3Err, "k2AperMag3Err"); + dPARSE (162, 161, k2AperMag4, "k2AperMag4"); + dPARSE (163, 162, k2AperMag4Err, "k2AperMag4Err"); + dPARSE (168, 163, k2PA, "k2PA"); + iPARSE (171, 168, k2Class, "k2Class"); + iPARSE (173, 171, k2ppErrBits, "k2ppErrBits"); + iPARSE (174, 173, k2SeqNum, "k2SeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (zHallMag, double, NELEM); + REALLOCATE (zHallMagErr, double, NELEM); + REALLOCATE (zAperMag3, double, NELEM); + REALLOCATE (zAperMag3Err, double, NELEM); + REALLOCATE (zAperMag4, double, NELEM); + REALLOCATE (zAperMag4Err, double, NELEM); + REALLOCATE (zPA, double, NELEM); + REALLOCATE (zClass, int, NELEM); + REALLOCATE (zppErrBits, int, NELEM); + REALLOCATE (zSeqNum, int, NELEM); + REALLOCATE (yHallMag, double, NELEM); + REALLOCATE (yHallMagErr, double, NELEM); + REALLOCATE (yAperMag3, double, NELEM); + REALLOCATE (yAperMag3Err, double, NELEM); + REALLOCATE (yAperMag4, double, NELEM); + REALLOCATE (yAperMag4Err, double, NELEM); + REALLOCATE (yPA, double, NELEM); + REALLOCATE (yClass, int, NELEM); + REALLOCATE (yppErrBits, int, NELEM); + REALLOCATE (ySeqNum, int, NELEM); + REALLOCATE (jHallMag, double, NELEM); + REALLOCATE (jHallMagErr, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + REALLOCATE (hHallMag, double, NELEM); + REALLOCATE (hHallMagErr, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (k1HallMag, double, NELEM); + REALLOCATE (k1HallMagErr, double, NELEM); + REALLOCATE (k1AperMag3, double, NELEM); + REALLOCATE (k1AperMag3Err, double, NELEM); + REALLOCATE (k1AperMag4, double, NELEM); + REALLOCATE (k1AperMag4Err, double, NELEM); + REALLOCATE (k1PA, double, NELEM); + REALLOCATE (k1Class, int, NELEM); + REALLOCATE (k1ppErrBits, int, NELEM); + REALLOCATE (k1SeqNum, int, NELEM); + REALLOCATE (k2HallMag, double, NELEM); + REALLOCATE (k2HallMagErr, double, NELEM); + REALLOCATE (k2AperMag3, double, NELEM); + REALLOCATE (k2AperMag3Err, double, NELEM); + REALLOCATE (k2AperMag4, double, NELEM); + REALLOCATE (k2AperMag4Err, double, NELEM); + REALLOCATE (k2PA, double, NELEM); + REALLOCATE (k2Class, int, NELEM); + REALLOCATE (k2ppErrBits, int, NELEM); + REALLOCATE (k2SeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // y measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = yHallMag[i]; + stars[Nstars].measure[0].dM = yHallMagErr[i]; + stars[Nstars].measure[0].Map = yAperMag3[i]; + stars[Nstars].measure[0].dMap = yAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = yAperMag4[i]; + stars[Nstars].measure[0].dMkron = yAperMag4Err[i]; + stars[Nstars].measure[0].posangle = yPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = yClass[i]; + stars[Nstars].measure[0].photFlags = yppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = ySeqNum[i]; + stars[Nstars].measure[0].photcode = codeY; + + // z measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = zHallMag[i]; + stars[Nstars].measure[1].dM = zHallMagErr[i]; + stars[Nstars].measure[1].Map = zAperMag3[i]; + stars[Nstars].measure[1].dMap = zAperMag3Err[i]; + stars[Nstars].measure[1].Mkron = zAperMag4[i]; + stars[Nstars].measure[1].dMkron = zAperMag4Err[i]; + stars[Nstars].measure[1].posangle = zPA[i]; // XXX units + stars[Nstars].measure[1].psfQF = zClass[i]; + stars[Nstars].measure[1].photFlags = zppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = zSeqNum[i]; + stars[Nstars].measure[1].photcode = codeZ; + + // J measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = jHallMag[i]; + stars[Nstars].measure[2].dM = jHallMagErr[i]; + stars[Nstars].measure[2].Map = jAperMag3[i]; + stars[Nstars].measure[2].dMap = jAperMag3Err[i]; + stars[Nstars].measure[2].Mkron = jAperMag4[i]; + stars[Nstars].measure[2].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[2].posangle = jPA[i]; // XXX units + stars[Nstars].measure[2].psfQF = jClass[i]; + stars[Nstars].measure[2].photFlags = jppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = jSeqNum[i]; + stars[Nstars].measure[2].photcode = codeJ; + + // H measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = hHallMag[i]; + stars[Nstars].measure[3].dM = hHallMagErr[i]; + stars[Nstars].measure[3].Map = hAperMag3[i]; + stars[Nstars].measure[3].dMap = hAperMag3Err[i]; + stars[Nstars].measure[3].Mkron = hAperMag4[i]; + stars[Nstars].measure[3].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[3].posangle = hPA[i]; // XXX units + stars[Nstars].measure[3].psfQF = hClass[i]; + stars[Nstars].measure[3].photFlags = hppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = hSeqNum[i]; + stars[Nstars].measure[3].photcode = codeH; + + // K1 measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = k1HallMag[i]; + stars[Nstars].measure[4].dM = k1HallMagErr[i]; + stars[Nstars].measure[4].Map = k1AperMag3[i]; + stars[Nstars].measure[4].dMap = k1AperMag3Err[i]; + stars[Nstars].measure[4].Mkron = k1AperMag4[i]; + stars[Nstars].measure[4].dMkron = k1AperMag4Err[i]; + stars[Nstars].measure[4].posangle = k1PA[i]; // XXX units + stars[Nstars].measure[4].psfQF = k1Class[i]; + stars[Nstars].measure[4].photFlags = k1ppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = k1SeqNum[i]; + stars[Nstars].measure[4].photcode = codeK; + + // K1 measure + stars[Nstars].measure[5].extID = sourceID[i]; + stars[Nstars].measure[5].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[5].R = ra[i]; + stars[Nstars].measure[5].D = dec[i]; + stars[Nstars].measure[5].psfChisq = pStar[i]; + stars[Nstars].measure[5].extNsigma = pGalaxy[i]; + stars[Nstars].measure[5].M = k2HallMag[i]; + stars[Nstars].measure[5].dM = k2HallMagErr[i]; + stars[Nstars].measure[5].Map = k2AperMag3[i]; + stars[Nstars].measure[5].dMap = k2AperMag3Err[i]; + stars[Nstars].measure[5].Mkron = k2AperMag4[i]; + stars[Nstars].measure[5].dMkron = k2AperMag4Err[i]; + stars[Nstars].measure[5].posangle = k2PA[i]; // XXX units + stars[Nstars].measure[5].psfQF = k2Class[i]; + stars[Nstars].measure[5].photFlags = k2ppErrBits[i]; + stars[Nstars].measure[5].photFlags2 = photFlags2; + stars[Nstars].measure[5].detID = k2SeqNum[i]; + stars[Nstars].measure[5].photcode = codeK; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (zHallMag); + FREE (zHallMagErr); + FREE (zAperMag3); + FREE (zAperMag3Err); + FREE (zAperMag4); + FREE (zAperMag4Err); + FREE (zPA); + FREE (zClass); + FREE (zppErrBits); + FREE (zSeqNum); + + FREE (yHallMag); + FREE (yHallMagErr); + FREE (yAperMag3); + FREE (yAperMag3Err); + FREE (yAperMag4); + FREE (yAperMag4Err); + FREE (yPA); + FREE (yClass); + FREE (yppErrBits); + FREE (ySeqNum); + + FREE (jHallMag); + FREE (jHallMagErr); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + FREE (hHallMag); + FREE (hHallMagErr); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (k1HallMag); + FREE (k1HallMagErr); + FREE (k1AperMag3); + FREE (k1AperMag3Err); + FREE (k1AperMag4); + FREE (k1AperMag4Err); + FREE (k1PA); + FREE (k1Class); + FREE (k1ppErrBits); + FREE (k1SeqNum); + + FREE (k2HallMag); + FREE (k2HallMagErr); + FREE (k2AperMag3); + FREE (k2AperMag3Err); + FREE (k2AperMag4); + FREE (k2AperMag4Err); + FREE (k2PA); + FREE (k2Class); + FREE (k2ppErrBits); + FREE (k2SeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ugps.c (revision 41170) @@ -0,0 +1,682 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + + 6 sigRa : + 7 sigDec : + 8 epoch : t ???? -- how is there a single epoch? + 9 muRa : + 10 muDec : + 11 sigMuRa : + 12 sigMuDec : + 13 chi2 : + 14 nFrames : + + 15 cx : + 16 cy : + 17 cz : + 18 htmID : + 19 l : + 20 b : + 21 lambda : + 22 eta : + 23 priOrSec : primary bit in dbFlags + + 24 jmhPnt : + 25 jmhPntErr : + 26 hmk1Pnt : + 27 hmk1PntErr : + 28 h2mk1Pnt : + 29 h2mk1PntErr : + + 30 mergedClassStat : + 31 mergedClass : psfQFperf or photFlags2 + 32 pStar : psfChisq + 33 pGalaxy : extNsigma + 34 pNoise : + 35 pSaturated : + + 36 jAperMag1 : M + 37 jAperMag1Err : dM + 38 jAperMag3 : Map + 39 jAperMag3Err : dMap + 40 jAperMag4 : Mkron + 41 jAperMag4Err : dMron + 42 jGausig : + 43 jEll : + 44 jPA : posangle + 45 jErrBits : + 46 jDeblend : + 47 jClass : psfQF + 48 jClassStat : + 49 jppErrBits : photFlags + 50 jSeqNum : detID + 51 jObjID : + 52 jXi : + 53 jEta : + 54 hAperMag1 : M + 55 hAperMag1Err : dM + 56 hAperMag3 : Map + 57 hAperMag3Err : dMap + 58 hAperMag4 : Mkron + 59 hAperMag4Err : dMron + 60 hGausig : + 61 hEll : + 62 hPA : posangle + 63 hErrBits : + 64 hDeblend : + 65 hClass : psfQF + 66 hClassStat : + 67 hppErrBits : photFlags + 68 hSeqNum : detID + 69 hObjID : + 70 hXi : + 71 hEta : + 72 k_1AperMag1 : M + 73 k_1AperMag1Err : dM + 74 k_1AperMag3 : Map + 75 k_1AperMag3Err : dMap + 76 k_1AperMag4 : Mkron + 77 k_1AperMag4Err : dMron + 78 k_1Gausig : + 79 k_1Ell : + 80 k_1PA : posangle + 81 k_1ErrBits : + 82 k_1Deblend : + 83 k_1Class : psfQF + 84 k_1ClassStat : + 85 k_1ppErrBits : photFlags + 86 k_1SeqNum : detID + 87 k_1ObjID : + 88 k_1Xi : + 89 k_1Eta : + 90 k_2AperMag1 : M + 91 k_2AperMag1Err : dM + 92 k_2AperMag3 : Map + 93 k_2AperMag3Err : dMap + 94 k_2AperMag4 : Mkron + 95 k_2AperMag4Err : dMron + 96 k_2Gausig : + 97 k_2Ell : + 98 k_2PA : posangle + 99 k_2ErrBits : +100 k_2Deblend : +101 k_2Class : psfQF +102 k_2ClassStat : +103 k_2ppErrBits : photFlags +104 k_2SeqNum : detID +105 k_2ObjID : +106 k_2Xi : +107 k_2Eta : +108 h_2AperMag1 : M +109 h_2AperMag1Err : dM +110 h_2AperMag3 : Map +111 h_2AperMag3Err : dMap +112 h_2AperMag4 : Mkron +113 h_2AperMag4Err : dMron +114 h_2Gausig : +115 h_2Ell : +116 h_2PA : posangle +117 h_2ErrBits : +118 h_2Deblend : +119 h_2Class : psfQF +120 h_2ClassStat : +121 h_2ppErrBits : photFlags +122 h_2SeqNum : detID +123 h_2ObjID : +124 h_2Xi : +125 h_2Eta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ugps (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (jAperMag1, double, NELEM); + ALLOCATE_PTR (jAperMag1Err, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + ALLOCATE_PTR (hAperMag1, double, NELEM); + ALLOCATE_PTR (hAperMag1Err, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (k1AperMag1, double, NELEM); + ALLOCATE_PTR (k1AperMag1Err, double, NELEM); + ALLOCATE_PTR (k1AperMag3, double, NELEM); + ALLOCATE_PTR (k1AperMag3Err, double, NELEM); + ALLOCATE_PTR (k1AperMag4, double, NELEM); + ALLOCATE_PTR (k1AperMag4Err, double, NELEM); + ALLOCATE_PTR (k1PA, double, NELEM); + ALLOCATE_PTR (k1Class, int, NELEM); + ALLOCATE_PTR (k1ppErrBits, int, NELEM); + ALLOCATE_PTR (k1SeqNum, int, NELEM); + + ALLOCATE_PTR (k2AperMag1, double, NELEM); + ALLOCATE_PTR (k2AperMag1Err, double, NELEM); + ALLOCATE_PTR (k2AperMag3, double, NELEM); + ALLOCATE_PTR (k2AperMag3Err, double, NELEM); + ALLOCATE_PTR (k2AperMag4, double, NELEM); + ALLOCATE_PTR (k2AperMag4Err, double, NELEM); + ALLOCATE_PTR (k2PA, double, NELEM); + ALLOCATE_PTR (k2Class, int, NELEM); + ALLOCATE_PTR (k2ppErrBits, int, NELEM); + ALLOCATE_PTR (k2SeqNum, int, NELEM); + + ALLOCATE_PTR (h2AperMag1, double, NELEM); + ALLOCATE_PTR (h2AperMag1Err, double, NELEM); + ALLOCATE_PTR (h2AperMag3, double, NELEM); + ALLOCATE_PTR (h2AperMag3Err, double, NELEM); + ALLOCATE_PTR (h2AperMag4, double, NELEM); + ALLOCATE_PTR (h2AperMag4Err, double, NELEM); + ALLOCATE_PTR (h2PA, double, NELEM); + ALLOCATE_PTR (h2Class, int, NELEM); + ALLOCATE_PTR (h2ppErrBits, int, NELEM); + ALLOCATE_PTR (h2SeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + dPARSE ( 8, 5, epoch, "epoch"); + jPARSE ( 23, 8, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 31, 23, mergedClass, "mergedClass"); + dPARSE ( 32, 31, pStar, "pStar"); + dPARSE ( 33, 32, pGalaxy, "pGalaxy"); + dPARSE ( 36, 33, jAperMag1, "jAperMag1"); + dPARSE ( 37, 36, jAperMag1Err, "jAperMag1Err"); + dPARSE ( 38, 37, jAperMag3, "jAperMag3"); + dPARSE ( 39, 38, jAperMag3Err, "jAperMag3Err"); + dPARSE ( 40, 39, jAperMag4, "jAperMag4"); + dPARSE ( 41, 40, jAperMag4Err, "jAperMag4Err"); + dPARSE ( 44, 41, jPA, "jPA"); + iPARSE ( 47, 44, jClass, "jClass"); + iPARSE ( 49, 47, jppErrBits, "jppErrBits"); + iPARSE ( 50, 49, jSeqNum, "jSeqNum"); + dPARSE ( 54, 50, hAperMag1, "hAperMag1"); + dPARSE ( 55, 54, hAperMag1Err, "hAperMag1Err"); + dPARSE ( 56, 55, hAperMag3, "hAperMag3"); + dPARSE ( 57, 56, hAperMag3Err, "hAperMag3Err"); + dPARSE ( 58, 57, hAperMag4, "hAperMag4"); + dPARSE ( 59, 58, hAperMag4Err, "hAperMag4Err"); + dPARSE ( 62, 59, hPA, "hPA"); + iPARSE ( 65, 62, hClass, "hClass"); + iPARSE ( 67, 65, hppErrBits, "hppErrBits"); + iPARSE ( 68, 67, hSeqNum, "hSeqNum"); + dPARSE ( 72, 68, k1AperMag1, "k1AperMag1"); + dPARSE ( 73, 72, k1AperMag1Err, "k1AperMag1Err"); + dPARSE ( 74, 73, k1AperMag3, "k1AperMag3"); + dPARSE ( 75, 74, k1AperMag3Err, "k1AperMag3Err"); + dPARSE ( 76, 75, k1AperMag4, "k1AperMag4"); + dPARSE ( 77, 76, k1AperMag4Err, "k1AperMag4Err"); + dPARSE ( 80, 77, k1PA, "k1PA"); + iPARSE ( 83, 80, k1Class, "k1Class"); + iPARSE ( 85, 83, k1ppErrBits, "k1ppErrBits"); + iPARSE ( 86, 85, k1SeqNum, "k1SeqNum"); + dPARSE ( 90, 86, k2AperMag1, "k2AperMag1"); + dPARSE ( 91, 90, k2AperMag1Err, "k2AperMag1Err"); + dPARSE ( 92, 91, k2AperMag3, "k2AperMag3"); + dPARSE ( 93, 92, k2AperMag3Err, "k2AperMag3Err"); + dPARSE ( 94, 93, k2AperMag4, "k2AperMag4"); + dPARSE ( 95, 94, k2AperMag4Err, "k2AperMag4Err"); + dPARSE ( 98, 95, k2PA, "k2PA"); + iPARSE (101, 98, k2Class, "k2Class"); + iPARSE (103, 101, k2ppErrBits, "k2ppErrBits"); + iPARSE (104, 103, k2SeqNum, "k2SeqNum"); + dPARSE (108, 104, h2AperMag1, "hAperMag1"); + dPARSE (109, 108, h2AperMag1Err, "hAperMag1Err"); + dPARSE (110, 109, h2AperMag3, "hAperMag3"); + dPARSE (111, 110, h2AperMag3Err, "hAperMag3Err"); + dPARSE (112, 111, h2AperMag4, "hAperMag4"); + dPARSE (113, 112, h2AperMag4Err, "hAperMag4Err"); + dPARSE (116, 113, h2PA, "hPA"); + iPARSE (119, 116, h2Class, "hClass"); + iPARSE (121, 119, h2ppErrBits, "hppErrBits"); + iPARSE (122, 121, h2SeqNum, "hSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (jAperMag1, double, NELEM); + REALLOCATE (jAperMag1Err, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + REALLOCATE (hAperMag1, double, NELEM); + REALLOCATE (hAperMag1Err, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (k1AperMag1, double, NELEM); + REALLOCATE (k1AperMag1Err, double, NELEM); + REALLOCATE (k1AperMag3, double, NELEM); + REALLOCATE (k1AperMag3Err, double, NELEM); + REALLOCATE (k1AperMag4, double, NELEM); + REALLOCATE (k1AperMag4Err, double, NELEM); + REALLOCATE (k1PA, double, NELEM); + REALLOCATE (k1Class, int, NELEM); + REALLOCATE (k1ppErrBits, int, NELEM); + REALLOCATE (k1SeqNum, int, NELEM); + REALLOCATE (k2AperMag1, double, NELEM); + REALLOCATE (k2AperMag1Err, double, NELEM); + REALLOCATE (k2AperMag3, double, NELEM); + REALLOCATE (k2AperMag3Err, double, NELEM); + REALLOCATE (k2AperMag4, double, NELEM); + REALLOCATE (k2AperMag4Err, double, NELEM); + REALLOCATE (k2PA, double, NELEM); + REALLOCATE (k2Class, int, NELEM); + REALLOCATE (k2ppErrBits, int, NELEM); + REALLOCATE (k2SeqNum, int, NELEM); + REALLOCATE (h2AperMag1, double, NELEM); + REALLOCATE (h2AperMag1Err, double, NELEM); + REALLOCATE (h2AperMag3, double, NELEM); + REALLOCATE (h2AperMag3Err, double, NELEM); + REALLOCATE (h2AperMag4, double, NELEM); + REALLOCATE (h2AperMag4Err, double, NELEM); + REALLOCATE (h2PA, double, NELEM); + REALLOCATE (h2Class, int, NELEM); + REALLOCATE (h2ppErrBits, int, NELEM); + REALLOCATE (h2SeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // J measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = jAperMag1[i]; + stars[Nstars].measure[0].dM = jAperMag1Err[i]; + stars[Nstars].measure[0].Map = jAperMag3[i]; + stars[Nstars].measure[0].dMap = jAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = jAperMag4[i]; + stars[Nstars].measure[0].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[0].posangle = jPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = jClass[i]; + stars[Nstars].measure[0].photFlags = jppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = jSeqNum[i]; + stars[Nstars].measure[0].photcode = codeJ; + + // H measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = hAperMag1[i]; + stars[Nstars].measure[1].dM = hAperMag1Err[i]; + stars[Nstars].measure[1].Map = hAperMag3[i]; + stars[Nstars].measure[1].dMap = hAperMag3Err[i]; + stars[Nstars].measure[1].Mkron = hAperMag4[i]; + stars[Nstars].measure[1].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[1].posangle = hPA[i]; // XXX units + stars[Nstars].measure[1].psfQF = hClass[i]; + stars[Nstars].measure[1].photFlags = hppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = hSeqNum[i]; + stars[Nstars].measure[1].photcode = codeH; + + // K1 measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = k1AperMag1[i]; + stars[Nstars].measure[2].dM = k1AperMag1Err[i]; + stars[Nstars].measure[2].Map = k1AperMag3[i]; + stars[Nstars].measure[2].dMap = k1AperMag3Err[i]; + stars[Nstars].measure[2].Mkron = k1AperMag4[i]; + stars[Nstars].measure[2].dMkron = k1AperMag4Err[i]; + stars[Nstars].measure[2].posangle = k1PA[i]; // XXX units + stars[Nstars].measure[2].psfQF = k1Class[i]; + stars[Nstars].measure[2].photFlags = k1ppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = k1SeqNum[i]; + stars[Nstars].measure[2].photcode = codeK; + + // K1 measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = k2AperMag1[i]; + stars[Nstars].measure[3].dM = k2AperMag1Err[i]; + stars[Nstars].measure[3].Map = k2AperMag3[i]; + stars[Nstars].measure[3].dMap = k2AperMag3Err[i]; + stars[Nstars].measure[3].Mkron = k2AperMag4[i]; + stars[Nstars].measure[3].dMkron = k2AperMag4Err[i]; + stars[Nstars].measure[3].posangle = k2PA[i]; // XXX units + stars[Nstars].measure[3].psfQF = k2Class[i]; + stars[Nstars].measure[3].photFlags = k2ppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = k2SeqNum[i]; + stars[Nstars].measure[3].photcode = codeK; + + // H2 measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = h2AperMag1[i]; + stars[Nstars].measure[4].dM = h2AperMag1Err[i]; + stars[Nstars].measure[4].Map = h2AperMag3[i]; + stars[Nstars].measure[4].dMap = h2AperMag3Err[i]; + stars[Nstars].measure[4].Mkron = h2AperMag4[i]; + stars[Nstars].measure[4].dMkron = h2AperMag4Err[i]; + stars[Nstars].measure[4].posangle = h2PA[i]; // XXX units + stars[Nstars].measure[4].psfQF = h2Class[i]; + stars[Nstars].measure[4].photFlags = h2ppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = h2SeqNum[i]; + stars[Nstars].measure[4].photcode = codeH; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (jAperMag1); + FREE (jAperMag1Err); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + FREE (hAperMag1); + FREE (hAperMag1Err); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (k1AperMag1); + FREE (k1AperMag1Err); + FREE (k1AperMag3); + FREE (k1AperMag3Err); + FREE (k1AperMag4); + FREE (k1AperMag4Err); + FREE (k1PA); + FREE (k1Class); + FREE (k1ppErrBits); + FREE (k1SeqNum); + + FREE (k2AperMag1); + FREE (k2AperMag1Err); + FREE (k2AperMag3); + FREE (k2AperMag3Err); + FREE (k2AperMag4); + FREE (k2AperMag4Err); + FREE (k2PA); + FREE (k2Class); + FREE (k2ppErrBits); + FREE (k2SeqNum); + + FREE (h2AperMag1); + FREE (h2AperMag1Err); + FREE (h2AperMag3); + FREE (h2AperMag3Err); + FREE (h2AperMag4); + FREE (h2AperMag4Err); + FREE (h2PA); + FREE (h2Class); + FREE (h2ppErrBits); + FREE (h2SeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_uhs.c (revision 41170) @@ -0,0 +1,338 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UHS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + 1 sourceID : extID + 2 cuEventID : + 3 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 4 ra : R + 5 dec : D + 6 cx : + 7 cy : + 8 cz : + 9 htmID : + 10 l : + 11 b : + 12 lambda : + 13 eta : + 14 priOrSec : primary bit in dbFlags + 15 mergedClassStat : + 16 mergedClass : psfQFperf or photFlags2 + 17 pStar : psfChisq + 18 pGalaxy : extNsigma + 19 pNoise : + 20 pSaturated : + 21 eBV : + 22 aJ : + 23 jHallMag : M + 24 jHallMagErr : dM + 25 jPetroMag : + 26 jPetroMagErr : + 27 jAperMag3 : Map + 28 jAperMag3Err : dMap + 29 jAperMag4 : Mkron + 30 jAperMag4Err : dMron + 31 jAperMag6 : + 32 jAperMag6Err : + 33 jGausig : + 34 jEll : + 35 jPA : posangle + 36 jErrBits : + 37 jDeblend : + 38 jClass : psfQF + 39 jClassStat : + 40 jppErrBits : photFlags + 41 jSeqNum : detID + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 1; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_uhs (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_UHS_J"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + ALLOCATE_PTR (jHallMag, double, NELEM); + ALLOCATE_PTR (jHallMagErr, double, NELEM); + ALLOCATE_PTR (jAperMag3, double, NELEM); + ALLOCATE_PTR (jAperMag3Err, double, NELEM); + ALLOCATE_PTR (jAperMag4, double, NELEM); + ALLOCATE_PTR (jAperMag4Err, double, NELEM); + ALLOCATE_PTR (jPA, double, NELEM); + ALLOCATE_PTR (jClass, int, NELEM); + ALLOCATE_PTR (jppErrBits, int, NELEM); + ALLOCATE_PTR (jSeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 3, 1, frameSetID, "frameSetID"); + dPARSE ( 4, 3, ra, "ra"); + dPARSE ( 5, 4, dec, "dec"); + jPARSE (14, 5, priOrSec, "priOrSec"); + iPARSE (16, 14, mergedClass, "mergedClass"); + dPARSE (17, 16, pStar, "pStar"); + dPARSE (18, 17, pGalaxy, "pGalaxy"); + dPARSE (23, 18, jHallMag, "jHallMag"); + dPARSE (24, 23, jHallMagErr, "jHallMagErr"); + dPARSE (27, 24, jAperMag3, "jAperMag3"); + dPARSE (28, 27, jAperMag3Err, "jAperMag3Err"); + dPARSE (29, 28, jAperMag4, "jAperMag4"); + dPARSE (30, 29, jAperMag4Err, "jAperMag4Err"); + dPARSE (35, 30, jPA, "jPA"); + iPARSE (38, 35, jClass, "jClass"); + iPARSE (40, 38, jppErrBits, "jppErrBits"); + iPARSE (41, 40, jSeqNum, "jSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (jHallMag, double, NELEM); + REALLOCATE (jHallMagErr, double, NELEM); + REALLOCATE (jAperMag3, double, NELEM); + REALLOCATE (jAperMag3Err, double, NELEM); + REALLOCATE (jAperMag4, double, NELEM); + REALLOCATE (jAperMag4Err, double, NELEM); + REALLOCATE (jPA, double, NELEM); + REALLOCATE (jClass, int, NELEM); + REALLOCATE (jppErrBits, int, NELEM); + REALLOCATE (jSeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have one UKIRT UHS (J-band) measurement per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = jHallMag[i]; + stars[Nstars].measure[0].dM = jHallMagErr[i]; + stars[Nstars].measure[0].Map = jAperMag3[i]; + stars[Nstars].measure[0].dMap = jAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = jAperMag4[i]; + stars[Nstars].measure[0].dMkron = jAperMag4Err[i]; + stars[Nstars].measure[0].posangle = jPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = jClass[i]; + stars[Nstars].measure[0].photFlags = jppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = jSeqNum[i]; + stars[Nstars].measure[0].photcode = codeJ; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + FREE (jHallMag); + FREE (jHallMagErr); + FREE (jAperMag3); + FREE (jAperMag3Err); + FREE (jAperMag4); + FREE (jAperMag4Err); + FREE (jPA); + FREE (jClass); + FREE (jppErrBits); + FREE (jSeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_readstars_ulas.c (revision 41170) @@ -0,0 +1,822 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +/* this function reads the values of interest from the UKIRT UGCS CSV files: + + data model description: doc/mapping_UKIDSS_unWISE_DVO.pdf + + N -- field (column) number in CSV file (1 counting) + | UKIRT field name : DVO measure field + + 1 sourceID : extID + 2 frameSetID : imageID (assumes we do not try to go backwards for these photcodes) + 3 ra : R + 4 dec : D + 5 sigRa : + 6 sigDec : + 7 epoch : t ???? -- how is there a single epoch? + 8 muRa : + 9 muDec : + 10 sigMuRa : + 11 sigMuDec : + 12 chi2 : + 13 nFrames : + 14 cx : + 15 cy : + 16 cz : + 17 htmID : + 18 l : + 19 b : + 20 lambda : + 21 eta : + 22 priOrSec : primary bit in dbFlags + 23 ymj_1Pnt : + 24 ymj_1PntErr : + 25 j_1mhPnt : + 26 j_1mhPntErr : + 27 hmkPnt : + 28 hmkPntErr : + 29 ymj_1Ext : + 30 ymj_1ExtErr : + 31 j_1mhExt : + 32 j_1mhExtErr : + 33 hmkExt : + 34 hmkExtErr : + 35 mergedClassStat : + 36 mergedClass : psfQFperf or photFlags2 + 37 pStar : psfChisq + 38 pGalaxy : extNsigma + 39 pNoise : + 40 pSaturated : + 41 eBV : + 42 aY : + 43 aJ + 44 aH + 45 aK + 46 yHallMag : M + 47 yHallMagErr : dM + 48 yPetroMag : + 49 yPetroMagErr : + 50 yAperMag3 : Map + 51 yAperMag3Err : dMap + 52 yAperMag4 : Mkron + 53 yAperMag4Err : dMron + 54 yAperMag6 : + 55 yAperMag6Err : + 56 yGausig : + 57 yEll : + 58 yPA : posangle + 59 yErrBits : + 60 yDeblend : + 61 yClass : psfQF + 62 yClassStat : + 63 yppErrBits : photFlags + 64 ySeqNum : detID + 65 yObjID : + 66 yXi : + 67 yEta : + 68 j_1HallMag : M + 69 j_1HallMagErr : dM + 70 j_1PetroMag : + 71 j_1PetroMagErr : + 72 j_1AperMag3 : Map + 73 j_1AperMag3Err : dMap + 74 j_1AperMag4 : Mkron + 75 j_1AperMag4Err : dMron + 76 j_1AperMag6 : + 77 j_1AperMag6Err : + 78 j_1Gausig : + 79 j_1Ell : + 80 j_1PA : posangle + 81 j_1ErrBits : + 82 j_1Deblend : + 83 j_1Class : psfQF + 84 j_1ClassStat : + 85 j_1ppErrBits : photFlags + 86 j_1SeqNum : detID + 87 j_1ObjID : + 88 j_1Xi : + 89 j_1Eta : + 90 j_2HallMag : M + 91 j_2HallMagErr : dM + 92 j_2PetroMag : + 93 j_2PetroMagErr : + 94 j_2AperMag3 : Map + 95 j_2AperMag3Err : dMap + 96 j_2AperMag4 : Mkron + 97 j_2AperMag4Err : dMron + 98 j_2AperMag6 : + 99 j_2AperMag6Err : +100 j_2Gausig : +101 j_2Ell : +102 j_2PA : posangle +103 j_2ErrBits : +104 j_2Deblend : +105 j_2Class : psfQF +106 j_2ClassStat : +107 j_2ppErrBits : photFlags +108 j_2SeqNum : detID +109 j_2ObjID : +110 j_2Xi : +111 j_2Eta : +112 hHallMag : M +113 hHallMagErr : dM +114 hPetroMag : +115 hPetroMagErr : +116 hAperMag3 : Map +117 hAperMag3Err : dMap +118 hAperMag4 : Mkron +119 hAperMag4Err : dMron +120 hAperMag6 : +121 hAperMag6Err : +122 hGausig : +123 hEll : +124 hPA : posangle +125 hErrBits : +126 hDeblend : +127 hClass : psfQF +128 hClassStat : +129 hppErrBits : photFlags +130 hSeqNum : detID +131 hObjID : +132 hXi : +133 hEta : +134 kHallMag : M +135 kHallMagErr : dM +136 kPetroMag : +137 kPetroMagErr : +138 kAperMag3 : Map +139 kAperMag3Err : dMap +140 kAperMag4 : Mkron +141 kAperMag4Err : dMron +142 kAperMag6 : +143 kAperMag6Err : +144 kGausig : +145 kEll : +146 kPA : posangle +147 kErrBits : +148 kDeblend : +149 kClass : psfQF +150 kClassStat : +151 kppErrBits : photFlags +152 kSeqNum : detID +153 kObjID : +154 kXi : +155 kEta : + +*/ + +# define iPARSE(NS,NE,FIELD,NAME) { \ + cA = iparse_csv_rpt (&ivalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = ivalue; } + +# define jPARSE(NS,NE,FIELD,NAME) { \ + cA = jparse_csv_rpt (&jvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = jvalue; } + +# define dPARSE(NS,NE,FIELD,NAME) { \ + cA = dparse_csv_rpt (&dvalue, (NS), (NE), cA, &readStatus); \ + if (!readStatus && VERBOSE) { \ + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", (NS), NAME, c0); \ + } \ + lineStatus &= readStatus; FIELD[Nelem] = dvalue; } + +static int Nline_read = 0; // track number of lines read so far (use to skip lines as well) +static int Nskip = 0; // UKIRT UHS CSV files have a single header row (and no special character to mark) + +UKIRT_Stars *loadukirt_uhs_readstars_ulas (FILE *f, char *buffer, int *nstart, AddstarClientOptions *options, int *nstars) { + + int codeY = GetPhotcodeCodebyName ("UKIRT_Y"); if (!codeY) Shutdown ("missing photcode UKIRT_Y"); + int codeJ = GetPhotcodeCodebyName ("UKIRT_J"); if (!codeJ) Shutdown ("missing photcode UKIRT_J"); + int codeH = GetPhotcodeCodebyName ("UKIRT_H"); if (!codeH) Shutdown ("missing photcode UKIRT_H"); + int codeK = GetPhotcodeCodebyName ("UKIRT_K"); if (!codeK) Shutdown ("missing photcode UKIRT_K"); + + // XXX I need a UKIRT UHS Mean Epoch + // time_t UKIRT_UHS_EPOCH = ohana_date_to_sec ("2016/01/01,00:00:00"); + // fprintf (stderr, "WARNING: using an invalid UKIRT_UHS_EPOCH (see loadukirt_uhs_readstars.c:60)\n"); + + int Nelem = 0; // number of valid rows read (vector elements) + int NELEM = 10000; // currently-allocated number of output rows + + // vectors to hold the data loaded from the csv file (names are UKIRT native names) + ALLOCATE_PTR (sourceID, uint64_t, NELEM); + ALLOCATE_PTR (frameSetID, uint64_t, NELEM); // XXX doc says 8-bytes for this value! + ALLOCATE_PTR (ra, double, NELEM); + ALLOCATE_PTR (dec, double, NELEM); + ALLOCATE_PTR (epoch, double, NELEM); + ALLOCATE_PTR (priOrSec, uint64_t, NELEM); + ALLOCATE_PTR (mergedClass, int, NELEM); // XXX doc says 2-bytes (could use a short) + ALLOCATE_PTR (pStar, double, NELEM); + ALLOCATE_PTR (pGalaxy, double, NELEM); + + ALLOCATE_PTR (yHallMag, double, NELEM); + ALLOCATE_PTR (yHallMagErr, double, NELEM); + ALLOCATE_PTR (yAperMag3, double, NELEM); + ALLOCATE_PTR (yAperMag3Err, double, NELEM); + ALLOCATE_PTR (yAperMag4, double, NELEM); + ALLOCATE_PTR (yAperMag4Err, double, NELEM); + ALLOCATE_PTR (yPA, double, NELEM); + ALLOCATE_PTR (yClass, int, NELEM); + ALLOCATE_PTR (yppErrBits, int, NELEM); + ALLOCATE_PTR (ySeqNum, int, NELEM); + + ALLOCATE_PTR (j1HallMag, double, NELEM); + ALLOCATE_PTR (j1HallMagErr, double, NELEM); + ALLOCATE_PTR (j1AperMag3, double, NELEM); + ALLOCATE_PTR (j1AperMag3Err, double, NELEM); + ALLOCATE_PTR (j1AperMag4, double, NELEM); + ALLOCATE_PTR (j1AperMag4Err, double, NELEM); + ALLOCATE_PTR (j1PA, double, NELEM); + ALLOCATE_PTR (j1Class, int, NELEM); + ALLOCATE_PTR (j1ppErrBits, int, NELEM); + ALLOCATE_PTR (j1SeqNum, int, NELEM); + + ALLOCATE_PTR (j2HallMag, double, NELEM); + ALLOCATE_PTR (j2HallMagErr, double, NELEM); + ALLOCATE_PTR (j2AperMag3, double, NELEM); + ALLOCATE_PTR (j2AperMag3Err, double, NELEM); + ALLOCATE_PTR (j2AperMag4, double, NELEM); + ALLOCATE_PTR (j2AperMag4Err, double, NELEM); + ALLOCATE_PTR (j2PA, double, NELEM); + ALLOCATE_PTR (j2Class, int, NELEM); + ALLOCATE_PTR (j2ppErrBits, int, NELEM); + ALLOCATE_PTR (j2SeqNum, int, NELEM); + + ALLOCATE_PTR (hHallMag, double, NELEM); + ALLOCATE_PTR (hHallMagErr, double, NELEM); + ALLOCATE_PTR (hAperMag3, double, NELEM); + ALLOCATE_PTR (hAperMag3Err, double, NELEM); + ALLOCATE_PTR (hAperMag4, double, NELEM); + ALLOCATE_PTR (hAperMag4Err, double, NELEM); + ALLOCATE_PTR (hPA, double, NELEM); + ALLOCATE_PTR (hClass, int, NELEM); + ALLOCATE_PTR (hppErrBits, int, NELEM); + ALLOCATE_PTR (hSeqNum, int, NELEM); + + ALLOCATE_PTR (kHallMag, double, NELEM); + ALLOCATE_PTR (kHallMagErr, double, NELEM); + ALLOCATE_PTR (kAperMag3, double, NELEM); + ALLOCATE_PTR (kAperMag3Err, double, NELEM); + ALLOCATE_PTR (kAperMag4, double, NELEM); + ALLOCATE_PTR (kAperMag4Err, double, NELEM); + ALLOCATE_PTR (kPA, double, NELEM); + ALLOCATE_PTR (kClass, int, NELEM); + ALLOCATE_PTR (kppErrBits, int, NELEM); + ALLOCATE_PTR (kSeqNum, int, NELEM); + + // we have a working buffer read from the file. we parse the lines in the working buffer + // until we reach the last chunk without an EOL char. at that point, we shift the start + // of the last (partial) line to the start of the buffer and re-fill. + + // we treat \n\r pair as a single EOL char to handle mac files: + + int Nstart = *nstart; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + while (!EndOfFile && (Nelem < NSTARS_MAX)) { + int Nbytes = BUFFER_SIZE - Nstart; + // we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + bzero (&buffer[Nstart], Nbytes + 1); + int Nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + break; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (Nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + char *c0 = buffer; // c0 always marks the start of a line + char *cA = NULL; // cA will carry the curr point within the line + while (bufferStatus) { + + // find the end of this current line (\n or \r). if we hit the end of the buffer (NULL), + // attempt to read more data. finish up when we hit the end of the file + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + Nline_read ++; + + // skip to the next line (but if EOF, do not overrun buffer) + if (Nline_read <= Nskip) { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // these are not needed: Gaia CSV files do not have any commented-out lines + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // for UKIRT UHS, we know which columns we want in advance + + int lineStatus = TRUE; + int readStatus; + double dvalue; + int ivalue; + uint64_t jvalue; + + // cA will follow the currently extracted field, c0 points to the start of the line + cA = c0; + + // Tref : TBD for UHS + + // the start of the line is the 1st element (fields are 1-counting) + jPARSE ( 1, 1, sourceID, "sourceID"); + iPARSE ( 2, 1, frameSetID, "frameSetID"); + dPARSE ( 3, 2, ra, "ra"); + dPARSE ( 4, 3, dec, "dec"); + dPARSE ( 7, 4, epoch, "epoch"); + jPARSE ( 22, 7, priOrSec, "priOrSec"); // watch for epoch + iPARSE ( 36, 22, mergedClass, "mergedClass"); + dPARSE ( 37, 36, pStar, "pStar"); + dPARSE ( 38, 37, pGalaxy, "pGalaxy"); + dPARSE ( 46, 38, yHallMag, "yHallMag"); + dPARSE ( 47, 46, yHallMagErr, "yHallMagErr"); + dPARSE ( 50, 47, yAperMag3, "yAperMag3"); + dPARSE ( 51, 50, yAperMag3Err, "yAperMag3Err"); + dPARSE ( 52, 51, yAperMag4, "yAperMag4"); + dPARSE ( 53, 52, yAperMag4Err, "yAperMag4Err"); + dPARSE ( 58, 53, yPA, "yPA"); + iPARSE ( 61, 58, yClass, "yClass"); + iPARSE ( 63, 61, yppErrBits, "yppErrBits"); + iPARSE ( 64, 63, ySeqNum, "ySeqNum"); + dPARSE ( 68, 64, j1HallMag, "j1HallMag"); + dPARSE ( 69, 68, j1HallMagErr, "j1HallMagErr"); + dPARSE ( 72, 69, j1AperMag3, "j1AperMag3"); + dPARSE ( 73, 72, j1AperMag3Err, "j1AperMag3Err"); + dPARSE ( 74, 73, j1AperMag4, "j1AperMag4"); + dPARSE ( 75, 74, j1AperMag4Err, "j1AperMag4Err"); + dPARSE ( 80, 75, j1PA, "j1PA"); + iPARSE ( 83, 80, j1Class, "j1Class"); + iPARSE ( 85, 83, j1ppErrBits, "j1ppErrBits"); + iPARSE ( 86, 85, j1SeqNum, "j1SeqNum"); + dPARSE ( 90, 86, j2HallMag, "j2HallMag"); + dPARSE ( 91, 90, j2HallMagErr, "j2HallMagErr"); + dPARSE ( 94, 91, j2AperMag3, "j2AperMag3"); + dPARSE ( 95, 94, j2AperMag3Err, "j2AperMag3Err"); + dPARSE ( 96, 95, j2AperMag4, "j2AperMag4"); + dPARSE ( 97, 96, j2AperMag4Err, "j2AperMag4Err"); + dPARSE (102, 97, j2PA, "j2PA"); + iPARSE (105, 102, j2Class, "j2Class"); + iPARSE (107, 105, j2ppErrBits, "j2ppErrBits"); + iPARSE (108, 107, j2SeqNum, "j2SeqNum"); + dPARSE (112, 108, hHallMag, "hHallMag"); + dPARSE (113, 112, hHallMagErr, "hHallMagErr"); + dPARSE (116, 113, hAperMag3, "hAperMag3"); + dPARSE (117, 116, hAperMag3Err, "hAperMag3Err"); + dPARSE (118, 117, hAperMag4, "hAperMag4"); + dPARSE (119, 118, hAperMag4Err, "hAperMag4Err"); + dPARSE (124, 119, hPA, "hPA"); + iPARSE (127, 124, hClass, "hClass"); + iPARSE (129, 127, hppErrBits, "hppErrBits"); + iPARSE (130, 129, hSeqNum, "hSeqNum"); + dPARSE (134, 130, kHallMag, "kHallMag"); + dPARSE (135, 134, kHallMagErr, "kHallMagErr"); + dPARSE (138, 135, kAperMag3, "kAperMag3"); + dPARSE (139, 138, kAperMag3Err, "kAperMag3Err"); + dPARSE (140, 139, kAperMag4, "kAperMag4"); + dPARSE (141, 140, kAperMag4Err, "kAperMag4Err"); + dPARSE (146, 141, kPA, "kPA"); + iPARSE (149, 146, kClass, "kClass"); + iPARSE (151, 149, kppErrBits, "kppErrBits"); + iPARSE (152, 151, kSeqNum, "kSeqNum"); + + if (!lineStatus && VERBOSE) { + // why do I need to copy temp here, does gprint modify the value of temp? + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + + REALLOCATE (sourceID, uint64_t, NELEM); + REALLOCATE (frameSetID, uint64_t, NELEM); + REALLOCATE (ra, double, NELEM); + REALLOCATE (dec, double, NELEM); + REALLOCATE (priOrSec, uint64_t, NELEM); + REALLOCATE (mergedClass, int, NELEM); + REALLOCATE (pStar, double, NELEM); + REALLOCATE (pGalaxy, double, NELEM); + REALLOCATE (yHallMag, double, NELEM); + REALLOCATE (yHallMagErr, double, NELEM); + REALLOCATE (yAperMag3, double, NELEM); + REALLOCATE (yAperMag3Err, double, NELEM); + REALLOCATE (yAperMag4, double, NELEM); + REALLOCATE (yAperMag4Err, double, NELEM); + REALLOCATE (yPA, double, NELEM); + REALLOCATE (yClass, int, NELEM); + REALLOCATE (yppErrBits, int, NELEM); + REALLOCATE (ySeqNum, int, NELEM); + REALLOCATE (j1HallMag, double, NELEM); + REALLOCATE (j1HallMagErr, double, NELEM); + REALLOCATE (j1AperMag3, double, NELEM); + REALLOCATE (j1AperMag3Err, double, NELEM); + REALLOCATE (j1AperMag4, double, NELEM); + REALLOCATE (j1AperMag4Err, double, NELEM); + REALLOCATE (j1PA, double, NELEM); + REALLOCATE (j1Class, int, NELEM); + REALLOCATE (j1ppErrBits, int, NELEM); + REALLOCATE (j1SeqNum, int, NELEM); + REALLOCATE (j2HallMag, double, NELEM); + REALLOCATE (j2HallMagErr, double, NELEM); + REALLOCATE (j2AperMag3, double, NELEM); + REALLOCATE (j2AperMag3Err, double, NELEM); + REALLOCATE (j2AperMag4, double, NELEM); + REALLOCATE (j2AperMag4Err, double, NELEM); + REALLOCATE (j2PA, double, NELEM); + REALLOCATE (j2Class, int, NELEM); + REALLOCATE (j2ppErrBits, int, NELEM); + REALLOCATE (j2SeqNum, int, NELEM); + REALLOCATE (hHallMag, double, NELEM); + REALLOCATE (hHallMagErr, double, NELEM); + REALLOCATE (hAperMag3, double, NELEM); + REALLOCATE (hAperMag3Err, double, NELEM); + REALLOCATE (hAperMag4, double, NELEM); + REALLOCATE (hAperMag4Err, double, NELEM); + REALLOCATE (hPA, double, NELEM); + REALLOCATE (hClass, int, NELEM); + REALLOCATE (hppErrBits, int, NELEM); + REALLOCATE (hSeqNum, int, NELEM); + REALLOCATE (kHallMag, double, NELEM); + REALLOCATE (kHallMagErr, double, NELEM); + REALLOCATE (kAperMag3, double, NELEM); + REALLOCATE (kAperMag3Err, double, NELEM); + REALLOCATE (kAperMag4, double, NELEM); + REALLOCATE (kAperMag4Err, double, NELEM); + REALLOCATE (kPA, double, NELEM); + REALLOCATE (kClass, int, NELEM); + REALLOCATE (kppErrBits, int, NELEM); + REALLOCATE (kSeqNum, int, NELEM); + } + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + + // Nelem is now the number of items (objects,stars) read from the Gaia CSV file + int NstarsIn = Nelem; + + double Rmin = +360.0; + double Rmax = -360.0; + double Dmin = +360.0; + double Dmax = -360.0; + + // start off where we finished on a previous read + int Nstars = *nstars; + int NSTARS = Nstars + 0.1*NstarsIn; + + ALLOCATE_PTR (stars, UKIRT_Stars, NSTARS); + + for (int i = 0; i < NstarsIn; i++) { + + Rmin = MIN (Rmin, ra[i]); + Rmax = MAX (Rmax, ra[i]); + Dmin = MIN (Dmin, dec[i]); + Dmax = MAX (Dmax, dec[i]); + + // only allocate the measures for stars as we define them + ALLOCATE (stars[Nstars].measure, Measure, UKIRT_NFILTER); + + // we have NFILTER (== 6) UKIRT measurements per object + dvo_average_init (&stars[Nstars].average); + for (int j = 0; j < UKIRT_NFILTER; j++) { + dvo_measure_init (&stars[Nstars].measure[j]); + } + + stars[Nstars].average.R = ra[i]; + stars[Nstars].average.D = dec[i]; + stars[Nstars].average.dR = NAN; + stars[Nstars].average.dD = NAN; + + stars[Nstars].flag = FALSE; + stars[Nstars].found = FALSE; + + int isPrimary = (!priOrSec[i] || (priOrSec[i] == frameSetID[i])) ? ID_MEAS_STACK_PRIMARY : 0x00; + int photFlags2 = isPrimary; + switch (mergedClass[i]) { + case 1: photFlags2 |= 0x01; break; // Galaxy + case 0: photFlags2 |= 0x02; break; // Noise + case -1: photFlags2 |= 0x04; break; // Star + case -2: photFlags2 |= 0x08; break; // probably star + case -3: photFlags2 |= 0x10; break; // probably galaxy + case -9: photFlags2 |= 0x20; break; // saturated + default: break; + } + + // y measure + stars[Nstars].measure[0].extID = sourceID[i]; + stars[Nstars].measure[0].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[0].R = ra[i]; + stars[Nstars].measure[0].D = dec[i]; + stars[Nstars].measure[0].psfChisq = pStar[i]; + stars[Nstars].measure[0].extNsigma = pGalaxy[i]; + stars[Nstars].measure[0].M = yHallMag[i]; + stars[Nstars].measure[0].dM = yHallMagErr[i]; + stars[Nstars].measure[0].Map = yAperMag3[i]; + stars[Nstars].measure[0].dMap = yAperMag3Err[i]; + stars[Nstars].measure[0].Mkron = yAperMag4[i]; + stars[Nstars].measure[0].dMkron = yAperMag4Err[i]; + stars[Nstars].measure[0].posangle = yPA[i]; // XXX units + stars[Nstars].measure[0].psfQF = yClass[i]; + stars[Nstars].measure[0].photFlags = yppErrBits[i]; + stars[Nstars].measure[0].photFlags2 = photFlags2; + stars[Nstars].measure[0].detID = ySeqNum[i]; + stars[Nstars].measure[0].photcode = codeY; + + // J_1 measure + stars[Nstars].measure[1].extID = sourceID[i]; + stars[Nstars].measure[1].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[1].R = ra[i]; + stars[Nstars].measure[1].D = dec[i]; + stars[Nstars].measure[1].psfChisq = pStar[i]; + stars[Nstars].measure[1].extNsigma = pGalaxy[i]; + stars[Nstars].measure[1].M = j1HallMag[i]; + stars[Nstars].measure[1].dM = j1HallMagErr[i]; + stars[Nstars].measure[1].Map = j1AperMag3[i]; + stars[Nstars].measure[1].dMap = j1AperMag3Err[i]; + stars[Nstars].measure[1].Mkron = j1AperMag4[i]; + stars[Nstars].measure[1].dMkron = j1AperMag4Err[i]; + stars[Nstars].measure[1].posangle = j1PA[i]; // XXX units + stars[Nstars].measure[1].psfQF = j1Class[i]; + stars[Nstars].measure[1].photFlags = j1ppErrBits[i]; + stars[Nstars].measure[1].photFlags2 = photFlags2; + stars[Nstars].measure[1].detID = j1SeqNum[i]; + stars[Nstars].measure[1].photcode = codeJ; + + // J measure + stars[Nstars].measure[2].extID = sourceID[i]; + stars[Nstars].measure[2].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[2].R = ra[i]; + stars[Nstars].measure[2].D = dec[i]; + stars[Nstars].measure[2].psfChisq = pStar[i]; + stars[Nstars].measure[2].extNsigma = pGalaxy[i]; + stars[Nstars].measure[2].M = j2HallMag[i]; + stars[Nstars].measure[2].dM = j2HallMagErr[i]; + stars[Nstars].measure[2].Map = j2AperMag3[i]; + stars[Nstars].measure[2].dMap = j2AperMag3Err[i]; + stars[Nstars].measure[2].Mkron = j2AperMag4[i]; + stars[Nstars].measure[2].dMkron = j2AperMag4Err[i]; + stars[Nstars].measure[2].posangle = j2PA[i]; // XXX units + stars[Nstars].measure[2].psfQF = j2Class[i]; + stars[Nstars].measure[2].photFlags = j2ppErrBits[i]; + stars[Nstars].measure[2].photFlags2 = photFlags2; + stars[Nstars].measure[2].detID = j2SeqNum[i]; + stars[Nstars].measure[2].photcode = codeJ; + + // H measure + stars[Nstars].measure[3].extID = sourceID[i]; + stars[Nstars].measure[3].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[3].R = ra[i]; + stars[Nstars].measure[3].D = dec[i]; + stars[Nstars].measure[3].psfChisq = pStar[i]; + stars[Nstars].measure[3].extNsigma = pGalaxy[i]; + stars[Nstars].measure[3].M = hHallMag[i]; + stars[Nstars].measure[3].dM = hHallMagErr[i]; + stars[Nstars].measure[3].Map = hAperMag3[i]; + stars[Nstars].measure[3].dMap = hAperMag3Err[i]; + stars[Nstars].measure[3].Mkron = hAperMag4[i]; + stars[Nstars].measure[3].dMkron = hAperMag4Err[i]; + stars[Nstars].measure[3].posangle = hPA[i]; // XXX units + stars[Nstars].measure[3].psfQF = hClass[i]; + stars[Nstars].measure[3].photFlags = hppErrBits[i]; + stars[Nstars].measure[3].photFlags2 = photFlags2; + stars[Nstars].measure[3].detID = hSeqNum[i]; + stars[Nstars].measure[3].photcode = codeH; + + // K measure + stars[Nstars].measure[4].extID = sourceID[i]; + stars[Nstars].measure[4].imageID = frameSetID[i]; // XXX check for frameSetID > 31-bit int + stars[Nstars].measure[4].R = ra[i]; + stars[Nstars].measure[4].D = dec[i]; + stars[Nstars].measure[4].psfChisq = pStar[i]; + stars[Nstars].measure[4].extNsigma = pGalaxy[i]; + stars[Nstars].measure[4].M = kHallMag[i]; + stars[Nstars].measure[4].dM = kHallMagErr[i]; + stars[Nstars].measure[4].Map = kAperMag3[i]; + stars[Nstars].measure[4].dMap = kAperMag3Err[i]; + stars[Nstars].measure[4].Mkron = kAperMag4[i]; + stars[Nstars].measure[4].dMkron = kAperMag4Err[i]; + stars[Nstars].measure[4].posangle = kPA[i]; // XXX units + stars[Nstars].measure[4].psfQF = kClass[i]; + stars[Nstars].measure[4].photFlags = kppErrBits[i]; + stars[Nstars].measure[4].photFlags2 = photFlags2; + stars[Nstars].measure[4].detID = kSeqNum[i]; + stars[Nstars].measure[4].photcode = codeK; + + stars[Nstars].average.Nmeasure = UKIRT_NFILTER; + Nstars ++; + + if (NSTARS >= Nstars) { + NSTARS += 10000; + REALLOCATE (stars, UKIRT_Stars, NSTARS); + } + } + + FREE (sourceID); + FREE (frameSetID); + FREE (ra); + FREE (dec); + FREE (epoch); + FREE (priOrSec); + FREE (mergedClass); + FREE (pStar); + FREE (pGalaxy); + + FREE (yHallMag); + FREE (yHallMagErr); + FREE (yAperMag3); + FREE (yAperMag3Err); + FREE (yAperMag4); + FREE (yAperMag4Err); + FREE (yPA); + FREE (yClass); + FREE (yppErrBits); + FREE (ySeqNum); + + FREE (j1HallMag); + FREE (j1HallMagErr); + FREE (j1AperMag3); + FREE (j1AperMag3Err); + FREE (j1AperMag4); + FREE (j1AperMag4Err); + FREE (j1PA); + FREE (j1Class); + FREE (j1ppErrBits); + FREE (j1SeqNum); + + FREE (j2HallMag); + FREE (j2HallMagErr); + FREE (j2AperMag3); + FREE (j2AperMag3Err); + FREE (j2AperMag4); + FREE (j2AperMag4Err); + FREE (j2PA); + FREE (j2Class); + FREE (j2ppErrBits); + FREE (j2SeqNum); + + FREE (hHallMag); + FREE (hHallMagErr); + FREE (hAperMag3); + FREE (hAperMag3Err); + FREE (hAperMag4); + FREE (hAperMag4Err); + FREE (hPA); + FREE (hClass); + FREE (hppErrBits); + FREE (hSeqNum); + + FREE (kHallMag); + FREE (kHallMagErr); + FREE (kAperMag3); + FREE (kAperMag3Err); + FREE (kAperMag4); + FREE (kAperMag4Err); + FREE (kPA); + FREE (kClass); + FREE (kppErrBits); + FREE (kSeqNum); + + *nstars = Nstars; + *nstart = Nstart; + return (stars); +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/addstar/src/loadukirt_uhs_table.c (revision 41170) @@ -0,0 +1,90 @@ +# include "addstar.h" +# include "ukirt_uhs.h" + +// one pass reads a single file at a time +int loadukirt_uhs_table (SkyList *skylistInput, char *filename, AddstarClientOptions *options) { + + // open the input file here and pass the pointer below + FILE *f = fopen (filename, "r"); + if (f == NULL) Shutdown ("can't read ukirt_uhs file: %s", filename); + + // we allocate one extra byte into which we never read so there will always be a NULL terminating the string + ALLOCATE_PTR (buffer, char, BUFFER_SIZE + 1); + bzero (buffer, BUFFER_SIZE + 1); + + // starting point of valid data in buffer; this is updated on each pass to readstars + int Nstart = 0; + + while (1) { + + // read the chunk from the file + fprintf (stderr, "loading %s\n", filename); + + // on each pass, we read a new chunk of data and receive Nstars + + int Nstars = 0; + UKIRT_Stars *stars = loadukirt_uhs_readstars (f, buffer, &Nstart, options, &Nstars); + if (!Nstars) { + fclose (f); + free (buffer); + free (stars); + return TRUE; // end of file reached + } + + fprintf (stderr, "writing %d stars to dvo\n", Nstars); + + // sort the stars by RA (Nmeasure agnostic) + loadukirt_uhs_sortStars (stars, Nstars); + + // scan through the stars, loading the containing catalogs + // skip through table for unsaved stars + for (int i = 0; i < Nstars; i++) { + if (stars[i].flag) continue; + + // scan forward until we read the UserPatch + if (stars[i].average.R < UserPatch.Rmin) continue; + if (stars[i].average.R > UserPatch.Rmax) break; + if (stars[i].average.D < UserPatch.Dmin) continue; + if (stars[i].average.D > UserPatch.Dmax) continue; + + // identify the relevant catalog + SkyList *skylist = SkyRegionByPoint_List (skylistInput, -1, stars[i].average.R, stars[i].average.D); + if (skylist[0].Nregions == 0) { + SkyListFree (skylist); + continue; + } + SkyRegion *region = skylist[0].regions[0]; + + // select stars matching this region (Nmeasure agnostic) + int Nsubset; + UKIRT_Stars *subset = loadukirt_uhs_make_subset (stars, Nstars, i, region, &Nsubset); + + // In parallel mode, write out the subset to a disk file. Block until a remote host + // is available. In serial mode, just match against the appropriate region and save + // NOTE: disable parallel mode for now: + // loadukirt_uhs_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0], options); + // loadukirt_uhs_catalog (Nmeasure agnostic) + loadukirt_uhs_catalog (subset, Nsubset, region, skylist[0].filename[0], options); + free (subset); + SkyListFree (skylist); + } + for (int i = 0; i < Nstars; i++) { + free (stars[i].measure); + } + free (stars); + } + + // wait for last remote clients to finish + // NOTE: disable parallel mode for now: + // harvest_all (); + + // we should not actually reach this point + return FALSE; +} + +/* modifying to read in smaller blocks of the input file at a time + + each pass on readstars needs to ... + +*/ + Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/include/dvomerge.h (revision 41170) @@ -31,5 +31,8 @@ int IMAGES_ONLY; int ACCEPT_MOTION; + +int ACCEPT_ASTROM; int RETAIN_AVE_PHOTOMETRY; + char CATDIR[DVO_MAX_PATH]; char GSCFILE[DVO_MAX_PATH]; Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/args.c (revision 41170) @@ -112,9 +112,15 @@ } - /* limit the impact of a dvomerge -parallel */ + /* accept input database average astrometry motions */ ACCEPT_MOTION = FALSE; if ((N = get_argument (*argc, argv, "-accept-motion"))) { remove_argument (N, argc, argv); ACCEPT_MOTION = TRUE; + } + /* accept input database average astrometry information */ + ACCEPT_ASTROM = FALSE; + if ((N = get_argument (*argc, argv, "-accept-astrom"))) { + remove_argument (N, argc, argv); + ACCEPT_ASTROM = TRUE; } @@ -337,9 +343,15 @@ } - /* limit the impact of a dvomerge -parallel */ + /* accept input database average astrometry motions */ ACCEPT_MOTION = FALSE; if ((N = get_argument (*argc, argv, "-accept-motion"))) { remove_argument (N, argc, argv); ACCEPT_MOTION = TRUE; + } + /* accept input database average astrometry information */ + ACCEPT_ASTROM = FALSE; + if ((N = get_argument (*argc, argv, "-accept-astrom"))) { + remove_argument (N, argc, argv); + ACCEPT_ASTROM = TRUE; } Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 41170) @@ -328,4 +328,5 @@ if (FORCE_MERGE) { strextend (&command, "-force-merge"); } if (ACCEPT_MOTION) { strextend (&command, "-accept-motion"); } + if (ACCEPT_ASTROM) { strextend (&command, "-accept-astrom"); } if (RETAIN_AVE_PHOTOMETRY) { strextend (&command, "-retain-ave-photometry"); } if (MATCHED_TABLES) { strextend (&command, "-matched-tables"); } Index: branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 41170) @@ -363,4 +363,5 @@ } + // XXX: add choice of secfilt // update the average properties to reflect the incoming entries: // if RETAIN_AVE_PHOTOMETRY is true and the original value is NAN, but the input value is not, accept the input: @@ -389,5 +390,5 @@ // we can choose to accept the proper-motion and parallax from the reference tgtcat - if (ACCEPT_MOTION) { + if (ACCEPT_MOTION || ACCEPT_ASTROM) { output[0].average[n].dR = input[0].average[N].dR; output[0].average[n].dD = input[0].average[N].dD; @@ -400,4 +401,8 @@ output[0].average[n].Tmean = input[0].average[N].Tmean; } + if (ACCEPT_ASTROM) { + output[0].average[n].R = input[0].average[N].R; + output[0].average[n].D = input[0].average[N].D; + } /* Nm is updated, but not written out in -update mode (for existing entries) @@ -461,5 +466,5 @@ // we can choose to accept the proper-motion and parallax from the reference tgtcat - if (ACCEPT_MOTION) { + if (ACCEPT_MOTION || ACCEPT_ASTROM) { output[0].average[Nave].dR = input[0].average[N].dR; output[0].average[Nave].dD = input[0].average[N].dD; @@ -471,4 +476,8 @@ output[0].average[Nave].dP = input[0].average[N].dP; output[0].average[Nave].Tmean = input[0].average[N].Tmean; + } + if (ACCEPT_ASTROM) { + output[0].average[Nave].R = input[0].average[N].R; + output[0].average[Nave].D = input[0].average[N].D; } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/Makefile (revision 41170) @@ -42,5 +42,5 @@ $(SRC)/bDrawObjects.$(ARCH).o $(SRC)/bDrawFrame.$(ARCH).o \ $(SRC)/bDrawLabels.$(ARCH).o $(SRC)/bDrawIt.$(ARCH).o \ -$(SRC)/bDrawImage.$(ARCH).o \ +$(SRC)/bDrawImage.$(ARCH).o $(SRC)/MemoryDump.$(ARCH).o \ $(SRC)/PNGit.$(ARCH).o $(SRC)/PPMit.$(ARCH).o \ $(SRC)/PSit.$(ARCH).o $(SRC)/CrossHairs.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/doc/outline.txt (revision 41170) @@ -0,0 +1,38 @@ + +This document outlines the way kapa works. + +Top-Level (kapa.c): + + * load arguments + * generate basic graphic structure (SetUpGraphic) + * this is held static by SetUpGraphic.c + * it currently does not carry all elements: sections are static elsewhere + * Graphic elements are a bit X-centric. + * generate basic window elements (InitLayout / Layout.c) + * run event loop + +Event Loop: + + * check pipe for messages + * in X context, check for various events (cursor interations, resizing, moving, raising) + +In X context, events which expose or reconfigure the window trigger Refresh(). + +In non-X context: + * PNGit & JPEGit trigger bDrawIt (equivalent to Refresh) + * PSit is equivalent to bDrawIt + +------ + +I would like to change from native X to native bDraw functions. This +would allow for opacity and anti-aliasing. To do this, I need to do +the following: + +* bDraw operations need to include a mask so we can distinguish layers + (e.g., text on image). anti-aliasing needs to be applied to text + and graph elements, but not to the images + +* Refresh currently calls DrawImage with generates both the main image + as well as the extra elements: colorbar, zoom window, pan window, + buttons. + Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/prototypes.h (revision 41170) @@ -3,7 +3,13 @@ int args PROTO((int *argc, char **argv)); void SetUpGraphic PROTO((int *argc, char **argv)); +void FreeGraphic PROTO((void)); void DefineLayout PROTO((int, char **)); int EventLoop PROTO((void)); void CloseDisplay PROTO((void)); +int MemoryDump PROTO((int sock)); +int MemoryDumpLines PROTO((int sock)); +int MemoryDumpOnExit PROTO((int sock)); +int MemoryDumpSetOnExit PROTO((int state)); +int MemoryDumpAndExit PROTO((void)); /* SetUpGraphic */ @@ -33,4 +39,6 @@ void DrawTextlines PROTO((KapaGraphWidget *graph)); void DrawConnect PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); +void DrawPolygon PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); +void DrawPolyfill PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); void DrawHistogram PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); int DrawObjectN PROTO((Graphic *graphic, KapaGraphWidget *graph, Gobjects *objects)); @@ -85,4 +93,5 @@ int SetColormapFromPipe PROTO((int sock)); int SetNanColorFromPipe PROTO((int sock)); +int SetSmoothSigma PROTO((int sock)); int LoadVectorData PROTO((int sock, KapaGraphWidget *graph, int N, char *type)); @@ -109,4 +118,5 @@ void InitLayout PROTO((int argc, char **argv)); +void FreeLayout PROTO((void)); /* PS drawing utilities */ @@ -142,4 +152,6 @@ void bDrawBars PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object, int mode)); void bDrawPoints PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); +void bDrawPolygons PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); +void bDrawFillPolygons PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawXErrors PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); void bDrawYErrors PROTO((bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object)); @@ -179,4 +191,5 @@ void InitButtonFunc PROTO((Button *button, int (*function)(Graphic *graphic, KapaImageWidget *image))); void DrawImage PROTO((KapaImageWidget *image)); +void DrawImageTool PROTO((KapaImageWidget *image)); void DrawButton PROTO((Graphic *graphic, Button *button)); void DrawBitmap PROTO((Graphic *graphic, int x, int y, int dx, int dy, unsigned char *bitmap, int mode)); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/include/structures.h (revision 41170) @@ -48,4 +48,6 @@ unsigned long back; // basic background color + float smooth_sigma; // anti-aliasing smoothing scale + unsigned long overlay_color[NOVERLAYS]; // image plotting colors } Graphic; @@ -131,9 +133,23 @@ } Overlay; -typedef struct { - float *x, *y, *z, *dxp, *dxm, *dyp, *dym; - int Npts; - int style, ptype, ltype, color, etype, ebar; - double lweight, size; +// a set of objects all have the same basic properties +typedef struct { + float *x; // x-coordinates of the points + float *y; // y-coordinates of the points + float *z; // size/colorscale for points + float *dxp; // lower-errorbar in x + float *dxm; // upper-errorbar in x + float *dyp; // lower-errorbar in y + float *dym; // upper-errorbar in y + int Npts; // number of points in this set + int style; // how are the object draw: CONNECT, HISTOGRAM, + int ptype; // shape of object at each point + // ptype is overloaded for NPOLYGON to be the number of points / polygon + int ltype; // style of line (solid, dot, dash, etc) + int color; // color for point (if not colorscaled) + int etype; // errorbars to draw (0x01 = y, 0x02 = x) + int ebar; // draw a cap on the error bar + double lweight; // line thickness + double size; // size of the object double x0, x1, y0, y1; /* limits for this object */ double alpha; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/CheckPipe.c (revision 41170) @@ -352,4 +352,28 @@ } + if (!strcmp (word, "SIGM")) { + SetSmoothSigma (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEMD")) { + MemoryDump (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEML")) { + MemoryDumpLines (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + + if (!strcmp (word, "MEMX")) { + MemoryDumpOnExit (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (TRUE); + } + fprintf (stderr, "unknown signal %s\n", word); KiiSendCommand (sock, 4, "DONE"); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/DrawObjects.c (revision 41170) @@ -97,26 +97,26 @@ switch (object[0].style) { case KAPA_PLOT_CONNECT: - // fprintf (stderr, "plot KAPA_PLOT_CONNECT: \n"); DrawConnect (graphic, graph, object); break; + case KAPA_PLOT_POLYGON: + DrawPolygon (graphic, graph, object); + break; + case KAPA_PLOT_POLYFILL: + DrawPolyfill (graphic, graph, object); + break; case KAPA_PLOT_HISTOGRAM: - // fprintf (stderr, "plot KAPA_PLOT_HISTOGRAM:\n"); DrawHistogram (graphic, graph, object); break; case KAPA_PLOT_BARS_SOLID: - // fprintf (stderr, "plot KAPA_PLOT_BARS_SOLID:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_SOLID); break; case KAPA_PLOT_BARS_OUTLINE: - // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTLINE:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTLINE); break; case KAPA_PLOT_BARS_OUTFILL: - // fprintf (stderr, "plot KAPA_PLOT_BARS_OUTFILL:\n"); DrawBars (graphic, graph, object, KAPA_PLOT_BARS_OUTFILL); break; case KAPA_PLOT_POINTS: default: - // fprintf (stderr, "plot KAPA_PLOT_POINTS:\n"); DrawPoints (graphic, graph, object); break; @@ -172,5 +172,137 @@ sx0 = sx1; sy0 = sy1; } - +} + +/******/ +void DrawPolygon (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0, sx1, sy1; + double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx + XCENTER; + sy0 = x[k]*myi + y[k]*myj + by + YCENTER; + + // last point connects to first + if (j == object[0].ptype - 1) { + sx1 = x[i]*mxi + y[i]*mxj + bx + XCENTER; + sy1 = x[i]*myi + y[i]*myj + by + YCENTER; + } else { + sx1 = x[k+1]*mxi + y[k+1]*mxj + bx + XCENTER; + sy1 = x[k+1]*myi + y[k+1]*myj + by + YCENTER; + } + ClipLine (graphic, sx0, sy0, sx1, sy1, X0, Y0, X1, Y1); + } + } +} + +/******/ +void DrawPolyfill (Graphic *graphic, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y, *z; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0; + // double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + /* + window boundary so we can clip objects + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + */ + + x = object[0].x; y = object[0].y; z = object[0].z; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + ALLOCATE_PTR (points, XPoint, object[0].ptype); + + // NOTE that LoadObject.c:45 limits the allow + // object styles which may have negative (scaled) colors + int scaleColor = (object[0].color < 0); + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx + XCENTER; + sy0 = x[k]*myi + y[k]*myj + by + YCENTER; + + points[j].x = sx0; + points[j].y = sy0; + } + + if (scaleColor) { + if (!finite(z[i])) continue; + int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); + XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); + } + XFillPolygon (graphic->display, graphic->window, graphic->gc, points, object[0].ptype, Convex, CoordModeOrigin); + } + + free (points); } @@ -580,4 +712,7 @@ int scaleColor = (object[0].color < 0); + // NOTE that LoadObject.c:45 limits the allow + // object styles which may have negative (scaled) colors + ds = 0.5 * (graphic->dx + graphic->dy) * 0.003 * object[0].size; dz = 0.5 * (graphic->dx + graphic->dy) * 0.010; @@ -916,8 +1051,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dxp[i]))) goto skip_dxp; if (D > fabs(dxp[i]*mxi)) goto skip_dxp; @@ -999,8 +1134,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dyp[i]))) goto skip_dyp; if (D > fabs(dyp[i]*myj)) goto skip_dyp; @@ -1023,6 +1158,6 @@ skip_dyp: if (!(finite(x[i]) && finite(y[i]) && finite(dym[i]))) continue; - if (object[0].ptype == 5) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } if (D > fabs(dym[i]*myj)) continue; sx0 = x[i]*mxi + y[i]*mxj + bx + XCENTER; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Image.c (revision 41170) @@ -148,4 +148,21 @@ void DrawImage (KapaImageWidget *image) { + Graphic *graphic; + + if (image == NULL) return; + + graphic = GetGraphic (); + + if (image[0].picture.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].picture.pix, 0, 0, + image[0].picture.x + 1, image[0].picture.y + 1, + image[0].picture.dx, image[0].picture.dy); + } +} + +// add the zoom, pan, crosshairs, status box, buttons +void DrawImageTool (KapaImageWidget *image) { + int i; Graphic *graphic; @@ -160,17 +177,4 @@ image[0].picture.dx+1, image[0].picture.dy+1); - if (image[0].picture.pix) { - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].picture.pix, 0, 0, - image[0].picture.x + 1, image[0].picture.y + 1, - image[0].picture.dx, image[0].picture.dy); - } - - for (i = 0; i < NOVERLAYS; i++) { - if (image[0].overlay[i].active) { - PaintOverlay (graphic, image, i); - } - } - if (image[0].location) { if (image[0].cmapbar.pix) { @@ -211,6 +215,4 @@ StatusBox (graphic, image); } - - FlushDisplay (); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/InterpretKeys.c (revision 41170) @@ -160,4 +160,6 @@ break; + case XK_plus: + case XK_equal: case XK_KP_Add: if (modstate & ControlMask) { @@ -171,4 +173,7 @@ Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; + + case XK_minus: + case XK_underscore: case XK_KP_Subtract: if (modstate & ControlMask) { Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/JPEGit24.c (revision 41170) @@ -212,4 +212,5 @@ } bDrawBufferFree (buffer); + free (palette); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Layout.c (revision 41170) @@ -45,2 +45,7 @@ AddSection ("default", 0.0, 0.0, 1.0, 1.0, -1); } + +void FreeLayout (void) { + FreeSections (); + FreeRotFonts (); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadFrame.c (revision 41170) @@ -93,20 +93,4 @@ SetSectionSizes (section); Refresh(); - // if (USE_XWINDOW) DrawFrame (graph); - // FlushDisplay (); - - /* XXX why did I do this??? */ -# if (0) - status = TRUE; - if (status) { - for (i = 0; i < Nsection; i++) { - PositionPicture (§ion[i]); - } - if (USE_XWINDOW) XClearWindow (graphic.display, graphic.window); - Refresh (); - } else { - FlushDisplay (); - } -# endif return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadLabels.c (revision 41170) @@ -26,4 +26,6 @@ label[Nbytes] = 0; + FREE (label); + c = GetRotFont (&size); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/LoadObject.c (revision 41170) @@ -42,5 +42,11 @@ graph[0].objects[N].color = KapaColormapSize() - 1; } - if ((graph[0].objects[N].style != KAPA_PLOT_POINTS) && (graph[0].objects[N].color < 0)) { + + // XXX watch out for this restriction in DrawObjects / bDrawObjects + int canScaleColor = FALSE; + canScaleColor |= (graph[0].objects[N].style == KAPA_PLOT_POINTS); + canScaleColor |= (graph[0].objects[N].style == KAPA_PLOT_POLYFILL); + + if (!canScaleColor && (graph[0].objects[N].color < 0)) { graph[0].objects[N].color = 0; } @@ -103,8 +109,13 @@ if (USE_XWINDOW) { - Graphic *graphic = GetGraphic(); - DrawObjectN (graphic, graph, &graph[0].objects[graph[0].Nobjects-1]); - } - FlushDisplay (); + if (0) { + // use this if we are not using buffered plotting + Graphic *graphic = GetGraphic(); + DrawObjectN (graphic, graph, &graph[0].objects[graph[0].Nobjects-1]); + FlushDisplay (); + } else { + Refresh (); + } + } return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/MemoryDump.c (revision 41170) @@ -0,0 +1,41 @@ +# include "Ximage.h" + +static int DumpOnExit = FALSE; + +int MemoryDump (int sock) { + OHANA_UNUSED_PARAM(sock); + + ohana_memdump_file (stderr, TRUE); + + return TRUE; +} + +int MemoryDumpLines (int sock) { + + int Nlines; + KiiScanMessage (sock, "%d", &Nlines); + + ohana_memdump_set_maxlines (Nlines); + + return TRUE; +} + +int MemoryDumpOnExit (int sock) { + OHANA_UNUSED_PARAM(sock); + + int state; + KiiScanMessage (sock, "%d", &state); + + MemoryDumpSetOnExit (state); + return TRUE; +} + +int MemoryDumpSetOnExit (int state) { + DumpOnExit = state; + return TRUE; +} + +int MemoryDumpAndExit (void) { + if (DumpOnExit) MemoryDump(0); + exit (0); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PNGit.c (revision 41170) @@ -98,4 +98,9 @@ png_write_info (png_ptr, info_ptr); + if (graphic->smooth_sigma == 0.0) { + fprintf (stderr, "making PNG without antialias smoothing\n"); + fprintf (stderr, "use 'antialias (value)' to set smoothing scale (0.4 - 0.7 recommended)\n"); + } + if (haveImage) { buffer = bDrawIt (palette, Npalette, 3); @@ -109,4 +114,6 @@ bDrawBufferFree (buffer); + free (palette); + fclose (f); return (TRUE); Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/PPMit.c (revision 41170) @@ -46,4 +46,6 @@ bDrawBufferFree (buffer); + free (palette); + return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/Refresh.c (revision 41170) @@ -1,5 +1,68 @@ # include "Ximage.h" +# define USE_BUFFERED_DRAW 1 + +void bDrawXimage (bDrawBuffer *buffer); +void Refresh_Buffered (void); +void Refresh_Unbuffered (void); + void Refresh (void) { + if (USE_BUFFERED_DRAW) { + Refresh_Buffered(); + } else { + Refresh_Unbuffered(); + } +} + +void Refresh_Buffered (void) { + + int Npalette; + Graphic *graphic; + + if (!USE_XWINDOW) return; + // if (HAVE_BACKING) return; + + graphic = GetGraphic(); + + // limit the png window to the min needed to contain the active graphic regions + SectionMinBoundary (graphic); + + // is the palette reasonable in modern context? + png_color *palette = KapaPNGPalette (&Npalette); + + bDrawBuffer *buffer = bDrawIt (palette, Npalette, 3); + + /* XClearWindow (graphic.display, graphic.window); */ + XSetForeground (graphic->display, graphic->gc, graphic->back); + XFillRectangle (graphic->display, graphic->window, graphic->gc, 0, 0, graphic->dx, graphic->dy); + XSetForeground (graphic->display, graphic->gc, graphic->fore); + + // copy buffer to Xwindow as image? + bDrawXimage (buffer); + bDrawBufferFree (buffer); + free (palette); + + // draw image tool for all sections + int Nsection = GetNumberOfSections (); + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + + KapaImageWidget *image = section->image; + DrawImageTool (image); + + /*** PaintOverlay is called in DrawImage *** + if (!image) continue; + for (int j = 0; j < NOVERLAYS; j++) { + if (image[0].overlay[j].active) { + PaintOverlay (graphic, image, j); + } + } + */ + } + + FlushDisplay (); +} + +void Refresh_Unbuffered (void) { int i, Nsection; @@ -20,8 +83,21 @@ Nsection = GetNumberOfSections (); for (i = 0; i < Nsection; i++) { - section = GetSectionByNumber (i); - DrawSectionBG (graphic, section); - DrawImage (section->image); - DrawGraph (section->graph); + section = GetSectionByNumber (i); + DrawSectionBG (graphic, section); + + KapaImageWidget *image = section->image; + DrawImage (image); + DrawImageTool (image); + + /*** the overlay is added in DrawImage ***/ + if (image) { + for (int j = 0; j < NOVERLAYS; j++) { + if (image[0].overlay[j].active) { + PaintOverlay (graphic, image, j); + } + } + } + + DrawGraph (section->graph); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetColormap.c (revision 41170) @@ -72,5 +72,5 @@ } } - fprintf (stderr, "ruff Npix: %d vs %d\n", i, graphic[0].Npixels); + // fprintf (stderr, "ruff Npix: %d vs %d\n", i, graphic[0].Npixels); // all other modes are 1D: set the flag: @@ -217,4 +217,5 @@ !strncmp (graphic->colormapName, "lgcy:", 5) || !strncmp (graphic->colormapName, "cetf:", 5) || + !strncmp (graphic->colormapName, "cetr:", 5) || !strncmp (graphic->colormapName, "csvf:", 5)) { @@ -233,9 +234,10 @@ int isCSV = !strncmp (graphic->colormapName, "csvf:", 5); int isCET = !strncmp (graphic->colormapName, "cetf:", 5); + int isCETRev = !strncmp (graphic->colormapName, "cetr:", 5); int isLegacy = !strncmp (graphic->colormapName, "lgcy:", 5); float fracIndex, fracRed, fracBlue, fracGreen; - if (isCET) fracIndex = 0.0; + if (isCET || isCETRev) fracIndex = 0.0; while (scan_line_maxlen (f, line, 1024) != EOF) { @@ -247,5 +249,5 @@ if (Nscan != 4) continue; } - if (isCET) { + if (isCET || isCETRev) { Nscan = sscanf (line, "%f,%f,%f", &fracRed, &fracGreen, &fracBlue); fracIndex += 1.0 / 256.0; @@ -256,5 +258,5 @@ if (Nscan != 4) continue; } - if (!isCSV && !isLegacy && !isCET) { + if (!isCSV && !isLegacy && !isCET && !isCETRev) { Nscan = sscanf (line, "%f %f %f %f", &fracIndex, &fracRed, &fracGreen, &fracBlue); if (Nscan != 4) continue; @@ -299,4 +301,16 @@ } } + + // reverse the color sequence: + if (isCETRev) { + for (i = 0; i < MaxValue / 2; i++) { + unsigned short tmp; + // fprintf (stderr, "swap: %d %d\n", graphic[0].cmap[i].red, graphic[0].cmap[MaxValue - 1 - i].red); + tmp = graphic[0].cmap[i].red; graphic[0].cmap[i].red = graphic[0].cmap[MaxValue - 1 - i].red; graphic[0].cmap[MaxValue - 1 - i].red = tmp; + tmp = graphic[0].cmap[i].blue; graphic[0].cmap[i].blue = graphic[0].cmap[MaxValue - 1 - i].blue; graphic[0].cmap[MaxValue - 1 - i].blue = tmp; + tmp = graphic[0].cmap[i].green; graphic[0].cmap[i].green = graphic[0].cmap[MaxValue - 1 - i].green; graphic[0].cmap[MaxValue - 1 - i].green = tmp; + } + } + goto store_colors; } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/SetUpGraphic.c (revision 41170) @@ -32,4 +32,6 @@ graphic->pixels = NULL; + graphic->smooth_sigma = 0.0; + if (!USE_XWINDOW) { ALLOCATE (graphic[0].pixels, unsigned long, NPIXELS_STATIC); @@ -50,5 +52,4 @@ CheckVisual (graphic, argc, argv); CheckColors (graphic, argc, argv); - icon.width = icon_width; @@ -84,2 +85,22 @@ } +int SetSmoothSigma (int sock) { + + float sigma; + KiiScanMessage (sock, "%f", &sigma); + + if (isfinite(sigma)) { + if ((sigma <= 1.1) && (sigma >= 0.0)) { + graphic->smooth_sigma = sigma; + } + } + return TRUE; +} + +void FreeGraphic () { + free (graphic->pixels); + free (graphic->cmap); + free (graphic->color); + free (graphic->colormapName); + free (graphic); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/args.c (revision 41170) @@ -23,4 +23,9 @@ NAME_WINDOW = strcreate (argv[N]); remove_argument(N, argc, argv); + } + + if ((N = get_argument (*argc, argv, "-memdump"))) { + remove_argument(N, argc, argv); + MemoryDumpSetOnExit (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawImage.c (revision 41170) @@ -1,3 +1,6 @@ # include "Ximage.h" + +// XXX for the moment, this function does NOT set the mask bits. +// since we lay graphics on top of images, this is OK for now # define WHITE_R 255 @@ -54,10 +57,18 @@ // the created buffer is supposed to contain the output windows + if (Xs < 0) { + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); + abort(); + } if (buffer[0].Nx < Xs + dx) { - fprintf (stderr, "invalid condition\n"); + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); + abort(); + } + if (Ys < 0) { + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); abort(); } if (buffer[0].Ny < Ys + dy) { - fprintf (stderr, "invalid condition\n"); + fprintf (stderr, "image display boundaries out of range of window (invalid condition) : fix Kapa\n"); abort(); } @@ -153,4 +164,27 @@ return (TRUE); +} + +void bDrawXimage (bDrawBuffer *buffer) { + + Graphic *graphic = GetGraphic (); + + ALLOCATE_PTR (data, char, 4*buffer->Nx*buffer->Ny); + + for (int iy = 0; iy < buffer->Ny; iy++) { + for (int ix = 0; ix < buffer->Nx; ix++) { + data[4*(iy*buffer->Nx + ix) + 0] = buffer->pixels[iy][3*ix + 2]; + data[4*(iy*buffer->Nx + ix) + 1] = buffer->pixels[iy][3*ix + 1]; + data[4*(iy*buffer->Nx + ix) + 2] = buffer->pixels[iy][3*ix + 0]; + data[4*(iy*buffer->Nx + ix) + 3] = 0; + } + } + + XImage *pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, + data, buffer->Nx, buffer->Ny, 32, 0); + + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, pix, 0, 0, 1, 1, buffer->Nx, buffer->Ny); + + free (data); } @@ -186,4 +220,5 @@ } bDrawBufferFree (buffer); + free (palette); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawIt.c (revision 41170) @@ -3,31 +3,47 @@ bDrawBuffer *bDrawIt (png_color *palette, int Npalette, int Nbyte) { - int i, j, Nsection; - bDrawBuffer *buffer; - bDrawColor black; - Graphic *graphic; - Section *section; + Graphic *graphic = GetGraphic(); + bDrawColor black = KapaColorByName ("black"); - graphic = GetGraphic(); + // get the number of sections + int Nsection = GetNumberOfSections (); - black = KapaColorByName ("black"); + // in order to allow the anti-aliasing to affect the text & graphs but not the images + // I need to generate the images in one buffer and the graphs in a second buffer + // then merge the two buffers. - // if we want to trim, we'll need to carry about the start in graphic coords and - // the dx,dy size. - buffer = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); - bDrawSetStyle (buffer, black, 0, 0, 1.0); + // base will hold the images + bDrawBuffer *base = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); + bDrawSetStyle (base, black, 0, 0, 1.0); - // reset the sizes for all sections - Nsection = GetNumberOfSections (); - for (i = 0; i < Nsection; i++) { - section = GetSectionByNumber (i); - bDrawImage (buffer, section->image, graphic); - for (j = 0; section->image && (j < NOVERLAYS); j++) { - if (section->image->overlay[j].active) bDrawOverlay (buffer, section->image, j); - } - bDrawGraph (buffer, section->graph); + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + bDrawImage (base, section->image, graphic); } - return (buffer); + // graph will hold the graphic overlay + bDrawBuffer *graph = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); + bDrawSetStyle (graph, black, 0, 0, 1.0); + + for (int i = 0; i < Nsection; i++) { + Section *section = GetSectionByNumber (i); + for (int j = 0; section->image && (j < NOVERLAYS); j++) { + if (section->image->overlay[j].active) bDrawOverlay (graph, section->image, j); + } + bDrawGraph (graph, section->graph); + } + + // apply anti-aliasing only to the graph + if (graphic->smooth_sigma > 0.0) { + // anything > 1.1 blurs the image too much + graphic->smooth_sigma = MIN (graphic->smooth_sigma, 1.1); + bDrawSmooth (graph, graphic->smooth_sigma); + } + + // place graph on base + bDrawMerge (base, graph); + bDrawBufferFree (graph); + + return (base); } Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawObjects.c (revision 41170) @@ -9,4 +9,6 @@ # define OpenTriangle(BUF,X1,Y1,X2,Y2,X3,Y3) (bDrawTriOpen (BUF, (X1), (Y1), (X2), (Y2), (X3), (Y3))) +// I should not have a local static variable for this: +// I should just pass the graphic structure to the called functions below static Graphic *graphic; @@ -35,4 +37,10 @@ case KAPA_PLOT_CONNECT: bDrawConnect (buffer, graph, object); + break; + case KAPA_PLOT_POLYGON: + bDrawPolygons (buffer, graph, object); + break; + case KAPA_PLOT_POLYFILL: + bDrawFillPolygons (buffer, graph, object); break; case KAPA_PLOT_HISTOGRAM: @@ -103,4 +111,153 @@ sx0 = sx1; sy0 = sy1; } +} + +void bDrawPolygons (bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0, sx1, sy1; + double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx; + sy0 = x[k]*myi + y[k]*myj + by; + + // last point connects to first + if (j == object[0].ptype - 1) { + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + } else { + sx1 = x[k+1]*mxi + y[k+1]*mxj + bx; + sy1 = x[k+1]*myi + y[k+1]*myj + by; + } + bDrawClipLine (buffer, sx0, sy0, sx1, sy1, X0, Y0, X1, Y1); + } + } +} + +void bDrawFillPolygons (bDrawBuffer *buffer, KapaGraphWidget *graph, Gobjects *object) { + + float *x, *y; + double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; + double sx0, sy0; + // double X0, X1, Y0, Y1; + + mxi = graph[0].axis[0].dfx / (object[0].x1 - object[0].x0); + mxj = graph[0].axis[1].dfx / (object[0].y1 - object[0].y0); + myi = graph[0].axis[0].dfy / (object[0].x1 - object[0].x0); + myj = graph[0].axis[1].dfy / (object[0].y1 - object[0].y0); + + bxi = graph[0].axis[0].fx - object[0].x0*graph[0].axis[0].dfx/(object[0].x1 - object[0].x0); + bxj = -object[0].y0*graph[0].axis[1].dfx/(object[0].y1 - object[0].y0); + byi = -object[0].x0*graph[0].axis[0].dfy/(object[0].x1 - object[0].x0); + byj = graph[0].axis[1].fy - object[0].y0*graph[0].axis[1].dfy/(object[0].y1 - object[0].y0); + + bx = bxi + bxj; + by = byi + byj; + + /* + window boundary so we can clip objects + X0 = graph[0].axis[0].fx; + X1 = graph[0].axis[0].fx + graph[0].axis[0].dfx; + Y0 = graph[0].axis[1].fy; + Y1 = graph[0].axis[1].fy + graph[0].axis[1].dfy; + */ + + x = object[0].x; y = object[0].y; + + // ptype must > 2 + // Npts % ptype must be 0 + // who must validate that? + + ALLOCATE_PTR (xpts, double, object[0].ptype); + ALLOCATE_PTR (ypts, double, object[0].ptype); + + int scaleColor = (object[0].color < 0); + unsigned char *pixel1, *pixel2, *pixel3; + float *z = object[0].z; + + if (scaleColor) { + // scaled colors use the colormap defined for the graphic + ALLOCATE (pixel1, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel2, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel3, unsigned char, graphic[0].Npixels); + + /** cmap[i].pixel must be defined even if X is not used **/ + for (int i = 0; i < graphic[0].Npixels; i++) { /* set up pixel array */ + pixel1[i] = graphic[0].cmap[i].red >> 8; + pixel2[i] = graphic[0].cmap[i].green >> 8; + pixel3[i] = graphic[0].cmap[i].blue >> 8; + } + + } + + // each polygon is made of (N = ptype) points + // we connect each point and the last one back + for (int i = 0; i < object[0].Npts; i += object[0].ptype) { + // first check for any invalid values for this polygon + int skipObject = FALSE; + for (int j = 0; (j < object[0].ptype) && !skipObject; j++) { + int k = i + j; + if (!(finite(x[k]) && finite(y[k]))) skipObject = TRUE; + } + if (skipObject) continue; + + for (int j = 0; (j < object[0].ptype); j++) { + int k = i + j; + sx0 = x[k]*mxi + y[k]*mxj + bx; + sy0 = x[k]*myi + y[k]*myj + by; + + xpts[j] = sx0; + ypts[j] = sy0; + } + + if (scaleColor) { + if (!finite(z[i])) continue; + int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); + buffer->bColor_R = pixel1[pixel]; + buffer->bColor_G = pixel2[pixel]; + buffer->bColor_B = pixel3[pixel]; + } + bDrawPolyFill (buffer, xpts, ypts, object[0].ptype); + } + free (xpts); + free (ypts); } @@ -886,8 +1043,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.66*dz*z[i] : 0.66*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dxp[i]))) goto skip_dxp; if (D > fabs(dxp[i]*mxi)) goto skip_dxp; @@ -969,8 +1126,8 @@ // for open circles, only go to the outer radius D = 0; - if (object[0].ptype == 7) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 1) { D = scaleSize ? dz*z[i] : ds; } - if (object[0].ptype == 5) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_CIRCLE_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_BOX_OPEN ) { D = scaleSize ? dz*z[i] : ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } if (!(finite(x[i]) && finite(y[i]) && finite(dyp[i]))) goto skip_dyp; if (D > fabs(dyp[i]*myj)) goto skip_dyp; @@ -993,6 +1150,6 @@ skip_dyp: if (!(finite(x[i]) && finite(y[i]) && finite(dym[i]))) continue; - if (object[0].ptype == 5) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } - if (object[0].ptype == 15) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN ) { D = scaleSize ? 0.58*dz*z[i] : 0.58*ds; } + if (object[0].ptype == KAPA_POINT_TRIANGLE_OPEN_DOWN) { D = scaleSize ? 1.15*dz*z[i] : 1.15*ds; } if (D > fabs(dym[i]*myj)) continue; sx0 = x[i]*mxi + y[i]*mxj + bx; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/bDrawOverlay.c (revision 41170) @@ -24,8 +24,13 @@ } - Xmin = 0; - Ymin = 0; - Xmax = image[0].picture.dx; - Ymax = image[0].picture.dy; + // Xmin = 0; + // Ymin = 0; + // Xmax = image[0].picture.dx; + // Ymax = image[0].picture.dy; + + Xmin = image[0].picture.x; + Ymin = image[0].picture.y; + Xmax = image[0].picture.x + image[0].picture.dx; // maybe this should be just dx? + Ymax = image[0].picture.y + image[0].picture.dy; if (N == INFRONT) { @@ -35,5 +40,7 @@ for (i = 0; i < image[0].overlay[N].Nobjects; i++) { - Image_to_Picture (&X, &Y, image[0].overlay[N].objects[i].x, image[0].overlay[N].objects[i].y, &image[0].picture); + // XXX the 0.5,0.5 offset is apparently needed here. + // work on rationalizing these functions in the context of their different plotting types + Image_to_Screen (&X, &Y, image[0].overlay[N].objects[i].x - 0.5, image[0].overlay[N].objects[i].y - 0.5, &image[0].picture); dX = image[0].overlay[N].objects[i].dx * expand; dY = image[0].overlay[N].objects[i].dy * expand; Index: branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/kapa2/src/kapa.c (revision 41170) @@ -11,4 +11,10 @@ CloseDisplay (); - exit (0); + + // free things + FreeLayout(); + FreeGraphic(); + FREE (NAME_WINDOW); + + MemoryDumpAndExit (); } Index: branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libdvo/src/dvosorts.c (revision 41170) @@ -76,5 +76,6 @@ } -/* sort a coordinate pair (X,Y) and the associated index (S) */ +/* sort the index of a coordinate pair (X,Y) on X (vector pair stays unsorted) */ +// XXX isn't this function equivalent to dsort_indexonly? void sort_coords_indexonly (double *X, double *Y, off_t *S, off_t N) { OHANA_UNUSED_PARAM(Y); Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/include/kapa.h (revision 41170) @@ -45,5 +45,7 @@ KAPA_PLOT_BARS_OUTLINE = 4, KAPA_PLOT_BARS_OUTFILL = 5, - KAPA_PLOT_INVALID_MAX = 6, + KAPA_PLOT_POLYGON = 6, + KAPA_PLOT_POLYFILL = 7, + KAPA_PLOT_INVALID_MAX = 8, } KapaPlotStyle; @@ -172,6 +174,8 @@ int Nx, Ny, Nbyte; bDrawColor **pixels; + char **mask; png_color *palette; int Npalette; + // current drawing values: int bWeight; @@ -256,4 +260,8 @@ int KapaGetImageData (int fd, KapaImageData *graphmode); int KapaSetToolbox (int fd, int location); +int KapaSetSmoothSigma (int fd, float sigma); +int KapaMemoryDump (int fd); +int KapaMemoryDumpLines (int fd, int Nlines); +int KapaMemoryDumpOnExit (int fd, int state); /* KapaColors */ @@ -272,4 +280,5 @@ /* RotFont.c */ void InitRotFonts PROTO((void)); +void FreeRotFonts PROTO((void)); int SetRotFont PROTO((char *name, int size)); char *GetRotFont PROTO((int *size)); @@ -292,4 +301,5 @@ bDrawBuffer *bDrawBufferCreate (int Nx, int Ny, int Nbyte, png_color *palette, int Npalette); void bDrawBufferFree (bDrawBuffer *buffer); +int bDrawMerge (bDrawBuffer *base, bDrawBuffer *layer); void bDrawSetBuffer (bDrawBuffer *buffer); void bDrawSetColor (bDrawBuffer *buffer, bDrawColor color); @@ -312,4 +322,7 @@ void bDrawTriOpen (bDrawBuffer *buffer, double x1, double y1, double x2, double y2, double x3, double y3); void bDrawTriFill (bDrawBuffer *buffer, double x1, double y1, double dx, double dy); +void bDrawPolyFill (bDrawBuffer *buffer, double *x, double *y, int Npoints); + +void bDrawSmooth (bDrawBuffer *buffer, float sigma); /* bDrawRotFont.c */ Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaStyles.c (revision 41170) @@ -54,4 +54,7 @@ if (!strcasecmp (string, "outline")) return KAPA_PLOT_BARS_OUTLINE; if (!strcasecmp (string, "outfill")) return KAPA_PLOT_BARS_OUTFILL; + + if (!strcasecmp (string, "polygon")) return KAPA_PLOT_POLYGON; + if (!strcasecmp (string, "polyfill")) return KAPA_PLOT_POLYFILL; if (strlen(string) > 2) { Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/KapaWindow.c (revision 41170) @@ -455,2 +455,33 @@ } +int KapaSetSmoothSigma (int fd, float sigma) { + + KiiSendCommand (fd, 4, "SIGM"); + KiiSendMessage (fd, "%4.1f ", sigma); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDump (int fd) { + + KiiSendCommand (fd, 4, "MEMD"); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDumpLines (int fd, int Nlines) { + + KiiSendCommand (fd, 4, "MEML"); + KiiSendMessage (fd, "%d ", Nlines); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KapaMemoryDumpOnExit (int fd, int state) { + + KiiSendCommand (fd, 4, "MEMX"); + KiiSendMessage (fd, "%d ", state); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/RotFont.c (revision 41170) @@ -32,4 +32,10 @@ strncpy (currentname, RotFonts[DEFFONT].name, 63); currentname[63] = 0; currentsize = RotFonts[DEFFONT].size; +} + +void FreeRotFonts (void) { + free (RotFonts); + RotFonts = FALSE; + RotFontInited = FALSE; } Index: branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libkapa/src/bDrawFuncs.c (revision 41170) @@ -1,14 +1,34 @@ # include -// move these to the bDrawBuffer type -// static int bWeight; -// static int bType; -// static bDrawColor bColor; -// static bDrawColor bColor_R; -// static bDrawColor bColor_G; -// static bDrawColor bColor_B; -// static bDrawBuffer *bBuffer; +// buffer->pixels carries the plot image +// buffer->mask is 0 if the pixel is untouched, 1 if the pixel has data void bDrawCircleSingle (bDrawBuffer *buffer, double xc, double yc, double radius); + +int bDrawMerge (bDrawBuffer *base, bDrawBuffer *layer) { + + // the two bDrawBuffers must match in size and depth + + if (base->Nx != layer->Nx) return FALSE; + if (base->Ny != layer->Ny) return FALSE; + if (base->Nbyte != layer->Nbyte) return FALSE; + + for (int j = 0; j < base->Ny; j++) { + for (int i = 0; i < base->Nx; i++) { + + if (!layer->mask[j][i]) continue; + + // completely opaque top layer: + if (base->Nbyte == 1) { + base->pixels[j][i] = layer->pixels[j][i]; + } else { + base->pixels[j][3*i+0] = layer->pixels[j][3*i+0]; + base->pixels[j][3*i+1] = layer->pixels[j][3*i+1]; + base->pixels[j][3*i+2] = layer->pixels[j][3*i+2]; + } + } + } + return TRUE; +} // create a drawing buffer with either 1 or 3 byte colors @@ -35,6 +55,8 @@ ALLOCATE (buffer[0].pixels, bDrawColor *, Ny); + ALLOCATE (buffer[0].mask, char *, Ny); for (i = 0; i < Ny; i++) { ALLOCATE (buffer[0].pixels[i], bDrawColor, Nbyte*Nx); + ALLOCATE (buffer[0].mask[i], char, Nx); for (j = 0; j < Nx; j++) { if (Nbyte == 1) { @@ -45,4 +67,5 @@ buffer[0].pixels[i][3*j+2] = white_B; } + buffer[0].mask[i][j] = 0; // for now: 0 = no data, 1 = data } } @@ -63,7 +86,9 @@ for (i = 0; i < buffer[0].Ny; i++) { free (buffer[0].pixels[i]); + free (buffer[0].mask[i]); } free (buffer[0].pixels); - free (buffer[0].palette); + free (buffer[0].mask); + // free (buffer[0].palette); free (buffer); return; @@ -121,11 +146,9 @@ buffer[0].pixels[y][x] = buffer->bColor; } else { - // buffer[0].pixels[y][3*x+0] = buffer->bColor_R; - // buffer[0].pixels[y][3*x+1] = buffer->bColor_G; - // buffer[0].pixels[y][3*x+2] = buffer->bColor_B; - buffer[0].pixels[y][3*x+0] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+0] + alpha * buffer->bColor_R)); + buffer[0].pixels[y][3*x+0] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+0] + alpha * buffer->bColor_R)); // was just buffer->bColor_R, etc buffer[0].pixels[y][3*x+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+1] + alpha * buffer->bColor_G)); buffer[0].pixels[y][3*x+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[y][3*x+2] + alpha * buffer->bColor_B)); } + buffer[0].mask[y][x] = 1; return; } @@ -150,37 +173,51 @@ void bDrawRectOpen (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { - int X1, Y1, X2, Y2; - if (x1 > x2) SWAP (x1, x2); if (y1 > y2) SWAP (y1, y2); + int X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); + int X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); + + int Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); + int Y2 = MIN (MAX (ROUND (y2), 1), buffer[0].Ny - 1); + + int dNs = -0.5*(buffer->bWeight - 1); + /* 0, 0, 0, -1, -1, -2, -2 */ + + int dNe = +0.5*buffer->bWeight + 1; + /* 1, 1, 2, 2, 2, 3, 3 */ + + for (int dN = dNs; dN < dNe; dN ++) { + // line on the bottom needs to run longer for the negative dN values + bDrawLineHorizontal (buffer, X1 + dN, X2 + 1 - dN, Y1 + dN); + + // line on the top needs to run longer for the positive dN values + bDrawLineHorizontal (buffer, X1 + dN, X2 + 1 - dN, Y2 - dN); + + // line on the left needs to run longer for the negative dN values + bDrawLineVertical (buffer, X1 + dN, Y1 + dN, Y2 - dN); + + // line on the right needs to run longer for the positive dN values + bDrawLineVertical (buffer, X2 - dN, Y1 + dN, Y2 - dN); + } + return; +} + +void bDrawRectFill (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { + + int i; + int X1, Y1, X2, Y2; + + if (x1 > x2) SWAP (x1, x2); + if (y1 > y2) SWAP (y1, y2); + X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); - Y2 = MIN (MAX (ROUND (y2), 1), buffer[0].Ny - 1); - - bDrawLineHorizontal (buffer, X1, X2 + 1, Y1); - bDrawLineHorizontal (buffer, X1, X2 + 1, Y2); - bDrawLineVertical (buffer, X1, Y1, Y2); - bDrawLineVertical (buffer, X2, Y1, Y2); - return; -} - -void bDrawRectFill (bDrawBuffer *buffer, double x1, double y1, double x2, double y2) { - - int i; - int X1, Y1, X2, Y2; - - if (x1 > x2) SWAP (x1, x2); - if (y1 > y2) SWAP (y1, y2); - - X1 = MIN (MAX (ROUND (x1), 0), buffer[0].Nx - 1); - X2 = MIN (MAX (ROUND (x2), 1), buffer[0].Nx - 1); - - Y1 = MIN (MAX (ROUND (y1), 0), buffer[0].Ny - 1); Y2 = MIN (MAX (ROUND (y2), 0), buffer[0].Ny - 1); for (i = Y1; i < Y2; i++) { + // this should be a line of width 1 or we duplicate pixels bDrawLineHorizontal (buffer, X1, X2, i); } @@ -235,4 +272,5 @@ // use the Bresenham line drawing technique // integer-only Bresenham line-draw version which is fast +// bresenham assumes x1 < x2 and (y2 - y1) < (x2 - x1) void bDrawLineBresen (bDrawBuffer *buffer, int X1, int Y1, int X2, int Y2, int swapcoords) { @@ -299,8 +337,6 @@ buffer[0].pixels[Y][3*i+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[Y][3*i+1] + alpha * buffer->bColor_G)); buffer[0].pixels[Y][3*i+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[Y][3*i+2] + alpha * buffer->bColor_B)); - // buffer[0].pixels[Y][3*i+0] = buffer->bColor_R; - // buffer[0].pixels[Y][3*i+1] = buffer->bColor_G; - // buffer[0].pixels[Y][3*i+2] = buffer->bColor_B; - } + } + buffer[0].mask[Y][i] = 1; } return; @@ -326,8 +362,6 @@ buffer[0].pixels[i][3*X+1] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[i][3*X+1] + alpha * buffer->bColor_G)); buffer[0].pixels[i][3*X+2] = MAX (0x00, MIN(0xff, beta * buffer[0].pixels[i][3*X+2] + alpha * buffer->bColor_B)); - // buffer[0].pixels[i][3*X+0] = buffer->bColor_R; - // buffer[0].pixels[i][3*X+1] = buffer->bColor_G; - // buffer[0].pixels[i][3*X+2] = buffer->bColor_B; - } + } + buffer[0].mask[i][X] = 1; } return; @@ -384,4 +418,148 @@ return; +} + +# define BOT_LEFT 0 +# define BOT_RGHT 1 +# define TOP_LEFT 2 +# define TOP_RGHT 3 + +// I should probably look up an appropriate recipe for this +// source code for XDrawPolyFill? + +int GetNextSeqNumber (int seq, int Npoint, int Clockwise, int LeftSide); +int bDrawFillBetweenSegments (bDrawBuffer *buffer, int *x, int *y, int ystart); + +void bDrawPolyFill (bDrawBuffer *buffer, double *x, double *y, int Npoints) { + + // The coord list which is passed in (x,y) must be in order around the contour + for (int i = 0; i < Npoints; i++) { + fprintf (stderr, "%d : %f,%f\n", i, x[i], y[i]); + } + + // First, find the lowest point and start at that sequence number + int iMin = 0; + double yMin = y[iMin]; + for (int i = 1; i < Npoints; i++) { + if (y[i] < yMin) { iMin = i; yMin = y[iMin]; } + } + + // we generate two line segments and will fill between them + int xval[4]; + int yval[4]; + + // the starting point is a vertex + xval[BOT_LEFT] = x[iMin]; yval[BOT_LEFT] = y[iMin]; + xval[BOT_RGHT] = x[iMin]; yval[BOT_RGHT] = y[iMin]; + + // get iNext, iPrev assuming ClockWise, then test + int isCW = TRUE; + int iNextLeft = GetNextSeqNumber (iMin, Npoints, isCW, TRUE); // TRUE => left-side + int iNextRght = GetNextSeqNumber (iMin, Npoints, isCW, FALSE); + + // if this is true, the points are not clockwise + if (x[iNextLeft] > x[iNextRght]) { + isCW = FALSE; + int tmp = iNextLeft; + iNextLeft = iNextRght; + iNextRght = tmp; + } + + int iTL = iNextLeft; // sequence number of the top,left point + int iTR = iNextRght; // sequence number of the top,right point + xval[TOP_LEFT] = x[iTL]; yval[TOP_LEFT] = y[iTL]; + xval[TOP_RGHT] = x[iTR]; yval[TOP_RGHT] = y[iTR]; + + // we need to track the starting row, start at the bottom + int ystart = yval[BOT_LEFT]; + + while (TRUE) { + int isLeft = bDrawFillBetweenSegments (buffer, xval, yval, ystart); + + // the last pair of segments in the sequence end at the same top point: + if (iTL == iTR) break; + + // after one pass, if isLeft is true, then cycle the points on the left + // segment, otherwise cycle the right segment: + + if (isLeft) { + iTL = GetNextSeqNumber (iTL, Npoints, isCW, TRUE); + xval[BOT_LEFT] = xval[TOP_LEFT]; yval[BOT_LEFT] = yval[TOP_LEFT]; + xval[TOP_LEFT] = x[iTL]; yval[TOP_LEFT] = y[iTL]; + ystart = yval[BOT_LEFT]; + } else { + iTR = GetNextSeqNumber (iTR, Npoints, isCW, FALSE); + xval[BOT_RGHT] = xval[TOP_RGHT]; yval[BOT_RGHT] = yval[TOP_RGHT]; + xval[TOP_RGHT] = x[iTR]; yval[TOP_RGHT] = y[iTR]; + ystart = yval[BOT_RGHT]; + } + } + return; +} + +// we have two line segments defined by x[],y[]. + +// [0],[1] is the left-side segment +// [2],[3] is the right-side segment + +// Fill the region between the two segments until +// the lower of the two y coordinates y[1], y[3] + +// return which of y[1] or y[3] was the ending row + +int bDrawFillBetweenSegments (bDrawBuffer *buffer, int *x, int *y, int ystart) { + + int dY_L = y[TOP_LEFT] - y[BOT_LEFT]; + int dX_L = x[TOP_LEFT] - x[BOT_LEFT]; + + int dY_R = y[TOP_RGHT] - y[BOT_RGHT]; + int dX_R = x[TOP_RGHT] - x[BOT_RGHT]; + + double Slope_L = (dY_L == 0) ? NAN : dX_L / (double) dY_L; + double Slope_R = (dY_R == 0) ? NAN : dX_R / (double) dY_R; + + // stop at whichever comes first + // XXX not sure if I need to use < or <= here: + // with <=, the value of Y after the loop will be one too high. + int Y; + for (Y = ystart; (Y < y[TOP_LEFT]) && (Y < y[TOP_RGHT]); Y++) { + + // calculating X_L, X_R based on the slope and current y point + int X_L = (dY_L == 0) ? x[BOT_LEFT] : (Y - y[BOT_LEFT]) * Slope_L + x[BOT_LEFT]; + int X_R = (dY_R == 0) ? x[BOT_RGHT] : (Y - y[BOT_RGHT]) * Slope_R + x[BOT_RGHT]; + + // draw horizontal line from X_L to X_R at Y + bDrawLineHorizontal (buffer, X_L, X_R, Y); + } + + if (Y == y[TOP_LEFT]) { + return TRUE; // TRUE is LEFT SIDE + } else { + return FALSE; + } +} + +int GetNextSeqNumber (int seq, int Npoints, int Clockwise, int LeftSide) { + + int iNext; + + if (Clockwise) { + if (LeftSide) { + // next point is next in sequence + iNext = (seq < Npoints - 1) ? seq + 1 : 0; + } else { + // next point is prev in sequence + iNext = (seq > 0) ? seq - 1 : Npoints - 1; + } + } else { + if (LeftSide) { + // next point is prev in sequence + iNext = (seq > 0) ? seq - 1 : Npoints - 1; + } else { + // next point is next in sequence + iNext = (seq < Npoints - 1) ? seq + 1 : 0; + } + } + return iNext; } @@ -585,4 +763,5 @@ if (x == 0) { if (decrementY) { + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc , Xc + 1, Yc + y); bDrawLineHorizontal (buffer, Xc , Xc + 1, Yc - y); @@ -590,4 +769,5 @@ // center line + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc ); return; @@ -595,4 +775,5 @@ if (x == y) { + // this should must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc + y); bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc - y); @@ -602,4 +783,5 @@ // only draw these two lines if we decrement y if (decrementY) { + // this must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc + y); bDrawLineHorizontal (buffer, Xc - x, Xc + x + 1, Yc - y); @@ -607,4 +789,5 @@ // always draw these two lines: + // this must be a line or width 1 or we duplicate pixels bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc + x); bDrawLineHorizontal (buffer, Xc - y, Xc + y + 1, Yc - x); @@ -648,5 +831,110 @@ } +// run a smoothing kernel on each channel for anti-aliasing +// XXX I need to consider the mask as well +void bDrawSmooth (bDrawBuffer *buffer, float sigma) { + + // generate a temp buffer for storage of the smoothed result + int Nx = buffer[0].Nx; + int Ny = buffer[0].Ny; + ALLOCATE_PTR (temp, bDrawColor, Nx*Ny); + + //* build a 1D gaussian + int Nsigma = 2; + int Ns = (int) (Nsigma*sigma + 0.5); + int Ngauss = 2*Ns + 1; + ALLOCATE_PTR (gaussnorm, float, Ngauss); + float *gauss = &gaussnorm[Ns]; + for (int i = -Ns; i < Ns + 1; i++) { + gauss[i] = exp ((i*i)/(-2*sigma*sigma)); + } + + // NOTE XXX: need to handle Nbyte = 1 or 3 + int Nchannel = 3; + + // we need to smooth the 3 channels independently, do this as three passes + for (int ch = 0; ch < Nchannel; ch++) { + + // smooth in X direction + for (int j = 0; j < Ny; j++) { + bDrawColor *vi = (bDrawColor *) &buffer[0].pixels[j][ch]; + bDrawColor *vo = &temp[j*Nx]; + for (int i = 0; i < Nx; i++, vo++, vi += Nchannel) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (i+n < 0) continue; + if (i+n >= Nx) continue; + s += gauss[n]*vi[Nchannel*n]; + g += gauss[n]; + } + *vo = s / g; + } + } + + // NOTE: output maps back into original image + // careful on the counting + + /* smooth in Y direction */ + for (int i = 0; i < Nx; i++) { + bDrawColor *vi = &temp[i]; + for (int j = 0; j < Ny; j++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (j+n < 0) continue; + if (j+n >= Ny) continue; + s += gauss[n]*vi[(n+j)*Nx]; // vi is a single-index array, so this is stepping by rows + g += gauss[n]; + } + buffer[0].pixels[j][Nchannel*i + ch] = s / g; + } + } + } + + // smooth the mask + ALLOCATE_PTR (temp_mask, float, Nx*Ny); + + // smooth in X direction + for (int j = 0; j < Ny; j++) { + char *vi = (char *) &buffer[0].mask[j][0]; + float *vo = &temp_mask[j*Nx]; + for (int i = 0; i < Nx; i++, vo++, vi++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (i+n < 0) continue; + if (i+n >= Nx) continue; + s += gauss[n]*vi[n]; + g += gauss[n]; + } + *vo = s / g; + } + } + + // NOTE: output maps back into original image + // careful on the counting + + /* smooth in Y direction */ + for (int i = 0; i < Nx; i++) { + float *vi = &temp_mask[i]; + for (int j = 0; j < Ny; j++) { + float g = 0.0, s = 0.0; + for (int n = -Ns; n <= Ns; n++) { + if (j+n < 0) continue; + if (j+n >= Ny) continue; + s += gauss[n]*vi[(n+j)*Nx]; // vi is a single-index array, so this is stepping by rows + g += gauss[n]; + } + buffer[0].mask[j][i] = (s / g > 0.05) ? 1 : 0; + } + } + + free (temp); + free (temp_mask); + free (gaussnorm); + + return; +} + /* +NOTES on Bresenham-style circles: the discriminant of inside or outside the circle is: Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_allocate.h (revision 41170) @@ -44,4 +44,7 @@ void ohana_memcheck_block_func (const char *myFile, int myLine, const char *myFunc, void *in); void real_free (void *in); + +void ohana_memdump_strings_file (FILE *f, int VERBOSE); +void ohana_memdump_set_maxlines (int N); # define ohana_memcheck(X) ohana_memcheck_func (__FILE__, __LINE__, __func__, X) Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/include/ohana_sort.h (revision 41170) @@ -83,3 +83,6 @@ void isortfour (int *X, int *Y, int *Z, int *W, int N); +void dsort_indexonly (double *X, off_t *S, off_t N); +void dsort_int_indexonly (double *X, int *S, int N); + #endif Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/ohana_allocate.c (revision 41170) @@ -45,4 +45,6 @@ static pthread_mutex_t memBlockListMutex = PTHREAD_MUTEX_INITIALIZER; + +static int NblockMaxDump = 100; void ohana_meminit () { @@ -435,5 +437,5 @@ fprintf (stderr, "memory corruption\n"); } - if (Nbad < 100) { + if (Nbad < NblockMaxDump) { fprintf (stderr, " file: %s, line: %d, func: %s\n", thisBlock->file, thisBlock->line, thisBlock->func); } @@ -613,5 +615,5 @@ Nbytes += thisBlock->size; - if (Ntotal < 100) { + if (Ntotal < NblockMaxDump) { if (good) { fprintf (f, " %zd %zd %zd GOOD %s %d, func: %s\n", Ntotal, thisBlock->size, Nbytes, thisBlock->file, thisBlock->line, thisBlock->func); @@ -677,2 +679,49 @@ return memstats; } + +void ohana_memdump_strings_file (FILE *f, int VERBOSE) { + + if (!lastBlock) { + if (VERBOSE) fprintf (f, "no memory allocated\n"); + return; + } + + OhanaMemblock *thisBlock = lastBlock; + + size_t Ntotal = 0; + size_t Nbytes = 0; + size_t Nstring = 0; + + fprintf (f, " entry | bytes | cumulative : line : string\n"); + + while (thisBlock) { + + // pointer to the start of the user memory + char *ptr = (char *)(thisBlock + 1); + + Ntotal ++; + Nbytes += thisBlock->size; + + int N = strlen(thisBlock->file); + if (N > 22) { + int found = !strcmp("libohana/src/string.c", &thisBlock->file[N - 21]); + if (found) { + if (Nstring < NblockMaxDump) { + fprintf (f, " %8zd | %8zd | %10zd : %4d : %s\n", Ntotal, thisBlock->size, Nbytes, thisBlock->line, ptr); + } + Nstring ++; + } + thisBlock = thisBlock->prevBlock; + } + } + + if (Nstring || VERBOSE) { + fprintf (f, "%zd strings allocated\n", Nstring); + } + + return; +} + +void ohana_memdump_set_maxlines (int N) { + NblockMaxDump = N; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/libohana/src/sorts.c (revision 41170) @@ -196,2 +196,33 @@ } +/* sort the index of a vector (vector stays unsorted) */ +void dsort_indexonly (double *X, off_t *S, off_t N) { + +# define SWAPFUNC(A,B){ off_t itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ +} +# define COMPARE(A,B)((!isfinite(X[S[A]]) && isfinite(X[S[B]])) || (X[S[A]] < X[S[B]])) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +/* sort the index of a vector (vector stays unsorted) */ +void dsort_int_indexonly (double *X, int *S, int N) { + +# define SWAPFUNC(A,B){ int itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ +} +# define COMPARE(A,B)((!isfinite(X[S[A]]) && isfinite(X[S[B]])) || (X[S[A]] < X[S[B]])) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdensify.c (revision 41170) @@ -39,4 +39,11 @@ } + int binning = 1; + if ((N = get_argument (argc, argv, "-binning"))) { + remove_argument (N, &argc, argv); + binning = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int PSFTYPE = IS_DOT; if ((N = get_argument (argc, argv, "-psf"))) { @@ -80,6 +87,9 @@ Ymax = graphmode.ymax; Ymin = graphmode.ymin; - dX = (Xmax - Xmin) / (Xpix - 1); - dY = (Ymax - Ymin) / (Ypix - 1); + + // (dX, dY) are the pixel scale of the output image (output pixels / input pixels) + + dX = binning * (Xmax - Xmin) / (Xpix - 1); + dY = binning * (Ymax - Ymin) / (Ypix - 1); CHECKVAL(Xmin); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/cdhistogram.c (revision 41170) @@ -13,4 +13,11 @@ int kapa; Graphdata graphmode; + + int binning = 1; + if ((N = get_argument (argc, argv, "-binning"))) { + remove_argument (N, &argc, argv); + binning = atoi(argv[N]); + remove_argument (N, &argc, argv); + } range = NULL; @@ -79,6 +86,9 @@ Ymax = graphmode.ymax; Ymin = graphmode.ymin; - dX = (Xmax - Xmin) / (Xpix - 1); - dY = (Ymax - Ymin) / (Ypix - 1); + + // (dX, dY) are the pixel scale of the output image (output pixels / input pixels) + + dX = binning * (Xmax - Xmin) / (Xpix - 1); + dY = binning * (Ymax - Ymin) / (Ypix - 1); CHECKVAL(Xmin); @@ -90,6 +100,6 @@ CHECKVAL(dY); - int Nx = (Xmax - Xmin) / dX + 1; - int Ny = (Ymax - Ymin) / dY + 1; + int Nx = abs((Xmax - Xmin) / dX) + 1; + int Ny = abs((Ymax - Ymin) / dY) + 1; Coords newcoords = graphmode.coords; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.astro/region.c (revision 41170) @@ -3,5 +3,5 @@ int region (int argc, char **argv) { - double Ra, Dec, Radius; + double Ra, Dec; float dx, dy; int N, kapa, NoClear, dXpix, dYpix; @@ -54,4 +54,26 @@ } + float XSIZE = NAN; + if ((N = get_argument (argc, argv, "-xsize"))) { + remove_argument (N, &argc, argv); + XSIZE = atof (argv[N]); + remove_argument (N, &argc, argv); + if (XSIZE <= 0) { + gprint (GP_ERR, "ERROR: xsize <= 0\n"); + return FALSE; + } + } + + float YSIZE = NAN; + if ((N = get_argument (argc, argv, "-ysize"))) { + remove_argument (N, &argc, argv); + YSIZE = atof (argv[N]); + remove_argument (N, &argc, argv); + if (YSIZE <= 0) { + gprint (GP_ERR, "ERROR: xsize <= 0\n"); + return FALSE; + } + } + if ((N = get_argument (argc, argv, "-ew"))) { remove_argument (N, &argc, argv); @@ -81,6 +103,11 @@ } - if ((argc != 4) && (argc != 5)) { - gprint (GP_ERR, "USAGE: region Ra Dec Radius [projection] [orientation]\n"); + int RadiusArg, CtypeArg; + if (!isnan(XSIZE) || !isnan(YSIZE)) { + RadiusArg = -1; + CtypeArg = 3; + if ((argc != 3) && (argc != 4)) { + gprint (GP_ERR, "USAGE: region Ra Dec [projection] [-xsize deg] [-ysize deg]\n"); + gprint (GP_ERR, " OR: region Ra Dec Radius [projection]\n"); gprint (GP_ERR, " [-image] [-ew] [+ew] [-ns] [+ns] [-no-clear] [-angle theta]\n"); gprint (GP_ERR, " current: %f %f (%f x %f) (%s)\n", @@ -90,28 +117,51 @@ &graphmode.coords.ctype[5]); return (FALSE); - } - + } + } else { + RadiusArg = 3; + CtypeArg = 4; + if ((argc != 4) && (argc != 5)) { + gprint (GP_ERR, "USAGE: region Ra Dec Radius [projection]\n"); + gprint (GP_ERR, " OR: region Ra Dec [projection] [-xsize deg] [-ysize deg]\n"); + gprint (GP_ERR, " [-image] [-ew] [+ew] [-ns] [+ns] [-no-clear] [-angle theta]\n"); + gprint (GP_ERR, " current: %f %f (%f x %f) (%s)\n", + graphmode.coords.crval1, graphmode.coords.crval2, + fabs(graphmode.xmax - graphmode.xmin), + fabs(graphmode.ymax - graphmode.ymin), + &graphmode.coords.ctype[5]); + return (FALSE); + } + } if (!ohana_str_to_radec (&Ra, &Dec, argv[1], argv[2])) return (FALSE); - Radius = atof (argv[3]); + + // region 0 0 sin -- should raise an error (radius = 0 or non-numeric) + + // I want to be able to support the old style call in which both of these were valid: + // region 0 0 90 ait + // region 0 0 90 <- uses existing, sticky projection type + // but I also want to be able to use + // region 0 0 ait -xsize 5 + // region 0 0 ait -ysize 5 + + double Radius = NAN; + if (RadiusArg >= 0) { + Radius = atof (argv[RadiusArg]); + } + InitCoords (&graphmode.coords, "DEC--TAN"); - - if (argc == 5) { - if (!strcasecmp (argv[4], "TAN")) - strcpy (graphmode.coords.ctype, "DEC--TAN"); - if (!strcasecmp (argv[4], "SIN")) - strcpy (graphmode.coords.ctype, "DEC--SIN"); - if (!strcasecmp (argv[4], "ARC")) - strcpy (graphmode.coords.ctype, "DEC--ARC"); - if (!strcasecmp (argv[4], "STG")) - strcpy (graphmode.coords.ctype, "DEC--STG"); - if (!strcasecmp (argv[4], "ZEA")) - strcpy (graphmode.coords.ctype, "DEC--ZEA"); - if (!strcasecmp (argv[4], "AIT")) - strcpy (graphmode.coords.ctype, "DEC--AIT"); - if (!strcasecmp (argv[4], "GLS")) - strcpy (graphmode.coords.ctype, "DEC--GLS"); - if (!strcasecmp (argv[4], "PAR")) - strcpy (graphmode.coords.ctype, "DEC--PAR"); - } + if (argc == CtypeArg + 1) { + if (!strcasecmp (argv[CtypeArg], "TAN")) { strcpy (graphmode.coords.ctype, "DEC--TAN"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "SIN")) { strcpy (graphmode.coords.ctype, "DEC--SIN"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "ARC")) { strcpy (graphmode.coords.ctype, "DEC--ARC"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "STG")) { strcpy (graphmode.coords.ctype, "DEC--STG"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "ZEA")) { strcpy (graphmode.coords.ctype, "DEC--ZEA"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "AIT")) { strcpy (graphmode.coords.ctype, "DEC--AIT"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "GLS")) { strcpy (graphmode.coords.ctype, "DEC--GLS"); goto got_ctype; } + if (!strcasecmp (argv[CtypeArg], "PAR")) { strcpy (graphmode.coords.ctype, "DEC--PAR"); goto got_ctype; } + gprint (GP_ERR, "ERROR: invalid projection type %s\n", argv[CtypeArg]); + gprint (GP_ERR, "allowed values: TAN, SIN, ARC, STG, ZEA, AIT, GLS, PAR\n"); + return FALSE; + } +got_ctype: graphmode.coords.crval1 = Ra; @@ -126,5 +176,6 @@ graphmode.coords.pc2_2 = cos(Angle*RAD_DEG)*pc2_2; - /* ask kapa for coordinate limits, to get the right aspect ratio */ + // ask kapa for coordinate limits, to get the right aspect ratio + // dx, dy are the size of the graph region in pixels KapaGetLimits (kapa, &dx, &dy); dx = fabs (dx); @@ -132,15 +183,38 @@ /* define limits for Ra, Dec at center, grid in degrees */ - if (dy < dx) { - graphmode.xmin = -(dx/dy)*Radius; - graphmode.ymin = -Radius; - graphmode.xmax = (dx/dy)*Radius; - graphmode.ymax = Radius; + if (RadiusArg >= 0) { + // force non-anamorphic projection with Radius set to smaller axis + if (dy < dx) { + graphmode.xmin = -(dx/dy)*Radius; + graphmode.ymin = -Radius; + graphmode.xmax = (dx/dy)*Radius; + graphmode.ymax = Radius; + } else { + graphmode.xmin = -Radius; + graphmode.ymin = -(dy/dx)*Radius; + graphmode.xmax = Radius; + graphmode.ymax = (dy/dx)*Radius; + } } else { - graphmode.xmin = -Radius; - graphmode.ymin = -(dy/dx)*Radius; - graphmode.xmax = Radius; - graphmode.ymax = (dy/dx)*Radius; - } + if (isnan(XSIZE)) { + graphmode.xmin = -(dx/dy)*YSIZE/2.0; + graphmode.ymin = -YSIZE/2.0; + graphmode.xmax = (dx/dy)*YSIZE/2.0; + graphmode.ymax = YSIZE/2.0; + } + if (isnan(YSIZE)) { + graphmode.xmin = -XSIZE/2.0; + graphmode.ymin = -(dy/dx)*XSIZE/2.0; + graphmode.xmax = XSIZE/2.0; + graphmode.ymax = (dy/dx)*XSIZE/2.0; + } + // anamorphic projection: + if (!isnan(XSIZE) && !isnan(YSIZE)) { + graphmode.xmin = -XSIZE/2.0; + graphmode.ymin = -YSIZE/2.0; + graphmode.xmax = XSIZE/2.0; + graphmode.ymax = YSIZE/2.0; + } + } set_variable ("XMIN", graphmode.xmin); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/init.c (revision 41170) @@ -73,4 +73,6 @@ {1, "??", list_vars, "list variables *"}, {1, "#", nop, "a NOP function"}, + {1, "##", nop, "a NOP function"}, + {1, "###", nop, "a NOP function"}, {1, "local", local, "define local variables"}, {1, "macro", macro, "deal with the macros *"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/memory.c (revision 41170) @@ -5,4 +5,11 @@ if (argc < 2) goto usage; + + if (!strcasecmp (argv[1], "max-lines")) { + if (argc != 3) goto usage; + int MaxLines = atoi(argv[2]); + ohana_memdump_set_maxlines (MaxLines); + return TRUE; + } if (!strcasecmp (argv[1], "all")) { @@ -43,6 +50,13 @@ } + if (!strncasecmp ("strings", argv[1], strlen(argv[1]))) { + ohana_memdump_strings_file (stderr, FALSE); + return (TRUE); + } + usage: - gprint (GP_ERR, "USAGE: memory (all/leaks)\n"); + gprint (GP_ERR, "USAGE: memory (mode) [-max-lines]\n"); + gprint (GP_ERR, "USAGE: memory (max-lines) (N)\n"); + gprint (GP_ERR, " mode options: all, leaks, check, checkfree, variables, vectors, buffers, macros, commands, strings\n"); return (FALSE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/if.sh (revision 41170) @@ -320,2 +320,82 @@ end end + +# check memleaks +macro memtest1 + + local a b i N + + $i = 0 + $a = 1 + + memory check + output /dev/null + for i 0 1000 + if ($a == 1) + echo "run" + end + end + output stdout + memory check +end + +# check memleaks +macro memtest2 + + local a b i N + + $i = 0 + $a = 1 + $b = 2 + + memory check + output /dev/null + for i 0 1000 + if (($a == 1) && ($b == 2)) + echo "run" + end + end + output stdout + memory check +end + +# memory test using continue +macro memtest3 + + local a b i N + + $i = 0 + $a = 1 + $b = 2 + + memory check + output /dev/null + for i 0 1000 + if (($a == 1) && ($b == 2)) + continue + echo "run" + end + end + output stdout + memory check +end + +# memory test using continue +macro memtest4 + + local a b i N bool + + + $i = 0 + $a = 1 + $b = 2 + $bool = 0 + + memory check + output /dev/null + for i 0 1000 + $bool = (($a == 1) && ($b == 2)) + end + output stdout + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.basic/test/macro.sh (revision 41170) @@ -7,4 +7,13 @@ # Is the macro working? macro test_prep + $test_var1 = check1 + $test_var2 = check2 + $var_count = $0 +end + +# Is the macro working? +macro test_local + local test_var test_var2 var_count + $test_var1 = check1 $test_var2 = check2 @@ -47,2 +56,30 @@ end end + +# Memory Test for macro +macro memtest2 + + $i = 0 + test_prep + + memory check + for i 0 10000 + test_prep + end + memory check + +end + +# Memory Test for macro +macro memtest3 + + $i = 0 + test_local + + memory check + for i 0 10000 + test_local + end + memory check + +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/Makefile (revision 41170) @@ -19,4 +19,5 @@ $(SRC)/init.$(ARCH).o \ $(SRC)/accum.$(ARCH).o \ +$(SRC)/antialias.$(ARCH).o \ $(SRC)/applyfit1d.$(ARCH).o \ $(SRC)/applyfit2d.$(ARCH).o \ @@ -78,4 +79,5 @@ $(SRC)/join.$(ARCH).o \ $(SRC)/jpeg.$(ARCH).o \ +$(SRC)/kapamemory.$(ARCH).o \ $(SRC)/kern.$(ARCH).o \ $(SRC)/keyword.$(ARCH).o \ @@ -89,8 +91,10 @@ $(SRC)/lookup.$(ARCH).o \ $(SRC)/matrix.$(ARCH).o \ +$(SRC)/match1d.$(ARCH).o \ $(SRC)/match2d.$(ARCH).o \ $(SRC)/mkrgb.$(ARCH).o \ $(SRC)/mcreate.$(ARCH).o \ $(SRC)/medacc.$(ARCH).o \ +$(SRC)/mgaussdev.$(ARCH).o \ $(SRC)/mget.$(ARCH).o \ $(SRC)/mget3d.$(ARCH).o \ @@ -99,4 +103,5 @@ $(SRC)/medimage.$(ARCH).o \ $(SRC)/medimage_commands.$(ARCH).o \ +$(SRC)/medimage_calc.$(ARCH).o \ $(SRC)/mset.$(ARCH).o \ $(SRC)/needles.$(ARCH).o \ @@ -169,4 +174,5 @@ $(SRC)/vgauss.$(ARCH).o \ $(SRC)/vlorentz.$(ARCH).o \ +$(SRC)/vsigmoid.$(ARCH).o \ $(SRC)/vellipse.$(ARCH).o \ $(SRC)/vmaxwell.$(ARCH).o \ @@ -176,4 +182,5 @@ $(SRC)/vzload.$(ARCH).o \ $(SRC)/vpeaks.$(ARCH).o \ +$(SRC)/vtransitions.$(ARCH).o \ $(SRC)/vpop.$(ARCH).o \ $(SRC)/vroll.$(ARCH).o \ @@ -182,4 +189,6 @@ $(SRC)/vsmooth.$(ARCH).o \ $(SRC)/vstats.$(ARCH).o \ +$(SRC)/virls.$(ARCH).o \ +$(SRC)/vwtmean.$(ARCH).o \ $(SRC)/xsection.$(ARCH).o \ $(SRC)/vsh.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/antialias.c (revision 41170) @@ -0,0 +1,26 @@ +# include "data.h" + +int antialias (int argc, char **argv) { + + int N, kapa; + Graphdata graphmode; + + char *name = NULL; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + name = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetGraph (&graphmode, &kapa, name)) return (FALSE); + FREE (name); + + if (argc != 2) { + gprint (GP_ERR, "USAGE: antialias (sigma)\n"); + return (FALSE); + } + + float sigma = atof (argv[1]); + + KapaSetSmoothSigma (kapa, sigma); + return (TRUE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/cut.c (revision 41170) @@ -1,10 +1,71 @@ # include "data.h" -enum {SUM, MEAN, MEDIAN}; +enum {SUM, MEAN, MEDIAN, INNER, NGOOD}; + +double cutstat (float *value, int Nvalue, int mode) { + + double output = 0.0; + + if (Nvalue == 0) return NAN; + + if (mode == MEDIAN) { + fsort (value, Nvalue); + if (Nvalue % 2) { + int Ncenter = Nvalue / 2; + myAssert ((Ncenter >= 0) && (Ncenter < Nvalue), "cutstat"); + output = value[Ncenter]; + } else { + int Ncenter = Nvalue / 2 - 1; + myAssert ((Ncenter >= 0) && (Ncenter < Nvalue - 1), "cutstat"); + output = 0.5*(value[Ncenter] + value[Ncenter + 1]); + } + return output; + } + if (mode == INNER) { + fsort (value, Nvalue); + + int Ns = 0, Ne = 0; + if (Nvalue % 2) { + // for an odd number of points take the same number below + // and above the center value + int Ncenter = Nvalue / 2; + int Nquarter = 0.25*Nvalue; + Ns = Ncenter - Nquarter; + Ne = Ncenter + Nquarter; + } else { + // for an even number, the middle lies between two points + // take the same number below as above + int Ncenter = (int)(Nvalue / 2) - 1; + int Nquarter = 0.25*Nvalue; + Ns = Ncenter - Nquarter; + Ne = Ncenter + 1 + Nquarter; + } + int Nv = 0; + for (int i = Ns; i <= Ne; i++) { + myAssert ((i >= 0) && (i < Nvalue), "cutstat"); + output += value[i]; + Nv ++; + } + output /= Nv; + return output; + } + if (mode == NGOOD) { + return Nvalue; + } + if ((mode == MEAN) || (mode == SUM)) { + for (int i = 0; i < Nvalue; i++) { + myAssert ((i >= 0) && (i < Nvalue), "cutstat"); + output += value[i]; + } + if (mode == MEAN) { output /= Nvalue; } + return output; + } + return NAN; +} int cut (int argc, char **argv) { int i, j, N, Nx, Ny, Mode; - float *Vin, *Vbuf, value; + float *Vin, *Vbuf; int sx, sy, nx, ny; Vector *xvec, *yvec; @@ -12,4 +73,8 @@ Mode = SUM; + if ((N = get_argument (argc, argv, "-sum"))) { + remove_argument (N, &argc, argv); + Mode = SUM; + } if ((N = get_argument (argc, argv, "-median"))) { remove_argument (N, &argc, argv); @@ -19,4 +84,12 @@ remove_argument (N, &argc, argv); Mode = MEAN; + } + if ((N = get_argument (argc, argv, "-inner"))) { + remove_argument (N, &argc, argv); + Mode = INNER; + } + if ((N = get_argument (argc, argv, "-ngood"))) { + remove_argument (N, &argc, argv); + Mode = NGOOD; } @@ -36,6 +109,7 @@ Ny = buf[0].matrix.Naxis[1]; - if ((sx < 0) || (sy < 0) || (sx+nx > Nx) || (sy+ny > Ny)) { - gprint (GP_ERR, "region out of range\n"); + // ny & nx do not need to be constrained by the buffer, but they do need to be sensible + if ((nx < 0) || (ny < 0) || (nx > 1e8) || (ny > 1e8)) { + gprint (GP_ERR, "warning : extraction size is crazy: %d,%d\n", nx, ny); return (FALSE); } @@ -57,21 +131,31 @@ for (i = 0; i < nx; i++) { + // for out-of-range areas, set the output pixels to NAN + if (i + sx < 0) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + if (i + sx >= Nx) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + /* accumulate values */ + // skip out-of-range areas in y + if (sy < 0) { + ny += sy; + sy = 0; + } + if (sy > Ny) sy = Ny; + if (sy + ny >= Ny) ny = Ny - sy; + Vin = (float *)(buf[0].matrix.buffer) + sy*Nx + sx + i; + int Npix = 0; for (j = 0; j < ny; j++, Vin += Nx) { - Vbuf[j] = *Vin; - } - /* apply stat of choice */ - if (Mode == MEDIAN) { - fsort (Vbuf, ny); - value = Vbuf[(int)(0.5*ny)]; - } else { - value = 0; - for (j = 0; j < ny; j++) { - value += Vbuf[j]; - } - if (Mode == MEAN) { value /= ny; } - } - yvec[0].elements.Flt[i] = value; + if (!isfinite(*Vin)) continue; + Vbuf[Npix] = *Vin; + Npix ++; + } + yvec[0].elements.Flt[i] = cutstat (Vbuf, Npix, Mode); } free (Vbuf); @@ -89,21 +173,31 @@ for (i = 0; i < ny; i++) { + // for out-of-range areas, set the output pixels to NAN + if (i + sy < 0) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + if (i + sy >= Ny) { + yvec[0].elements.Flt[i] = NAN; + continue; + } + /* accumulate values */ + // skip out-of-range areas in x + if (sx < 0) { + nx += sx; + sx = 0; + } + if (sx > Nx) sx = Nx; + if (sx + nx >= Nx) nx = Nx - sx; + Vin = (float *)(buf[0].matrix.buffer) + (sy + i)*Nx + sx; + int Npix = 0; for (j = 0; j < nx; j++, Vin ++) { - Vbuf[j] = *Vin; - } - /* apply stat of choice */ - if (Mode == MEDIAN) { - fsort (Vbuf, nx); - value = Vbuf[(int)(0.5*nx)]; - } else { - value = 0; - for (j = 0; j < nx; j++) { - value += Vbuf[j]; - } - if (Mode == MEAN) { value /= nx; } - } - yvec[0].elements.Flt[i] = value; + if (!isfinite(*Vin)) continue; + Vbuf[Npix] = *Vin; + Npix ++; + } + yvec[0].elements.Flt[i] = cutstat (Vbuf, Npix, Mode); } free (Vbuf); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/extract.c (revision 41170) @@ -1,11 +1,25 @@ # include "data.h" + +/* : source image, must exist + : target image -- if it does not exist, it is created of size (Nx,Ny) + sx, sy : source starting coordinate -- need not be on a valid image pixel + nx, ny : number of source pixels -- currently must not go out of bounds -> allow out-of-bounds + Sx, Sy : target starting coordinate -- need not be on a valid image pixel + Nx, Ny : redundant information UNLESS does exist (in which case it is required information) -> make Nx,Ny optional if exists? + */ int extract (int argc, char **argv) { - int i, j; - float *Vin, *Vout; - int sx, sy, nx, ny, NX, NY; - int Sx, Sy, Nx, Ny; + int N; Buffer *in, *out; + + float initValue = 0.0; + int initOutput = FALSE; + if ((N = get_argument (argc, argv, "-init"))) { + remove_argument (N, &argc, argv); + initValue = atof (argv[N]); + remove_argument (N, &argc, argv); + initOutput = TRUE; + } if (argc != 11) { @@ -15,22 +29,24 @@ if ((in = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - NX = in[0].matrix.Naxis[0]; - NY = in[0].matrix.Naxis[1]; + int NX = in[0].matrix.Naxis[0]; + int NY = in[0].matrix.Naxis[1]; - sx = atof (argv[3]); - sy = atof (argv[4]); - nx = atof (argv[5]); - ny = atof (argv[6]); + int sx = atof (argv[3]); + int sy = atof (argv[4]); + int nx = atof (argv[5]); + int ny = atof (argv[6]); - Sx = atof (argv[7]); - Sy = atof (argv[8]); - Nx = atof (argv[9]); - Ny = atof (argv[10]); + int Sx = atof (argv[7]); + int Sy = atof (argv[8]); + int Nx = atof (argv[9]); + int Ny = atof (argv[10]); if ((Sy + ny > Ny) || (Sx + nx > Nx)) { - gprint (GP_ERR, "mismatch between source and dest regions\n"); + gprint (GP_ERR, "source pixels extend beyond target pixels\n"); gprint (GP_ERR, "%d + %d > %d or %d + %d > %d\n", Sy, ny, Ny, Sx, nx, Nx); - return (FALSE); + // return (FALSE); } + + // XXX : allow source region to fall outside source image /* region is not on first image */ @@ -39,14 +55,10 @@ (sy > in[0].matrix.Naxis[1])) { gprint (GP_ERR, "region outside of source image\n"); - return (FALSE); + // return (FALSE); } - if ((Sx + nx > Nx) || (Sy + ny > Ny)) { - gprint (GP_ERR, "source region larger than dest region\n"); - return (FALSE); - } if ((Sx < 0) || (Sy < 0)) { gprint (GP_ERR, "dest region out of range\n"); - return (FALSE); + // return (FALSE); } @@ -76,13 +88,61 @@ } - for (j = 0; j < ny; j++) { - if (j + sy < 0) continue; - if (j + sy >= NY) continue; - Vin = (float *)(in[0].matrix.buffer) + (j + sy)*in[0].matrix.Naxis[0] + sx; - Vout = (float *)(out[0].matrix.buffer) + (j + Sy)*out[0].matrix.Naxis[0] + Sx; - for (i = 0; i < nx; i++, Vin++, Vout++) { + // (NX,NY) : source image dimensions + // (Nx,Ny) : target image dimensions + + // allow (sx, sy), (Sx, Sy) to be off image + // allow (sx+nx, sy+ny) to be off image + + // if (sx < 0) { + // nx += sx; + // sx = 0; + // } + // if (sx > NX) sx = NX; + // + // if (Sx < 0) { + // Nx += sx; + // sx = 0; + // } + // if (sx > NX) sx = NX; + + float *Vin = (float *)(in[0].matrix.buffer); + float *Vout = (float *)(out[0].matrix.buffer); + + if (initOutput) { + for (int j = 0; j < Ny; j++) { + for (int i = 0; i < Nx; i++) { + Vout[i + Nx*j] = initValue; + } + } + } + + int Nps = NX*NY; + int Npt = Nx*Ny; + + for (int j = 0; j < ny; j++) { + if (j + sy < 0) continue; // not yet on source image + if (j + Sy < 0) continue; // not yet on target image + if (j + sy >= NY) continue; // past edge of source image + if (j + Sy >= Ny) continue; // past edge of target image + + // Vin = (float *)(in[0].matrix.buffer) + (j + sy)*in[0].matrix.Naxis[0] + sx; + // Vout = (float *)(out[0].matrix.buffer) + (j + Sy)*out[0].matrix.Naxis[0] + Sx; + + for (int i = 0; i < nx; i++) { if (i + sx < 0) continue; if (i + sx >= NX) continue; - *Vout = *Vin; + if (i + Sx < 0) continue; + if (i + Sx >= Nx) continue; + + int ps = i + sx + (j + sy)*NX; + int pt = i + Sx + (j + Sy)*Nx; + + myAssert (pt >= 0, "oops 1"); + myAssert (pt < Npt, "oops 2"); + + myAssert (ps >= 0, "oops 3"); + myAssert (ps < Nps, "oops 4"); + + Vout[pt] = Vin[ps]; } } @@ -91,3 +151,2 @@ } - Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/init.c (revision 41170) @@ -3,4 +3,5 @@ int accum PROTO((int, char **)); +int antialias PROTO((int, char **)); int applyfit PROTO((int, char **)); int applyfit1d PROTO((int, char **)); @@ -67,4 +68,5 @@ int join PROTO((int, char **)); int jpeg PROTO((int, char **)); +int kapamemory PROTO((int, char **)); int kern PROTO((int, char **)); int keyword PROTO((int, char **)); @@ -79,8 +81,10 @@ int lookup PROTO((int, char **)); int matrix PROTO((int, char **)); +int match1d PROTO((int, char **)); int match2d PROTO((int, char **)); int mkrgb PROTO((int, char **)); int mcreate PROTO((int, char **)); int medacc PROTO((int, char **)); +int mgaussdev PROTO((int, char **)); int mget PROTO((int, char **)); int mget3d PROTO((int, char **)); @@ -156,4 +160,5 @@ int vgauss PROTO((int, char **)); int vlorentz PROTO((int, char **)); +int vsigmoid PROTO((int, char **)); int vellipse PROTO((int, char **)); int vmaxwell PROTO((int, char **)); @@ -163,7 +168,11 @@ int vzload PROTO((int, char **)); int vstats PROTO((int, char **)); +int vstats PROTO((int, char **)); +int virls PROTO((int, char **)); +int vwtmean PROTO((int, char **)); int vroll PROTO((int, char **)); int vshift PROTO((int, char **)); int vpeaks PROTO((int, char **)); +int vtransitions PROTO((int, char **)); int vpop PROTO((int, char **)); int vsmooth PROTO((int, char **)); @@ -187,4 +196,5 @@ static Command cmds[] = { {1, "accum", accum, "accumulate vector values in another vector"}, + {1, "antialias", antialias, "set anti-alias sigma value for display"}, {1, "applyfit", applyfit1d, "apply 1-d fit to new vector"}, {1, "applyfit1d", applyfit1d, "apply 1-d fit to new vector"}, @@ -255,4 +265,5 @@ {1, "join", join, "find the join of two ID vectors"}, {1, "jpeg", jpeg, "convert display image to JPEG"}, + {1, "kapamemory", kapamemory, "manage kapa memory dump options"}, {1, "kern", kern, "convolve with 3x3 kernel"}, {1, "keyword", keyword, "extract a FITS keyword from image header"}, @@ -265,4 +276,5 @@ {1, "imcreate", mcreate, "create an image"}, {1, "medacc", medacc, "accumulate vector values in another vector"}, + {1, "mgaussdev", mgaussdev, "generate a gaussian deviate image"}, {1, "mget", mget, "extract a vector from an image"}, {1, "mget3d", mget3d, "extract a vector from a 3D image"}, @@ -273,4 +285,5 @@ {1, "medimage", medimage_command, "median image manipulation"}, {1, "matrix", matrix, "matrix math operations"}, + {1, "match1d", match1d, "match 2 vectors and return matched indexes"}, {1, "match2d", match2d, "match 2 pairs of X,Y vectors and return matched indexes"}, {1, "mkrgb", mkrgb, "convert 3 images to rgb jpeg (use Kapa for better control)"}, @@ -347,4 +360,5 @@ {1, "vgauss", vgauss, "fit a Gaussian to a vector"}, {1, "vlorentz", vlorentz, "fit a Lorentzian to a vector"}, + {1, "vsigmoid", vsigmoid, "fit a Sigmoid to a vector"}, {1, "vellipse", vellipse, "fit a Ellipse to a vector pair"}, {1, "vgrid", vgrid, "generate an image from a triplet of vectors"}, @@ -354,5 +368,6 @@ {1, "vload", vload, "load vectors as overlay on image display"}, {1, "vmaxwell", vmaxwell, "fit a Maxwellian to a vector"}, - {1, "vpeaks", vpeaks, "fine coord and flux of peaks in vector"}, + {1, "vpeaks", vpeaks, "find coord and flux of peaks in vector"}, + {1, "vtransitions", vtransitions, "find points in vector that cross the transition value"}, {1, "vpop", vpop, "remove first element of a vector"}, {1, "vroll", vroll, "roll vector elements by 1 entry"}, @@ -360,4 +375,6 @@ {1, "vsmooth", vsmooth, "Gaussian smooth of a vector"}, {1, "vstats", vstats, "statistics on a vector"}, + {1, "vwtmean", vwtmean, "weighted mean of a vector"}, + {1, "virls", virls, "IRLS mean of a vector"}, {1, "vzload", vzload, "load vectors as overlay on image display (scaled points)"}, {1, "vsh", vsh, "Vector Spherical Harmonics"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/interpolate_presort.c (revision 41170) @@ -7,4 +7,11 @@ double x0, dx, dy, y0; Vector *xout, *yout, *xin, *yin; + + int N; + int FillEnds = FALSE; + if ((N = get_argument (argc, argv, "-fill-ends"))) { + FillEnds = TRUE; + remove_argument (N, &argc, argv); + } /** check basic syntax **/ @@ -36,10 +43,12 @@ y0 = yin[0].elements.Flt[0]; - /* in and out vectors are sorted */ - j = 0; + // fill in the start with NANs + for (i = 0; (i < xout[0].Nelements) && (xout[0].elements.Flt[i] < xin[0].elements.Flt[0]); i++) { + yout[0].elements.Flt[i] = FillEnds ? yin[0].elements.Flt[0] : NAN; + } // every output pixel should get a value unless // we are below the lowest in or above the highest in - for (i = 0, j = 0; (i < xout[0].Nelements) && (j < xin[0].Nelements - 1); ) { + for (j = 0; (i < xout[0].Nelements) && (j < xin[0].Nelements - 1); ) { yout[0].elements.Flt[i] = NAN; @@ -66,10 +75,10 @@ // fill in the rest with NANs + int NinLast = xin[0].Nelements - 1; while (i < yout[0].Nelements) { - yout[0].elements.Flt[i] = NAN; + yout[0].elements.Flt[i] = FillEnds ? yin[0].elements.Flt[NinLast] : NAN; i++; } return (TRUE); - } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/kapamemory.c (revision 41170) @@ -0,0 +1,56 @@ +# include "data.h" + +int kapamemory (int argc, char **argv) { + + int N, kapa; + + /* display source */ + char *name = NULL; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + name = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetImage (NULL, &kapa, name)) return (FALSE); + FREE (name); + + if (argc < 2) { + gprint (GP_ERR, "USAGE: kapamemory (mode)\n"); + gprint (GP_ERR, " mode = dump, nlines, exit\n"); + return (FALSE); + } + + if (!strcasecmp(argv[1], "dump")) { + KapaMemoryDump (kapa); + return TRUE; + } + + if (!strcasecmp(argv[1], "exit")) { + if (argc != 3) goto exit_usage; + if (!strcasecmp(argv[2], "on")) { + KapaMemoryDumpOnExit (kapa, TRUE); + return TRUE; + } + if (!strcasecmp(argv[2], "off")) { + KapaMemoryDumpOnExit (kapa, FALSE); + return TRUE; + } + exit_usage: + gprint (GP_ERR, "USAGE: kapamemory exit (status)\n"); + gprint (GP_ERR, " statue = 'on' or 'off'\n"); + return FALSE; + } + + if (!strcasecmp(argv[1], "nlines")) { + if (argc != 3) { + gprint (GP_ERR, "USAGE: kapamemory nlines (N)\n"); + return FALSE; + } + int Nlines = atoi (argv[2]); + KapaMemoryDumpLines (kapa, Nlines); + return TRUE; + } + gprint (GP_ERR, "ERROR: unknown kapamemory mode %s\n", argv[1]); + return FALSE; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/match1d.c (revision 41170) @@ -0,0 +1,248 @@ +# include "data.h" + +int find_matches1d (Vector *X1, Vector *X2, double Radius, Vector *index1, Vector *index2, Vector *radiusMatch); +int find_matches1d_closest (Vector *X1, Vector *X2, double Radius, Vector *index); + +// match1d (X1) (X2) (Radius) [-index1 (index1)] [-index2 (index2)] [-nomatch1 nomatch1] [-nomatch2 nomatch2] +int match1d (int argc, char **argv) { + + int N, CLOSEST; + double Radius; + char *endptr; + Vector *X1vec, *X2vec; + Vector *index1, *index2; + + if ((N = get_argument (argc, argv, "-h"))) goto usage; + if ((N = get_argument (argc, argv, "--help"))) goto usage; + + CLOSEST = FALSE; + if ((N = get_argument (argc, argv, "-closest"))) { + remove_argument (N, &argc, argv); + CLOSEST = TRUE; + } + + if ((N = get_argument (argc, argv, "-index1"))) { + remove_argument (N, &argc, argv); + if ((index1 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index1 = SelectVector ("index1", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + if ((N = get_argument (argc, argv, "-index2"))) { + remove_argument (N, &argc, argv); + if ((index2 = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { + if ((index2 = SelectVector ("index2", ANYVECTOR, TRUE)) == NULL) return (FALSE); + } + + Vector *radiusMatch = NULL; + if ((N = get_argument (argc, argv, "-radius"))) { + if (CLOSEST) { + gprint (GP_ERR, "error: -radius and -closest are currently incompatible\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); + if ((radiusMatch = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 4) { + gprint (GP_ERR, "USAGE: match1d X1 X2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); + gprint (GP_ERR, " use -h or --help for more detail\n"); + return (FALSE); + } + + /* + we have two modes of operation: + + without -closest, we are finding all matched pairs within the match radius. in this + case, the two index vectors have the same length, one entry per matched pair. + x1[index1],y1[index1] matches to x2[index2],y2[index2]. + + with -closest selected, we are finding the closest element of set 1 to each of set 2 + and vice versa. in this case, index1 is always the same length as x1,y1, while index2 + is the same lengths as x2,y2. x2[index1],y2[index1] matches x1,y1 while + x1[index2],y1[index2] matches x2,y2 + + */ + + if ((X1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((X2vec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + REQUIRE_VECTOR_FLT (X1vec, FALSE); + REQUIRE_VECTOR_FLT (X2vec, FALSE); + + Radius = strtod (argv[3], &endptr); + if (*endptr) { + gprint (GP_ERR, "Radius must be numerical (%s)\n", argv[3]); + return (FALSE); + } + + if (CLOSEST) { + find_matches1d_closest (X1vec, X2vec, Radius, index1); + find_matches1d_closest (X2vec, X1vec, Radius, index2); + } else { + find_matches1d (X1vec, X2vec, Radius, index1, index2, radiusMatch); + } + return (TRUE); + +usage: + gprint (GP_ERR, "we have two modes of operation:\n\n"); + + gprint (GP_ERR, "without -closest, we are finding all matched pairs within the match radius. in this\n"); + gprint (GP_ERR, "case, the two index vectors have the same length, one entry per matched pair.\n"); + gprint (GP_ERR, "x1[index1] matches to x2[index2].\n\n"); + + gprint (GP_ERR, "with -closest selected, we are finding the closest element of set 1 to each of set 2\n"); + gprint (GP_ERR, "and vice versa. in this case, index1 is always the same length as x1, while index2\n"); + gprint (GP_ERR, "is the same lengths as x2. x2[index1] matches x1 while\n"); + gprint (GP_ERR, "x1[index2] matches x2\n\n"); + + gprint (GP_ERR, "if -index1 or -index2 is not supplied, the vectors are created with names index1 or index2\n"); + gprint (GP_ERR, "use 'reindex' to generate new vectors based on these index vectors\n"); + + gprint (GP_ERR, "if -radius (vector) is supplied, the vector will be filled with the distance between the matched pairs\n"); + gprint (GP_ERR, " not valid with -closest\n"); + return FALSE; +} + +// Radius is a 1-D separation +int find_matches1d (Vector *X1, Vector *X2, double Radius, Vector *index1, Vector *index2, Vector *radiusMatch) { + + off_t i, j, first_j, I, J, *N1, *N2, Nmatch, NMATCH, DMATCH; + double dX, dR; + + NMATCH = MAX(MAX(0.01*X1->Nelements, 0.01*X2->Nelements), 100); + DMATCH = NMATCH; + + ResetVector (index1, OPIHI_INT, NMATCH); + ResetVector (index2, OPIHI_INT, NMATCH); + if (radiusMatch) ResetVector (radiusMatch, OPIHI_FLT, NMATCH); + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + dsort_indexonly (X1->elements.Flt, N1, X1->Nelements); + dsort_indexonly (X2->elements.Flt, N2, X2->Nelements); + + Nmatch = 0; + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + if (!isfinite(X1->elements.Flt[I])) { i++; continue; } + if (!isfinite(X2->elements.Flt[J])) { j++; continue; } + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { i++; continue; } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for all matches of list2() to list1(i) + first_j = j; + for (j = first_j; (dX > -1.02*Radius) && (j < X2->Nelements); j++) { + J = N2[j]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dR = fabs(dX); + if (dR < Radius) { + index1->elements.Int[Nmatch] = I; + index2->elements.Int[Nmatch] = J; + if (radiusMatch) radiusMatch->elements.Flt[Nmatch] = dR; + + // XXX track matches 1 and 2 with internal vector, save new nomatch index vectors + // after this loop + + Nmatch ++; + if (Nmatch >= NMATCH) { + NMATCH += DMATCH; + REALLOCATE (index1->elements.Int, opihi_int, NMATCH); + REALLOCATE (index2->elements.Int, opihi_int, NMATCH); + if (radiusMatch) { REALLOCATE (radiusMatch->elements.Flt, opihi_flt, NMATCH); } + } + } + } + j = first_j; + i++; + } + index1->Nelements = Nmatch; + index2->Nelements = Nmatch; + if (radiusMatch) radiusMatch->Nelements = Nmatch; + + free (N1); + free (N2); + + return (TRUE); +} + +// find the elements of X2,Y2 which are closest to each element of X1,Y1 (-1 if no match) +int find_matches1d_closest (Vector *X1, Vector *X2, double Radius, Vector *index) { + + off_t i, j, Jmin, Ji, I, J, *N1, *N2, NMATCH; + double dX, dR, Rmin; + + NMATCH = X1->Nelements; + ResetVector (index, OPIHI_INT, NMATCH); + + for (i = 0; i < index->Nelements; i++) { index->elements.Int[i] = -1; } + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + dsort_indexonly (X1->elements.Flt, N1, X1->Nelements); + dsort_indexonly (X2->elements.Flt, N2, X2->Nelements); + + // find the closest entry in list 2 to the current entry in list 1: + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { + // no match in list 2 to this entry + index->elements.Int[I] = -1; + i++; + continue; + } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for closest matches of list2() to list1(i) + Jmin = -1; + for (Ji = j; (dX > -1.02*Radius) && (Ji < X2->Nelements); Ji++) { + J = N2[Ji]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dR = fabs(dX); + if (dR > Radius) continue; + if (dR < Rmin) { + Rmin = dR; + Jmin = J; + } + } + + // no match in list 2 to this entry + if (Jmin == -1) { + index->elements.Int[I] = -1; + i++; + continue; + } + index->elements.Int[I] = Jmin; + i++; + } + index->Nelements = NMATCH; + + free (N1); + free (N2); + + return (TRUE); +} + + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_calc.c (revision 41170) @@ -0,0 +1,365 @@ +# include "data.h" + +float weight_cauchy_square_flt (float x2); +float irls_mean (float *val, float *wgt, int N, float *outvar, int *Npt); +float irls_fraction_interpolate (float *values, float fraction, int Npts); +float irls_robust_stdev (float *values, int Npts); +float irls_wtmean (float *val, float *wgt, int *idx, int Npt, float *variance); +void irls_bootstrap (int *idx, int Npts); +void irls_init (int N); +void irls_free (void); + +# define IRLS_TOLERANCE 1e-4 +int BOOTSTRAP = FALSE; +int BOOTSTRAP_NITER = 100; + +static float *irls_valsub = NULL; +static float *irls_wgtsub = NULL; +static int *irls_idx = NULL; +static float *irls_testval = NULL; + +enum {CALC_MEDIAN, CALC_MEAN, CALC_IRLS, CALC_WTMEAN}; + +int medimage_calc (int argc, char **argv) { + + int ix, iy, n, N; + Buffer *output; + + BOOTSTRAP = FALSE; + if ((N = get_argument (argc, argv, "-bootstrap"))) { + BOOTSTRAP = TRUE; + remove_argument (N, &argc, argv); + } + BOOTSTRAP_NITER = 100; + if ((N = get_argument (argc, argv, "-bootstrap-iter"))) { + remove_argument (N, &argc, argv); + BOOTSTRAP_NITER = atoi (argv[N]); + remove_argument (N, &argc, argv); + BOOTSTRAP = TRUE; + } + + int mode = CALC_MEDIAN; + if ((N = get_argument (argc, argv, "-mean"))) { + mode = CALC_MEAN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-irls"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_IRLS; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-wtmean"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_WTMEAN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-median"))) { + if (mode != CALC_MEDIAN) { gprint (GP_ERR, "supply only one of -mean, -median, -irls, -wtmean\n"); return FALSE; } + mode = CALC_MEDIAN; + remove_argument (N, &argc, argv); + } + + Buffer *variance = NULL; + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((variance = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage calc (name) (output) [-mean,-irls,-wtmean,-median] [-variance output] [-bootstrap] [-bootstrap-iter (N)]\n"); + gprint (GP_ERR, " calculate the median image for the median image set\n"); + gprint (GP_ERR, " This function calculates the 'average' image for the supplied set of images.\n"); + gprint (GP_ERR, " Four options are available to calculate the average:\n"); + gprint (GP_ERR, " -mean (straight arithmetic mean)\n"); + gprint (GP_ERR, " -median\n"); + gprint (GP_ERR, " -wtmean (arithmetic mean, weighted by supplied variances)\n"); + gprint (GP_ERR, " -irls (iteratively-reweighted least-squares)\n"); + gprint (GP_ERR, " If -variance is supplied the returned image represents the estimate of the variance on the average image\n"); + gprint (GP_ERR, " For the four average methods above, the sqrt of the variance has the following meanings:\n"); + gprint (GP_ERR, " -mean : standard deviation / sqrt(Npts)\n"); + gprint (GP_ERR, " -median : standard deviation / sqrt(Npts)\n"); + gprint (GP_ERR, " -wtmean : formal error on the weighted mean [sum (1 / variances)]\n"); + gprint (GP_ERR, " -irls : formal error on the pixels used (excluding ~3 sigma outliers)\n"); + gprint (GP_ERR, " note that the irls formal error slightly underestimates the observed standard deviation\n"); + return FALSE; + } + + MedImageType *median = FindMedImage (argv[1]); + if (!median) { + gprint (GP_ERR, "median image %s not found\n", argv[1]); + return FALSE; + } + + if ((output = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) return (FALSE); + + int Ninput = median->Ninput; + int Nx = median->Nx; + int Ny = median->Ny; + + ALLOCATE_PTR (val, float, Ninput); + ALLOCATE_PTR (wgt, float, Ninput); + + if (mode == CALC_IRLS) irls_init (Ninput); + + gfits_free_matrix (&output->matrix); + gfits_free_header (&output->header); + if (!CreateBuffer (output, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + + float *outvalue = (float *) output->matrix.buffer; + float *varvalue = NULL; + if (variance) { + gfits_free_matrix (&variance->matrix); + gfits_free_header (&variance->header); + if (!CreateBuffer (variance, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + + varvalue = (float *) variance->matrix.buffer; + } + + // save the number of points per pixel + Buffer *nptbuf = SelectBuffer ("irls_npt", ANYBUFFER, TRUE); + gfits_free_matrix (&nptbuf->matrix); + gfits_free_header (&nptbuf->header); + if (!CreateBuffer (nptbuf, Nx, Ny, -32, 0.0, 1.0)) return FALSE; + float *NptVal = (float *) nptbuf->matrix.buffer; + + for (iy = 0; iy < Ny; iy++) { + for (ix = 0; ix < Nx; ix++) { + + int N = 0; + int Npix = ix + Nx*iy; + for (n = 0; n < Ninput; n++) { + float v = median->flx[n][Npix]; + if (!isfinite(v)) continue; + val[N] = v; + wgt[N] = 1.0; + if (median->var[n]) { + float s = median->var[n][Npix]; + if (!isfinite(s)) continue; + if (fabs(s) < 2*FLT_MIN) s = 2*FLT_MIN; + wgt[N] = 1.0 / s; + } + N++; + } + if (N == 0) continue; + + switch (mode) { + case CALC_MEDIAN: + fsort (val, N); + outvalue[Npix] = val[(int)(0.5*N)]; + if (varvalue) { + float sum = 0.0; + for (n = 0; n < N; n++) { + sum += SQ(val[n] - outvalue[Npix]); + } + // variance on the mean (stdev / sqrt(N))^2 + varvalue[Npix] = sum / (N - 1) / N; + } + break; + case CALC_MEAN: { + float sum = 0.0; + for (n = 0; n < N; n++) { + sum += val[n]; + } + outvalue[Npix] = sum / (float) N; + + if (varvalue) { + float Sum = 0.0; + for (n = 0; n < N; n++) { + Sum += SQ(val[n] - outvalue[Npix]); + } + // variance on the mean (stdev / sqrt(N))^2 + varvalue[Npix] = Sum / (N - 1) / N; + } + break; + } + case CALC_WTMEAN: { + float variance; + outvalue[Npix] = irls_wtmean (val, wgt, NULL, N, &variance); + if (varvalue) { varvalue[Npix] = variance; } + break; + } + case CALC_IRLS: + if (varvalue) { + int Npts = 0; + outvalue[Npix] = irls_mean (val, wgt, N, &varvalue[Npix], &Npts); + NptVal[Npix] = Npts; + } else { + int Npts = 0; + outvalue[Npix] = irls_mean (val, wgt, N, NULL, &Npts); + NptVal[Npix] = Npts; + } + } + } + } + irls_free(); + return TRUE; +} + +// allocate temporary vectors +void irls_init (int N) { + + ALLOCATE (irls_valsub, float, N); + ALLOCATE (irls_wgtsub, float, N); + ALLOCATE (irls_idx, int, N); + ALLOCATE (irls_testval, float, BOOTSTRAP_NITER); +} + +// free temporary vectors +void irls_free (void) { + FREE (irls_valsub); + FREE (irls_wgtsub); + FREE (irls_idx); + FREE (irls_testval); + irls_valsub = NULL; + irls_wgtsub = NULL; + irls_idx = NULL; + irls_testval = NULL; +} + +float irls_mean (float *val, float *wgt, int N, float *outvar, int *Npt) { + + // calculate weighted mean + float Value = irls_wtmean (val, wgt, NULL, N, NULL); + + int converged = FALSE; + for (int i = 0; (i < 10) && !converged; i++) { + float ValueLast = Value; + float S1 = 0.0, S2 = 0.0; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int n = 0; n < N; n++) { + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + S1 += Mod * wgt[n] * val[n]; + S2 += Mod * wgt[n]; + } + Value = S1 / S2; + + float delta = fabs(Value - ValueLast); + if (delta < Value * IRLS_TOLERANCE) converged = TRUE; + } + + if (outvar) { + if (BOOTSTRAP) { + // generate a subset vector of just the accepted points + + // calculate stdev of high-weight points + int npt = 0; + for (int n = 0; n < N; n++) { + + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) continue; // totally ad-hoc number + + irls_valsub[npt] = val[n]; + irls_wgtsub[npt] = wgt[n]; + npt ++; + } + + for (int iter = 0; iter < BOOTSTRAP_NITER; iter++) { + irls_bootstrap (irls_idx, npt); // fill idx with the index of the resampled points + irls_testval[iter] = irls_wtmean (irls_valsub, irls_wgtsub, irls_idx, npt, NULL); + } + + float sigma = irls_robust_stdev (irls_testval, BOOTSTRAP_NITER); + + *outvar = SQ(sigma); + *Npt = npt; + } else { + // calculate stdev of high-weight points + float S1 = 0.0, S2 = 0.0; + int npt = 0; + for (int n = 0; n < N; n++) { + + float dV = (val[n] - Value); + float d2 = SQ(dV) * wgt[n]; + + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) continue; // totally ad-hoc number + + S1 += SQ(dV); + S2 += wgt[n]; + npt ++; + } + *Npt = npt; + *outvar = 1 / S2; + } + } + + return Value; +} + +void irls_bootstrap (int *idx, int Npts) { + // generate an index (idx) containing the index values + // for the Npts randomly resampled points + for (int i = 0; i < Npts; i++) { + idx[i] = Npts * drand48(); + } +} + +float irls_wtmean (float *val, float *wgt, int *idx, int Npt, float *variance) { + float S1 = 0.0, S2 = 0.0; + for (int n = 0; n < Npt; n++) { + int N = idx ? idx[n] : n; + S1 += wgt[N] * val[N]; + S2 += wgt[N]; + } + float Value = S1 / S2; + if (variance) *variance = 1.0 / S2; + return Value; +} + +float irls_robust_stdev (float *values, int Npts) { + + fsort (values, Npts); + + float Slo = irls_fraction_interpolate (values, 0.158655, Npts); + float Shi = irls_fraction_interpolate (values, 0.841345, Npts); + float sigma = (Shi - Slo) / 2.0; + + return sigma; +} + +float irls_fraction_interpolate (float *values, float fraction, int Npts) { + + float F = fraction * Npts; + int N = fraction * Npts; + + if (N < 0 ) return NAN; + if (N >= Npts - 2) return NAN; + + // interpolate between N,N+1 + + float S = (F - N) * (values[N+1] - values[N]) + values[N]; + return S; +} + + +// exp(-(x^2/s^2)/2) = (1/2) +// -(x^2/s^2)/2 = ln(1/2) +// (x^2/s^2)/2 = ln(2) +// (x^2/s^2) = 2ln(2) +// (x /s) = sqrt(2ln(2)) : half-width at half-max +// FWHM = 2sqrt(2ln(2)) + +// R2 = (X / 2.385)^2 = (X^2 / 2.385^2) + +# define CAUCY_FACTOR 2.0 + +/* +float weight_cauchy_square_flt (float x2) { + return (1.0); +} +*/ + +float weight_cauchy_square_flt (float x2) { + float r2 = x2 / CAUCY_FACTOR; + return (1.0 / (1.0 + r2)); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/medimage_commands.c (revision 41170) @@ -15,13 +15,30 @@ int medimage_add (int argc, char **argv) { + int N; Buffer *image; + Buffer *var = NULL; + + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((var = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage add (name) (image)\n"); + gprint (GP_ERR, "USAGE: medimage add (name) (image) [-variance variance]\n"); gprint (GP_ERR, " add the given image to the set of images to be medianed\n"); + gprint (GP_ERR, " optionally supply variance image (for weighted calculations)\n"); return FALSE; } if ((image = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + if (var) { + if ((var->matrix.Naxis[0] != image->matrix.Naxis[0]) || + (var->matrix.Naxis[1] != image->matrix.Naxis[1])) { + gprint (GP_ERR, "variance buffer does not match image buffer dimensions\n"); + return FALSE; + } + } MedImageType *median = FindMedImage (argv[1]); @@ -46,76 +63,17 @@ // new image should match existing medimage dimensions + // AddMedImage (median, image, var); int Ninput = median->Ninput; median->Ninput ++; - REALLOCATE (median->buffers, float *, median->Ninput); + REALLOCATE (median->flx, float *, median->Ninput); + REALLOCATE (median->var, float *, median->Ninput); - ALLOCATE (median->buffers[Ninput], float, median->Nx*median->Ny); - memcpy (median->buffers[Ninput], image->matrix.buffer, sizeof(float)*median->Nx*median->Ny); + ALLOCATE (median->flx[Ninput], float, median->Nx*median->Ny); + memcpy (median->flx[Ninput], image->matrix.buffer, sizeof(float)*median->Nx*median->Ny); - return TRUE; -} - -int medimage_calc (int argc, char **argv) { - - int ix, iy, n, N; - Buffer *output; - - int CALC_MEAN = FALSE; - if ((N = get_argument (argc, argv, "-mean"))) { - CALC_MEAN = TRUE; - remove_argument (N, &argc, argv); - } - - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage calc (name) (output) [-mean]\n"); - gprint (GP_ERR, " calculate the median image for the median image set\n"); - return FALSE; - } - - MedImageType *median = FindMedImage (argv[1]); - if (!median) { - gprint (GP_ERR, "median image %s not found\n", argv[1]); - return FALSE; - } - - if ((output = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) return (FALSE); - - int Ninput = median->Ninput; - int Nx = median->Nx; - int Ny = median->Ny; - - float *value = NULL; - ALLOCATE (value, float, Ninput); - - gfits_free_matrix (&output->matrix); - gfits_free_header (&output->header); - if (!CreateBuffer (output, Nx, Ny, -32, 0.0, 1.0)) return FALSE; - - float *outvalue = (float *) output->matrix.buffer; - - for (iy = 0; iy < Ny; iy++) { - for (ix = 0; ix < Nx; ix++) { - - int N = 0; - int Npix = ix + Nx*iy; - for (n = 0; n < Ninput; n++) { - float v = median->buffers[n][Npix]; - if (!isfinite(v)) continue; - value[N] = v; - N++; - } - if (N == 0) continue; - - if (CALC_MEAN) { - float sum = 0.0; - for (n = 0; n < N; n++) { - sum += value[n]; - } - outvalue[Npix] = sum / (float) N; - } else { - fsort (value, N); - outvalue[Npix] = value[(int)(0.5*N)]; - } - } + median->var[Ninput] = NULL; + if (var) { + ALLOCATE (median->var[Ninput], float, median->Nx*median->Ny); + memcpy (median->var[Ninput], var->matrix.buffer, sizeof(float)*median->Nx*median->Ny); } @@ -124,39 +82,39 @@ /* -int medimage_save (int argc, char **argv) { + int medimage_save (int argc, char **argv) { - int N; + int N; - int APPEND = FALSE; - if ((N = get_argument (argc, argv, "-append"))) { - APPEND = TRUE; - remove_argument (N, &argc, argv); - } + int APPEND = FALSE; + if ((N = get_argument (argc, argv, "-append"))) { + APPEND = TRUE; + remove_argument (N, &argc, argv); + } - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage save (name) (filename) [-append]\n"); - return FALSE; - } + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage save (name) (filename) [-append]\n"); + return FALSE; + } - if (!SaveMedImage(argv[2], argv[1], APPEND)) { - gprint (GP_ERR, "failed to save medimage %s\n", argv[1]); - return (FALSE); - } - return TRUE; -} + if (!SaveMedImage(argv[2], argv[1], APPEND)) { + gprint (GP_ERR, "failed to save medimage %s\n", argv[1]); + return (FALSE); + } + return TRUE; + } -int medimage_load (int argc, char **argv) { + int medimage_load (int argc, char **argv) { - if (argc != 3) { - gprint (GP_ERR, "USAGE: medimage load (name) (filename)\n"); - return FALSE; - } + if (argc != 3) { + gprint (GP_ERR, "USAGE: medimage load (name) (filename)\n"); + return FALSE; + } - if (!LoadMedImage(argv[2], argv[1])) { - gprint (GP_ERR, "failed to load medimage %s\n", argv[1]); - return (FALSE); - } - return TRUE; -} + if (!LoadMedImage(argv[2], argv[1])) { + gprint (GP_ERR, "failed to load medimage %s\n", argv[1]); + return (FALSE); + } + return TRUE; + } */ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/mgaussdev.c (revision 41170) @@ -0,0 +1,34 @@ +# include "data.h" + +int mgaussdev (int argc, char **argv) { + + Buffer *buf; + + if (argc != 6) goto usage; + + if ((buf = SelectBuffer (argv[1], ANYBUFFER, TRUE)) == NULL) return (FALSE); + int Nx = atof (argv[2]); + int Ny = atof (argv[3]); + + double mean = atof (argv[4]); + double sigma = atof (argv[5]); + + /* I should encapsulate this in a create_default_buffer */ + gfits_free_matrix (&buf[0].matrix); + gfits_free_header (&buf[0].header); + + // 3D CUBE OPTION: if (!CreateBuffer3D (buf, Nx, Ny, Nz, -32, 1.0, 0.0)) return FALSE; + if (!CreateBuffer (buf, Nx, Ny, -32, 1.0, 0.0)) return FALSE; + + ohana_gaussdev_init (); + + float *v = (float *) buf[0].matrix.buffer; + for (int i = 0; i < Nx*Ny; i++, v++) { + *v = ohana_gaussdev_rnd (mean, sigma); + } + return (TRUE); + + usage: + gprint (GP_ERR, "USAGE: mgaussdev (buff) Nx Ny mean sigma\n"); + return (FALSE); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/peak.c (revision 41170) @@ -39,4 +39,5 @@ if (*X < start) continue; if (*X > end) continue; + if (!isfinite(*Y)) continue; if (*Y < ymax) continue; xmax = *X; @@ -63,4 +64,5 @@ if (*X < start) continue; if (*X > end) continue; + if (!isfinite(*Y)) continue; if (*Y < ymax) continue; xmax = *X; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/roll.c (revision 41170) @@ -3,7 +3,9 @@ int roll (int argc, char **argv) { - int Nbytes, Nextra; - int dX, dx, dy, nx, ny; Buffer *buf; + + // this function is probably not needed + gprint (GP_ERR, "ERROR: use 'shift' instead of 'roll'\n"); + return (FALSE); if (argc != 4) { @@ -14,32 +16,71 @@ if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - dx = atof (argv[2]); - dy = atof (argv[3]); - if (dy != 0) { - gprint (GP_ERR, "only x rolls implemented for now\n"); + int dx = atoi (argv[2]); + int dy = atoi (argv[3]); + + /* if (dx,dy < 0), we are moving the start position backwards, + if (dx,dy > 0), we are moving the start position forward */ + + int Nx = buf[0].matrix.Naxis[0]; + int Ny = buf[0].matrix.Naxis[1]; + + if (dy == 0) { + // moves in just dx can be done row-by-row + + int dX = abs(dx); + int Nbytes = (Nx * Ny - dX) * sizeof (float); + int Nextra = dX * sizeof (float); + + if (dx < 0) { + memmove (buf[0].matrix.buffer, &buf[0].matrix.buffer[dX*sizeof(float)], Nbytes); + memset (&buf[0].matrix.buffer[Nbytes], 0, Nextra); + } else { + memmove (&buf[0].matrix.buffer[dX*sizeof(float)], buf[0].matrix.buffer, Nbytes); + memset (buf[0].matrix.buffer, 0, Nextra); + } + + return TRUE; } - /* if (dx < 0), we are moving the start position back by dx pixels, - if (dx > 0), we are moving the start position forward by dx pixels */ - - nx = buf[0].matrix.Naxis[0]; - ny = buf[0].matrix.Naxis[1]; + if (dx == 0) { + // moves in just dy can be done row-by-row - dX = abs(dx); - Nbytes = nx * ny * sizeof (float); - Nextra = (nx * ny + dX) * sizeof (float); + ALLOCATE_PTR (output, float, Nx*Ny); - REALLOCATE (buf[0].matrix.buffer, char, Nextra); + float *input = (float *) buf[0].matrix.buffer; - if (dx < 0) { - memmove (buf[0].matrix.buffer, &buf[0].matrix.buffer[dX*sizeof(float)], Nbytes); - } else { - memmove (&buf[0].matrix.buffer[dX*sizeof(float)], buf[0].matrix.buffer, Nbytes); + for (int iy = 0; iy < Ny; iy++) { + if (iy + dy < 0) continue; + if (iy + dy >= Ny) continue; + memcpy (&output[iy*Nx], &input[(iy + dy)*Nx], Nx); + } + if (dy > 0) { + memset (output, 0, dy*Nx*sizeof(float)); + } else { + int Nbytes = Nx * (Ny - abs(dy)) * sizeof (float); + int Nextra = abs(dy) * Nx * sizeof (float); + memset (&output[Nbytes], 0, Nextra); + } + free (buf[0].matrix.buffer); + buf[0].matrix.buffer = (char *) output; + return TRUE; } - REALLOCATE (buf[0].matrix.buffer, char, Nbytes); + ALLOCATE_PTR (output, float, Nx*Ny); + float *input = (float *) buf[0].matrix.buffer; - return (TRUE); - + for (int iy = 0; iy < Ny; iy++) { + if (iy + dy < 0) continue; + if (iy + dy >= Ny) continue; + for (int ix = 0; ix < Nx; ix++) { + if (ix + dx < 0) continue; + if (ix + dx >= Nx) continue; + output[ix + iy*Nx] = input[(ix + dx) + (iy + dy)*Nx]; + } + } + free (buf[0].matrix.buffer); + buf[0].matrix.buffer = (char *) output; + return TRUE; } + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/rotate.c (revision 41170) @@ -3,19 +3,19 @@ int rotate (int argc, char **argv) { - int i, j, NX, NY, X, Y, Lx, Ly, N; + int i, j, NX, NY; float *in_buff, *out_buff, *c; - double angle, CosAngle, SinAngle, Xo, Yo, dX, dY, fx, fy, x, y, X1, Y1; + double Xo, Yo, x, y, X1, Y1; double pc11, pc12, pc21, pc22, PC11, PC12, PC21, PC22; Buffer *buf; - Xo = 0; - Yo = 0; - if ((N = get_argument (argc, argv, "-center"))) { - remove_argument (N, &argc, argv); - Xo = atof(argv[N]); - remove_argument (N, &argc, argv); - Yo = atof(argv[N]); - remove_argument (N, &argc, argv); - } +// Xo = 0; +// Yo = 0; +// if ((N = get_argument (argc, argv, "-center"))) { +// remove_argument (N, &argc, argv); +// Xo = atof(argv[N]); +// remove_argument (N, &argc, argv); +// Yo = atof(argv[N]); +// remove_argument (N, &argc, argv); +// } if (argc != 3) { @@ -221,30 +221,26 @@ } - angle = atof (argv[2]); - CosAngle = cos (angle*RAD_DEG); - SinAngle = sin (angle*RAD_DEG); + double angle = atof (argv[2]); + double CosAngle = cos (angle*RAD_DEG); + double SinAngle = sin (angle*RAD_DEG); gprint (GP_ERR, "rotating: %f %f %f\n", angle, CosAngle, SinAngle); - Lx = NX*fabs(CosAngle) + NY*fabs(SinAngle); - Ly = NX*fabs(SinAngle) + NY*fabs(CosAngle); - dX = MAX(0,NY*SinAngle); - dY = MAX(0,-NX*SinAngle); - /* - gprint (GP_ERR, "%f %f --> ", Xo, Yo); - X1 = Xo*CosAngle - Yo*SinAngle + dX; - Y1 = Xo*SinAngle + Yo*CosAngle + dY; - gprint (GP_ERR, "%f %f\n", X1, Y1); - */ + // we are rotating about the center pixel, (NX/2, NY/2), + // but we are then putting the result in a new image + // of size (Lx,Ly). + + // (x,y) = (i - Nx/2),(j - Ny/2) + // (x',y') = R(theta) (x,y) + // (I,J) = (x' + Lx/2),(y' + Ly/2) + + int Lx = NX*fabs(CosAngle) + NY*fabs(SinAngle); + int Ly = NX*fabs(SinAngle) + NY*fabs(CosAngle); /* fix reference pixel */ gfits_scan (&buf[0].header, "CRPIX1", "%lf", 1, &Xo); gfits_scan (&buf[0].header, "CRPIX2", "%lf", 1, &Yo); - /* - X1 = (Xo - dX)*CosAngle - (Yo - dY)*SinAngle; - Y1 = (dX - Xo)*SinAngle + (Yo - dY)*CosAngle; - */ - X1 = Xo*CosAngle - Yo*SinAngle + dX; - Y1 = Xo*SinAngle + Yo*CosAngle + dY; + X1 = (Xo - NX/2)*CosAngle + (Yo - NY/2)*SinAngle + Lx/2; + Y1 = (NX/2 - Xo)*SinAngle + (Yo - NY/2)*CosAngle + Ly/2; gfits_modify (&buf[0].header, "CRPIX1", "%lf", 1, X1); gfits_modify (&buf[0].header, "CRPIX2", "%lf", 1, Y1); @@ -255,8 +251,8 @@ gfits_scan (&buf[0].header, "PC002001", "%lf", 1, &pc21); gfits_scan (&buf[0].header, "PC002002", "%lf", 1, &pc22); - PC11 = pc11*CosAngle - pc21*SinAngle; - PC12 = pc12*CosAngle - pc22*SinAngle; - PC21 = pc21*CosAngle + pc11*SinAngle; - PC22 = pc22*CosAngle + pc12*SinAngle; + PC11 = pc11*CosAngle + pc21*SinAngle; + PC12 = pc12*CosAngle + pc22*SinAngle; + PC21 = pc21*CosAngle - pc11*SinAngle; + PC22 = pc22*CosAngle - pc12*SinAngle; gfits_modify (&buf[0].header, "PC001001", "%le", 1, PC11); gfits_modify (&buf[0].header, "PC001002", "%le", 1, PC12); @@ -270,22 +266,32 @@ gfits_create_matrix (&buf[0].header, &buf[0].matrix); gfits_print_alt (&buf[0].header, "HISTORY", "%S", 1, "WARNING: rotated image!"); + out_buff = (float *)buf[0].matrix.buffer; for (j = 0; j < Ly; j++) { for (i = 0; i < Lx; i++, out_buff++) { - x = (i - dX)*CosAngle + (j - dY)*SinAngle; - y = (dX - i)*SinAngle + (j - dY)*CosAngle; - X = (int) x; - Y = (int) y; - - if (X < 0) continue; - if (X >= NX - 1) continue; - if (Y < 0) continue; - if (Y >= NY - 1) continue; - - c = &in_buff[X + NX*Y]; - fx = x - X; - fy = y - Y; + float xo = (i - Lx/2); + float yo = (j - Ly/2); + + x = xo*CosAngle + yo*SinAngle; + y = -xo*SinAngle + yo*CosAngle; + + int I = x + NX/2; + int J = y + NY/2; + + if (I < 0) continue; + if (I >= NX - 1) continue; + if (J < 0) continue; + if (J >= NY - 1) continue; + + c = &in_buff[I + NX*J]; + + int X = (int) x; + int Y = (int) y; + double fx = x - X; + double fy = y - Y; + *out_buff = (c[0]*(1-fx) + c[1]*fx)*(1-fy) + (c[NX+1]*fx + c[NX]*(1-fx))*fy; +// *out_buff = c[0]; } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline.c (revision 41170) @@ -7,4 +7,5 @@ int spline_load (int argc, char **argv); int spline_save (int argc, char **argv); +int spline_print (int argc, char **argv); int spline_delete (int argc, char **argv); int spline_rename (int argc, char **argv); @@ -18,4 +19,5 @@ {1, "load", spline_load, "write a spline to a FITS file"}, {1, "save", spline_save, "read a spline from a FITS file"}, + {1, "print", spline_print, "print a spline to stdout"}, {1, "delete", spline_delete, "delete a spline"}, {1, "rename", spline_rename, "rename a spline"}, @@ -35,4 +37,5 @@ gprint (GP_ERR, " spline load (spline) (filename) : load a spline from a FITS file\n"); gprint (GP_ERR, " spline save (spline) (filename) [-append] : save a spline in FITS format\n"); + gprint (GP_ERR, " spline print (spline) : print a spline to stdout\n"); return FALSE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/spline_commands.c (revision 41170) @@ -28,4 +28,9 @@ if (xvec->Nelements != yvec->Nelements) { gprint (GP_ERR, "x and y vectors must be the same length\n"); + return FALSE; + } + + if (xvec->Nelements < 3) { + gprint (GP_ERR, "cannot make a spline with fewer than 3 knots\n"); return FALSE; } @@ -93,4 +98,26 @@ } +int spline_print (int argc, char **argv) { + + int i; + + if (argc != 2) { + gprint (GP_ERR, "USAGE: spline print (name)\n"); + return FALSE; + } + + Spline *myspline = FindSpline (argv[1]); + if (!myspline) { + gprint (GP_ERR, "spline %s not found\n", argv[1]); + return (FALSE); + } + + for (i = 0; i < myspline->Nknots; i++) { + gprint (GP_LOG, "%lf : %lf : %lf\n", myspline->xk[i], myspline->yk[i], myspline->y2[i]); + } + + return TRUE; +} + int spline_load (int argc, char **argv) { @@ -109,6 +136,11 @@ int spline_delete (int argc, char **argv) { - int status; - Spline *spline; + int N, status; + + int QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } if (argc != 2) { @@ -117,6 +149,7 @@ } - spline = FindSpline (argv[1]); + Spline *spline = FindSpline (argv[1]); if (spline == NULL) { + if (QUIET) return TRUE; gprint (GP_ERR, "spline %s not found\n", argv[1]); return FALSE; @@ -130,6 +163,4 @@ int spline_rename (int argc, char **argv) { - Spline *spline; - if (argc != 3) { gprint (GP_ERR, "USAGE: spline rename (spline) (newname)\n"); @@ -137,8 +168,13 @@ } - spline = FindSpline (argv[1]); + Spline *spline = FindSpline (argv[1]); if (spline == NULL) { gprint (GP_ERR, "spline %s not found\n", argv[1]); return FALSE; + } + + Spline *newname = FindSpline (argv[2]); + if (newname != NULL) { + DeleteSpline (newname); } @@ -147,24 +183,2 @@ return TRUE; } - -/* -int spline_listspline (int argc, char **argv) { - - Spline *spline; - - if (argc != 2) { - gprint (GP_ERR, "USAGE: spline listspline (spline)\n"); - return FALSE; - } - - spline = FindSpline (argv[1]); - if (spline == NULL) { - gprint (GP_ERR, "spline %s not found\n", argv[1]); - return FALSE; - } - - ListPages (spline); - return TRUE; -} -*/ - Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/cut.sh (revision 41170) @@ -4,4 +4,46 @@ memtest1 end + +macro test2 + + mgaussdev z 50 100 0.0 1.0 + + cut z x y_sum x 0 0 z[][0] z[0][] + cut z x y_median x 0 0 z[][0] z[0][] -median + cut z x y_mean x 0 0 z[][0] z[0][] -mean + cut z x y_inner x 0 0 z[][0] z[0][] -inner + +end + +macro checkrange + if ($0 != 2) + echo "USAGE: checkrange (Npts)" + break + end + + $Npts = $1 + if ($Npts % 2) + $Ncenter = int($Npts / 2) + $Nquarter = int(0.25 * $Npts) + $Ns = $Ncenter - $Nquarter + $Ne = $Ncenter + $Nquarter + else + $Ncenter = int($Npts / 2) - 1 + $Nquarter = int(0.25 * $Npts) + $Ns = $Ncenter - $Nquarter + $Ne = $Ncenter + $Nquarter + 1 + end + if ($Ns < 0) + echo "error: Ns < 0: $Ns $Ncenter $Nquarter" + end + if ($Ne >= $Npts) + echo "error: Ne >= Npts: $Ne $Ncenter $Nquarter" + end + echo "$Npts : $Ncenter : $Nquarter : $Ns $Ne" + for i 0 $Npts + echo $i {($i >= $Ns) && ($i <= $Ne)} + end +end + # Test if cut works @@ -37,2 +79,42 @@ end end + +macro memtest1 + + $i = 0 + mcreate tim 100 100 + cut tim xdir imx X 40 4 60 6 + # do one to set up memory that should stay used + + memory check + for i 0 100 + cut tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut tim xdir imx y 40 4 60 6 + end + memory check + + for i 0 100 + cut -median tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut -median tim xdir imx y 40 4 60 6 + end + memory check + + for i 0 100 + cut -mean tim xdir imx X 40 4 60 6 + end + memory check + + for i 0 100 + cut -mean tim xdir imx y 40 4 60 6 + end + memory check + +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/delete.sh (revision 41170) @@ -49,2 +49,13 @@ end + +# Test delete +macro test2 + + $PASS = 1 + + for i 0 1000 + create v 0 200 + delete v + end +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/headtest.sh (revision 41170) @@ -0,0 +1,15 @@ + +macro init.test + create x 0 10 + set y = x^2 + mcreate blank 0 0 + keyword blank TE0 -wd 56 + keyword blank TE1 -w "TEST LINE" + keyword blank TE2 -wf 5.12342341 + keyword blank TE3 -wb T + keyword blank TE4 -wc "not sure what do say" + keyword blank TE4 -wf 4.5 + keyword blank TE5 -ws "this is a long comment" + keyword blank TE2 -wc "this is a short comment" + write -fits DATA test2.fits x y -header blank +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/medimage.sh (revision 41170) @@ -1,11 +1,540 @@ -macro go - mcreate a 30 30 - for i 0 40 - set a$i = zero(a) + $i - end - - for i 0 40 - medimage add t1 a$i - end -end +macro test.mean + + $Nsample = 16 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 1.0 + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + medimage calc t1 T -mean + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "expect {1/sqrt($Nsample)} : $C1" + plot -c red -x line x yf +end + +macro test.median + + $Nsample = 16 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 1.0 + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "expect {1/sqrt($Nsample)} : $C1 (actually should be a bit higher)" + plot -c red -x line x yf +end + +macro test.wtmean + + $Nsample = 8 + $sig1 = 1.0 + $sig2 = 3.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig1 + set v = $sig1^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig2 + set v = $sig2^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -wtmean + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + $S1 = $Nsample / $sig1^2 + $Nsample / $sig2^2 + echo "expect {1/sqrt($S1)} : $C1" + plot -c red -x line x yf +end + +macro test.irls + medimage delete -q t1 + $Nsample = 16 + $sig = 1.0 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf +end + + +###################33 + + +macro test.mean.var + + $Nsample = 64 + $sig = 2.0 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 100 100 0.0 $sig + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + medimage calc t1 T -mean -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + imhist Tv xv yv -range -1 4 -delta 0.1 + lim -n 2 xv yv; clear; box; plot xv yv -x hist + + stat Tv + echo "$C1 vs {sqrt($MEDIAN)} : expect {$sig/sqrt($Nsample)}" +end + +macro test.median.var + + $Nsample = 64 + $sig = 2.0 + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + medimage add t1 t + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + imhist Tv xv yv -range -1 4 -delta 0.1 + lim -n 2 xv yv; clear; box; plot xv yv -x hist + + stat Tv + echo "$C1 vs {sqrt($MEDIAN)} : expect {$sig/sqrt($Nsample)}" +end + +macro test.wtmean.var + + $Nsample = 32 + $sig1 = 1.0 + $sig2 = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig1 + set v = $sig1^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig2 + set v = $sig2^2 + zero(t) + medimage add t1 t -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # note that median of a gaussian distributed variable is not distributed with sigma' = sigma / sqrt(N) + # (somewhat higher scatter) + medimage calc t1 T -wtmean -variance Tv + + imhist -q T x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + plot -c red -x line x yf + + stat -q Tv + $S1 = $Nsample / $sig1^2 + $Nsample / $sig2^2 + echo $C1 vs {sqrt($MEDIAN)} : expect {1/sqrt($S1)} +end + +macro test.irls.var + + $Nsample = 16 + $sig = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 50 50 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range -1 4 -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} (ideal is {$sig/sqrt($Nsample)})" +end + +macro test.irls.boot.var + + $Nsample = 64 + $sig = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 200 200 0.0 $sig + set v = $sig^2 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + mgaussdev noise 200 200 0.0 0.5 + set ts = ts + noise + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + # echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.02 + lim -n 1 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig / sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss -q x y con yf + echo "sigma from straight stdev: $C1" + # stats Tm + + plot -c red -x line x yf + + # get stats for irls + date + medimage calc t1 Ti -irls -variance Tv -bootstrap-iter 100 + date + + imhist -q Ti x y -range -10 10 -delta 0.02 + lim -n 2 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig / sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss -q x y con yf + echo "sigma from irls: $C1 (ideal is {$sig/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range 0 {5*$sig/sqrt($Nsample)} -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} (ideal is {$sig/sqrt($Nsample)})" +end + +############################## +macro test.irls.boot.test + + $Nsample = 100 + $sig1 = 1.0 + + medimage delete -q t1 + for i 0 $Nsample + mgaussdev t 100 100 0.0 $sig1 + set v = $sig1^2 + zero(t) + + medimage add t1 t -variance v + end + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv -bootstrap + + imhist -q Ti x y -range {-10*$sig1/sqrt($Nsample)} {10*$sig1/sqrt($Nsample)} -delta 0.01 + lim -n 2 x y; clear; box; plot -x hist x y + peak -q x y + $C0 = $peakpos + $C1 = 1.5*$sig1/sqrt($Nsample) + $C2 = $peakval + $C3 = 0 + vgauss x y con yf + echo "sigma from irls: $C1 (ideal is {$sig1/sqrt($Nsample)})" + # stats Ti + + plot -c red -x line x yf + + set dTv = sqrt(Tv) + imhist dTv xv yv -range 0 {5*$sig1/sqrt($Nsample)} -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist + + stat -q irls_npt + $Npix = $MEAN + + stat -q Tv + echo "sigma of irls average: $C1, sqrt(mean) of irls variance: {sqrt($MEAN)}, (ideal is {$sig1/sqrt($Npix)})" +end + +macro test.irls.range.var + medimage delete -q t1 + for i 0 8 + mgaussdev t 50 50 0.0 1.0 + set v = 1.0 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + end + + for i 0 8 + mgaussdev t 50 50 0.0 3.0 + set v = 3.0 + zero(t) + + set bad = (rnd(t) < 0.05) ? 10*rnd(t) + 5 : zero(t) + set ts = t + bad + + medimage add t1 ts -variance v + imhist -q t x y -range -10 10 -delta 0.1 + + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + end + + # get stats for straight mean: + medimage calc t1 Tm -mean -variance Tv + + imhist -q Tm x y -range -10 10 -delta 0.1 + lim -n 1 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + stats Tm + + plot -c red -x line x yf + + # get stats for irls + medimage calc t1 Ti -irls -variance Tv + + imhist -q Ti x y -range -10 10 -delta 0.1 + lim -n 2 x y; clear; box; plot -x hist x y + $C0 = 0 + $C1 = 1.5 + $C2 = 400 + $C3 = 0 + vgauss -q x y con yf + echo $C1 + stats Ti + + plot -c red -x line x yf + + stat -q Tv + echo $C1 vs {sqrt($MEDIAN)} + + set dTv = sqrt(Tv) + imhist dTv xv yv -range -1 4 -delta 0.02; lim -n 3 xv yv; clear; box; plot xv yv -x hist +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/set.sh (revision 41170) @@ -0,0 +1,43 @@ + +macro memtest1 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = exp(x) + end + memory check + +end + +macro memtest2 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = (x < 0) ? x : exp(x) + end + memory check + +end + +macro memtest3 + + create x -10 10 0.1 + set y = zero(x) + $i = 0 + + memory check + for i 0 1000 + set y = (x < 5 + 2*$i) ? x : exp(x) + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/spline.sh (revision 41170) @@ -0,0 +1,11 @@ + +macro test1 + + for i 0 100 + create x 0 50 + set y = x^2 + spline create t$i x y + spline rename t$i test + spline delete test + end +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vsigmoid.sh (revision 41170) @@ -0,0 +1,263 @@ + +list tests + test1 + test2 + test3 +end + +macro test1 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + + vsigmoid -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +# noise of 0.1 +macro test2 + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 50 + 1000 / (1 + exp((x - 3)/2)) + set dy = (rnd(y) - 0.5)/0.5 + set y = y + dy + + $C0 = 2 + $C1 = 3 + $C2 = 900 + $C3 = 1 + + vsigmoid -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 1000.0) > 0.1) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.1) + $PASS = 0 + end +end + +# poisson-distributed noise +macro test3 + $PASS = 1 + break -auto off + + $C0o = 3 + $C1o = 2 + $C2o = 20 + $C3o = -2 + + create x -10 10 0.1 + set y = $C3o + $C2o / (1 + exp((x - $C0o)/$C1o)) + + gaussdev dY y[] 0.0 1.0 + set dy = dY * sqrt(abs(y)) + set y = y + dy + + $C0 = $C0o + 2 + $C1 = $C1o + 2 + $C2 = $C2o + 50 + $C3 = $C3o + 10 + + vsigmoid x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + $dS = 3.0 / sqrt(y[]) + + if (abs($C0 - $C0o) > $dS) + $PASS = 0 + end + if (abs($C1 - $C1o)/$C1o > $dS) + $PASS = 0 + end + if (abs($C2 - $C2o)/$C2o > $dS) + $PASS = 0 + end + if (abs($C3 - $C3o) > $dS) + $PASS = 0 + end +end + +macro test1rev + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 5 / (1 + exp(-(x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(abs(y)) + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 5.0) > 0.01) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.01) + $PASS = 0 + end +end + +# noise of 0.1 +macro test2rev + $PASS = 1 + break -auto off + + create x -10 10 0.1 + set y = 50 + 1000 / (1 + exp(-(x - 3)/2)) + set dy = (rnd(y) - 0.5)/0.5 + set y = y + dy + + $C0 = 2 + $C1 = 3 + $C2 = 900 + $C3 = 1 + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + if (abs($C0 - 0.0) > 0.01) + $PASS = 0 + end + if (abs($C1 - 3.0) > 0.01) + $PASS = 0 + end + if (abs($C2 - 1000.0) > 0.1) + $PASS = 0 + end + if (abs($C3 - 0.0) > 0.1) + $PASS = 0 + end +end + +# poisson-distributed noise +macro test3rev + $PASS = 1 + break -auto off + + $C0o = 3 + $C1o = 2 + $C2o = 20 + $C3o = -2 + + create x -10 10 0.1 + set y = $C3o + $C2o / (1 + exp(-(x - $C0o)/$C1o)) + + gaussdev dY y[] 0.0 1.0 + set dy = dY * sqrt(abs(y)) + set y = y + dy + + $C0 = $C0o + 2 + $C1 = $C1o + 2 + $C2 = $C2o + 50 + $C3 = $C3o + 10 + + vsigmoid -r -q x y dy yfit + lim x y; clear; box; plot -x 1 -c black x y; plot -c red x yfit + # cursor + + $dS = 3.0 / sqrt(y[]) + + if (abs($C0 - $C0o) > $dS) + $PASS = 0 + end + if (abs($C1 - $C1o)/$C1o > $dS) + $PASS = 0 + end + if (abs($C2 - $C2o)/$C2o > $dS) + $PASS = 0 + end + if (abs($C3 - $C3o) > $dS) + $PASS = 0 + end +end + +macro memtest1 + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + set yfit = zero(y) + + $i = 0 + + memory check + for i 0 1000 + vsigmoid -q x y dy yfit + end + memory check + +end + +macro memtest2 + + create x -10 10 0.1 + set y = 0 + 5 / (1 + exp((x - 3)/4)) + + $C0 = 1 + $C1 = 2 + $C2 = 10 + $C3 = 1 + set dy = sqrt(y) + set yfit = zero(y) + + $i = 0 + + memory check + for i 0 1000 + vsigmoid -q -r x y dy yfit + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/test/vstat.sh (revision 41170) @@ -0,0 +1,28 @@ + +macro memtest1 + + # preset variables so memory usage does not change + $PMIN = -10 + $PMAX = 9.9 + $MIN = -10 + $MAX = 9.9 + $MEDIAN = -0.1097 + $MEAN = -0.05 + $MODE = -10 + $TOTAL = -10 + $NPIX = 200 + $NPTS = 200 + $NUSED = 200 + $SIGMA = 5.7879184514 + + create x -10 10 0.1 + $i = 0 + + memory check + for i 0 1000 + vstat -q x + end + memory check + +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/threshold.c (revision 41170) @@ -25,4 +25,14 @@ remove_argument (N, &argc, argv); BinMax = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + float ValMin = NAN; + float ValMax = NAN; + if ((N = get_argument (argc, argv, "-vrange"))) { + remove_argument (N, &argc, argv); + ValMin = atof (argv[N]); + remove_argument (N, &argc, argv); + ValMax = atof (argv[N]); remove_argument (N, &argc, argv); } @@ -64,4 +74,16 @@ } + // ValMin, ValMax not compatible with BinMin, BinMax + if (isfinite(ValMin)) { + int binTest = ohana_bisection_double (vecx[0].elements.Flt, vecx[0].Nelements, ValMin); + BinMin = (vecx[0].elements.Flt[binTest+1] == ValMin) ? binTest : binTest + 1; + // XXX dangerous: check that this does not run off the end, etc + } + if (isfinite(ValMax)) { + int binTest = ohana_bisection_double (vecx[0].elements.Flt, vecx[0].Nelements, ValMax); + BinMax = (vecx[0].elements.Flt[binTest+1] == ValMax) ? binTest : binTest + 1; + // XXX dangerous: check that this does not run off the end, etc + } + // use bisection to find the value @@ -71,4 +93,13 @@ if (!REVERSE) { + // this algorithm assumes vecy[BinMax] > threshold, vecy[BinMin] < threshold + if (vecy[0].elements.Flt[BinMin] > value) { + gprint (GP_ERR, "ERROR: all values above threshold\n"); + return FALSE; + } + if (vecy[0].elements.Flt[BinMax] < value) { + gprint (GP_ERR, "ERROR: all values below threshold\n"); + return FALSE; + } while (Nhi - Nlo > 10) { N = 0.5*(Nlo + Nhi); @@ -89,4 +120,13 @@ } } else { + // this algorithm assumes vecy[BinMin] > threshold, vecy[BinMax] < threshold + if (vecy[0].elements.Flt[BinMin] < value) { + gprint (GP_ERR, "ERROR: all values below threshold\n"); + return FALSE; + } + if (vecy[0].elements.Flt[BinMax] > value) { + gprint (GP_ERR, "ERROR: all values above threshold\n"); + return FALSE; + } while (Nhi - Nlo > 10) { N = 0.5*(Nlo + Nhi); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vgauss.c (revision 41170) @@ -3,4 +3,5 @@ /* local private functions */ opihi_flt fgaussOD (opihi_flt, opihi_flt *, int, opihi_flt *); +int vgauss_apply (int argc, char **argv); # define GET_VAR(V,A) \ @@ -20,4 +21,10 @@ Vector *xvec, *yvec, *svec, *ovec; char *c, name[16]; + + if ((N = get_argument (argc, argv, "-apply"))) { + remove_argument (N, &argc, argv); + int status = vgauss_apply (argc, argv); + return status; + } Quiet = FALSE; @@ -104,4 +111,6 @@ } if (!Quiet) gprint (GP_ERR, "%d iterations\n", i); + set_variable ("Chisq", chisq); + set_variable ("ChisqNu", chisq / (float) (Npts - Npar)); for (i = 0; i < Npts; i++) { @@ -125,4 +134,38 @@ } +int vgauss_apply (int argc, char **argv) { + + opihi_flt par[4]; + Vector *xvec, *ovec; + char *c; + + // NOTE: -apply has already been stripped by vgauss call + if (argc != 3) { + gprint (GP_ERR, "USAGE: vgauss -apply \n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ovec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + CastVector (xvec, OPIHI_FLT); + + GET_VAR (par[0], "C0"); + GET_VAR (par[1], "C1"); + GET_VAR (par[2], "C2"); + GET_VAR (par[3], "C3"); + int Npar = 4; + + int Npts = xvec[0].Nelements; + ResetVector (ovec, OPIHI_FLT, Npts); + + for (int i = 0; i < Npts; i++) { + ovec[0].elements.Flt[i] = fgaussOD (xvec[0].elements.Flt[i], par, Npar, NULL); + } + ovec[0].Nelements = Npts; + + return TRUE; +} + /* pars: x_o, sigma, I, back */ opihi_flt fgaussOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { @@ -135,8 +178,10 @@ f = par[2]*r + par[3]; - dpar[0] = par[2]*r*z/par[1]; - dpar[1] = par[2]*r*z*z/par[1]; - dpar[2] = r; - dpar[3] = 1; + if (dpar) { + dpar[0] = par[2]*r*z/par[1]; + dpar[1] = par[2]*r*z*z/par[1]; + dpar[2] = r; + dpar[3] = 1; + } return (f); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/virls.c (revision 41170) @@ -0,0 +1,103 @@ +# include "data.h" + +double opihi_irls_mean_dbl (double *val, double *wgt, int N, int *Converged); +double weight_cauchy_square_dbl (double x2); + +# define IRLS_TOLERANCE 1e-4 + +int virls (int argc, char **argv) { + + if (argc != 3) { + gprint (GP_ERR, "USAGE: virls (values) (sigmas)\n"); + gprint (GP_ERR, "NOTE: sigmas with abs values < %e will be truncated to that value\n", FLT_MIN); + return (FALSE); + } + + Vector *value, *sigma; + + if ((value = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((sigma = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if (value->Nelements != sigma->Nelements) { + gprint (GP_ERR, "vectors are not the same length\n"); + return FALSE; + } + if (value->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[1]); + return FALSE; + } + if (sigma->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[2]); + return FALSE; + } + + ALLOCATE_PTR (val, opihi_flt, value->Nelements); + ALLOCATE_PTR (wgt, opihi_flt, value->Nelements); + + int N = 0; + for (int i = 0; i < value->Nelements; i++) { + if (!isfinite(value->elements.Flt[i])) continue; + + double s = sigma->elements.Flt[i]; + if (!isfinite(s)) continue; + + if (fabs(s) < FLT_MIN) s = FLT_MIN; + + val[N] = value->elements.Flt[i]; + wgt[N] = 1.0 / SQ(s); + N ++; + } + + int converged = TRUE; + double Value = opihi_irls_mean_dbl (val, wgt, N, &converged); + + set_variable ("WTMEAN", Value); + set_int_variable ("NUSED", N); + set_int_variable ("CONVERGED", converged); + return (TRUE); +} + +double opihi_irls_mean_dbl (double *val, double *wgt, int N, int *Converged) { + + // calculate weighted mean + double S1 = 0.0, S2 = 0.0; + for (int n = 0; n < N; n++) { + S1 += wgt[n] * val[n]; + S2 += wgt[n]; + } + double Value = S1 / S2; + + int converged = FALSE; + for (int i = 0; (i < 10) && !converged; i++) { + double ValueLast = Value; + + double S1 = 0.0, S2 = 0.0; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int n = 0; n < N; n++) { + double dV = (val[n] - Value); + double d2 = SQ(dV) * wgt[n]; + + double Mod = weight_cauchy_square_dbl (d2); + S1 += Mod * wgt[n] * val[n]; + S2 += Mod * wgt[n]; + fprintf (stderr, "%f %f : %f %f : %f\n", val[n], wgt[n], dV, sqrt(d2), Mod); + } + Value = S1 / S2; + + double delta = fabs(Value - ValueLast); + if (delta < Value * IRLS_TOLERANCE) converged = TRUE; + + // XXX if the answer is close to zero, we might not converge + } + *Converged = converged; + return Value; +} + +# define CAUCY_FACTOR 1.0 + +double weight_cauchy_square_dbl (double x2) { + double r2 = x2 / CAUCY_FACTOR; + return (1.0 / (1.0 + r2)); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vsigmoid.c (revision 41170) @@ -0,0 +1,192 @@ +# include "data.h" + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); + +/* local private functions */ +opihi_flt fsigmoidOD (opihi_flt, opihi_flt *, int, opihi_flt *); +opihi_flt rsigmoidOD (opihi_flt, opihi_flt *, int, opihi_flt *); + +# define GET_VAR(V,A) { \ + char *c = get_variable (A); \ + if (c == NULL) { \ + gprint (GP_ERR, "missing fit parameter A\n"); \ + return (FALSE); \ + } \ + V = atof (c); \ + free (c); } + +// XXX this fitting function works but fails if the input vectors have NANs +int vsigmoid (int argc, char **argv) { + + int N; + Vector *xvec, *yvec, *svec, *ovec; + + FitFunc *FUNC = fsigmoidOD; + if ((N = get_argument (argc, argv, "-r"))) { + FUNC = rsigmoidOD; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-rev"))) { + FUNC = rsigmoidOD; + remove_argument (N, &argc, argv); + } + + int Quiet = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: vsigmoid [-q] [-r] [-rev] (out)\n"); + gprint (GP_ERR, " may be the words 'con[stant]' or 'poi[sson]', in which case the error is constructed'\n"); + gprint (GP_ERR, " uses guesses: C0 (center), C1 (sigma), C2 (top), C3 (bottom)\n"); + gprint (GP_ERR, " forward sigmoid : bottom + top / (1 + exp(-z)), z = (x - center) / sigma\n"); + gprint (GP_ERR, " -r / -rev : reverse sigmoid : bottom + top / (1 + exp(z))\n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((ovec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + CastVector (xvec, OPIHI_FLT); + CastVector (yvec, OPIHI_FLT); + + int Nsvec = strlen(argv[3]); + + if ((Nsvec >= 3) && (!strncasecmp ("poisson", argv[3], Nsvec) || !strncasecmp ("constant", argv[3], Nsvec))) { + svec = SelectVector ("vsigmoid_err", ANYVECTOR, TRUE); + if (svec == NULL) return (FALSE); + MatchVector (svec, xvec, OPIHI_FLT); + opihi_flt *v1 = svec[0].elements.Flt; + opihi_flt *v2 = yvec[0].elements.Flt; + if (!strncasecmp ("poisson", argv[3], Nsvec)) { + for (int i = 0; i < svec[0].Nelements; i++) { + v1[i] = (isfinite(v2[i]) && (v2[i] > 0)) ? sqrt(v2[i]) : 1.0; + } + } else { + for (int i = 0; i < svec[0].Nelements; i++) { + v1[i] = 1.0; + } + } + } else { + if ((svec = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + } + + CastVector (svec, OPIHI_FLT); + // XXX Cast is failing. + + int Npts = xvec[0].Nelements; + ResetVector (ovec, OPIHI_FLT, Npts); + + ALLOCATE_PTR (dy, opihi_flt, Npts); + + opihi_flt par[4]; + GET_VAR (par[0], "C0"); + GET_VAR (par[1], "C1"); + GET_VAR (par[2], "C2"); + GET_VAR (par[3], "C3"); + int Npar = 4; + + // mrqmin takes the inverse variance (do not generate NANs) + opihi_flt *v1 = svec[0].elements.Flt; + opihi_flt *v2 = dy; + for (int i = 0; i < Npts; i++, v1++, v2++) { + *v2 = (*v1 == 0.0) ? 0.0 : 1.0 / (*v1 * *v1); + } + + opihi_flt ochisq = mrqinit (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, FUNC, !Quiet); + opihi_flt dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + opihi_flt chisq = mrqmin (xvec[0].elements.Flt, yvec[0].elements.Flt, dy, Npts, par, Npar, FUNC, !Quiet); + dchisq = ochisq - chisq; + ochisq = chisq; + if (!Quiet) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (!Quiet) gprint (GP_ERR, "%d iterations\n", Niter); + + for (int i = 0; i < Npts; i++) { + ovec[0].elements.Flt[i] = FUNC (xvec[0].elements.Flt[i], par, Npar, dy); + } + ovec[0].Nelements = Npts; + /* set output *before* variable renomalization */ + + opihi_flt **covar = mrqcovar (Npar); + for (int i = 0; i < Npar; i++) { + char name[16]; + sprintf (name, "C%d", i); + set_variable (name, par[i]); + if (!Quiet) gprint (GP_ERR, "%d %f %f\n", i, par[i], sqrt(covar[i][i])); + sprintf (name, "dC%d", i); + set_variable (name, sqrt(covar[i][i])); + } + + free (dy); + mrqfree (Npar); + return (TRUE); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt rsigmoidOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + dpar[0] = par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + + return (f); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt fsigmoidOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (-z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = -r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = -par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = -par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + dpar[0] = -par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + + return (f); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vtransitions.c (revision 41170) @@ -0,0 +1,175 @@ +# include "data.h" + +static int isFltX = FALSE; +static int isFltY = FALSE; +static double interpolate_position (Vector *vecx, Vector *vecy, int bin, float value); + +int vtransitions (int argc, char **argv) { + + int N; + Vector *vecx, *vecy, *outbin, *outval, *outpos; + + int QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + + int UP_TRANSITIONS = TRUE; + if ((N = get_argument (argc, argv, "-dn"))) { + UP_TRANSITIONS = FALSE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-down"))) { + UP_TRANSITIONS = FALSE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-up"))) { + UP_TRANSITIONS = TRUE; + remove_argument (N, &argc, argv); + } + + int BinMin = -1; + int BinMax = -1; + if ((N = get_argument (argc, argv, "-range"))) { + remove_argument (N, &argc, argv); + BinMin = atoi (argv[N]); + remove_argument (N, &argc, argv); + BinMax = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 4) { + gprint (GP_ERR, "USAGE: vtransitions (value) (-up|-down,-dn) [-q] [-range BinMin BinMax]\n"); + gprint (GP_ERR, " find the x coordinates at which we pass the specified value going up\n"); + gprint (GP_ERR, " -q : quiet mode\n"); + gprint (GP_ERR, " -r : reverse (find downward transition)\n"); + return (FALSE); + } + + if ((vecx = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((vecy = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if (vecx[0].Nelements != vecy[0].Nelements) return FALSE; + int Nelements = vecx[0].Nelements; + + if ((outbin = SelectVector ("vtr_bin", ANYVECTOR, TRUE)) == NULL) return (FALSE); // bins containing transitions + if ((outval = SelectVector ("vtr_val", ANYVECTOR, TRUE)) == NULL) return (FALSE); // value of bins containing transitions + if ((outpos = SelectVector ("vtr_pos", ANYVECTOR, TRUE)) == NULL) return (FALSE); // interpolated coordinate of transitions + + double value = atof (argv[3]); + + isFltX = (vecx[0].type == OPIHI_FLT); + isFltY = (vecy[0].type == OPIHI_FLT); + + // BinMin = -1 -> range not provide + if (BinMin == -1) { + BinMin = 0; + BinMax = Nelements - 1; + } else { + if ((BinMin >= vecx[0].Nelements) || (BinMin < -1*vecx[0].Nelements)) { + gprint (GP_ERR, "vector subscript out of range\n"); + return FALSE; + } + if (BinMin < 0) BinMin += vecx[0].Nelements; + if ((BinMax >= vecx[0].Nelements) || (BinMax < -1*vecx[0].Nelements)) { + gprint (GP_ERR, "vector subscript out of range\n"); + return FALSE; + } + if (BinMax < 0) BinMax += vecx[0].Nelements; + } + + int Nout = 0; + int NOUT = 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + + // starting disposition + int isUp = isFltY ? (vecy[0].elements.Flt[BinMin] > value) : (vecy[0].elements.Int[BinMin] > value); + + for (int i = BinMin; i <= BinMax; i++) { + double testval = isFltY ? vecy[0].elements.Flt[i] : vecy[0].elements.Int[i]; + if (isUp && (testval < value)) { + isUp = FALSE; + if (!UP_TRANSITIONS) { + // save this value + outbin->elements.Int[Nout] = i; + if (isFltY) { + outval->elements.Flt[Nout] = testval; + } else { + outval->elements.Int[Nout] = testval; + } + outpos->elements.Flt[Nout] = interpolate_position (vecx, vecy, i, value); + Nout ++; + if (Nout == NOUT) { + NOUT += 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + } + } + continue; + } + if (!isUp && (testval > value)) { + isUp = TRUE; + if (UP_TRANSITIONS) { + // save this value + outbin->elements.Int[Nout] = i; + if (isFltY) { + outval->elements.Flt[Nout] = testval; + } else { + outval->elements.Int[Nout] = testval; + } + outpos->elements.Flt[Nout] = interpolate_position (vecx, vecy, i, value); + Nout ++; + if (Nout == NOUT) { + NOUT += 100; + ResetVector (outbin, OPIHI_INT, NOUT); + ResetVector (outval, vecy[0].type, NOUT); + ResetVector (outpos, OPIHI_FLT, NOUT); + } + } + } + } + + ResetVector (outbin, OPIHI_INT, Nout); + ResetVector (outval, vecy[0].type, Nout); + ResetVector (outpos, OPIHI_FLT, Nout); + + if (!QUIET) gprint (GP_LOG, "found %d transitions\n", Nout); + + return (TRUE); +} + +static double interpolate_position (Vector *vecx, Vector *vecy, int bin, float value) { + + double x0, x1, y0, y1, Xvalue; + + if (vecy[0].Nelements == 0) return NAN; + if (vecy[0].Nelements == 1) return isFltY ? vecy[0].elements.Flt[0] : vecy[0].elements.Int[0]; + + if (bin == 0) { + // interpolate to value: + y0 = isFltY ? vecy[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + y1 = isFltY ? vecy[0].elements.Flt[bin+1] : vecy[0].elements.Int[bin+1]; + x0 = isFltX ? vecx[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + x1 = isFltX ? vecx[0].elements.Flt[bin+1] : vecy[0].elements.Int[bin+1]; + if (y0 == y1) { + Xvalue = 0.5*(x0 + x1); + } else { + Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0; + } + } else { + // interpolate to value: + y0 = isFltY ? vecy[0].elements.Flt[bin-1] : vecy[0].elements.Int[bin-1]; + y1 = isFltY ? vecy[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + x0 = isFltX ? vecx[0].elements.Flt[bin-1] : vecy[0].elements.Int[bin-1]; + x1 = isFltX ? vecx[0].elements.Flt[bin] : vecy[0].elements.Int[bin]; + if (y0 == y1) { + Xvalue = 0.5*(x0 + x1); + } else { + Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0; + } + } + return Xvalue; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/vwtmean.c (revision 41170) @@ -0,0 +1,70 @@ +# include "data.h" + +double opihi_wt_mean_dbl (double *val, double *wgt, int N, double *Sigma); + +int vwtmean (int argc, char **argv) { + + if (argc != 3) { + gprint (GP_ERR, "USAGE: vwtmean (values) (sigmas)\n"); + gprint (GP_ERR, "NOTE: sigmas with abs values < %e will be truncated to that value\n", FLT_MIN); + return (FALSE); + } + + Vector *value, *sigma; + + if ((value = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((sigma = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if (value->Nelements != sigma->Nelements) { + gprint (GP_ERR, "vectors are not the same length\n"); + return FALSE; + } + if (value->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[1]); + return FALSE; + } + if (sigma->type != OPIHI_FLT) { + gprint (GP_ERR, "%s is not a floating point vector\n", argv[2]); + return FALSE; + } + + ALLOCATE_PTR (val, opihi_flt, value->Nelements); + ALLOCATE_PTR (wgt, opihi_flt, value->Nelements); + + int N = 0; + for (int i = 0; i < value->Nelements; i++) { + if (!isfinite(value->elements.Flt[i])) continue; + + double s = sigma->elements.Flt[i]; + if (!isfinite(s)) continue; + + if (fabs(s) < FLT_MIN) s = FLT_MIN; + + val[N] = value->elements.Flt[i]; + wgt[N] = 1.0 / SQ(s); + N ++; + } + + double Sigma = NAN; + double Value = opihi_wt_mean_dbl (val, wgt, N, &Sigma); + + set_variable ("WTMEAN", Value); + set_variable ("WTSIGMA", Sigma); + set_int_variable ("NUSED", N); + return (TRUE); +} + +double opihi_wt_mean_dbl (double *val, double *wgt, int N, double *Sigma) { + + // calculate weighted mean + double S1 = 0.0, S2 = 0.0; + for (int n = 0; n < N; n++) { + S1 += wgt[n] * val[n]; + S2 += wgt[n]; + } + double Value = S1 / S2; + *Sigma = 1.0 / S2; + + return Value; +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/cmd.data/write_vectors.c (revision 41170) @@ -32,12 +32,23 @@ } - /* option generate a FITS output table */ + // option generate a FITS output table (FITS holds the filename) + // in FITS output context, -header is interpretted as a buffer containing + // header keywords to supplement the FITS table header char *FITS = NULL; + Header *fitsheader = NULL; + Buffer *headbuffer = NULL; if ((N = get_argument (argc, argv, "-fits"))) { remove_argument (N, &argc, argv); FITS = strcreate (argv[N]); remove_argument (N, &argc, argv); - } - + + if ((N = get_argument (argc, argv, "-header"))) { + remove_argument (N, &argc, argv); + if ((headbuffer = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + fitsheader = &headbuffer->header; + remove_argument (N, &argc, argv); + } + } + /* option generate a FITS output table */ int CSV = FALSE; @@ -124,5 +135,5 @@ if (FITS) { - int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, compress, format, Ntile); + int status = WriteVectorTableFITS (argv[1], FITS, fitsheader, vec, Nvec, append, compress, format, Ntile); free (vec); return status; @@ -260,4 +271,6 @@ gprint (GP_ERR, " -append : write to the end of the existing file\n"); gprint (GP_ERR, " -fits NAME : write a fits table (extention name is NAME, column names match vector names)\n"); + gprint (GP_ERR, " in FITS output context, -header takes an additional argument which is interpretted\n"); + gprint (GP_ERR, " as a buffer containing header keywords to supplement the FITS table header\n"); gprint (GP_ERR, " -csv : write a comma-separated values file (eg, to read in excel)\n"); gprint (GP_ERR, " -f \"format\" : provide formatting codes for output:\n"); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/Makefile (revision 41170) @@ -67,4 +67,5 @@ $(SRC)/gstar.$(ARCH).o \ $(SRC)/hosts.$(ARCH).o \ +$(SRC)/cimages.$(ARCH).o \ $(SRC)/images.$(ARCH).o \ $(SRC)/imbox.$(ARCH).o \ Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avextract.c (revision 41170) @@ -272,5 +272,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (RESULT_FILE && !SKIP_RESULTS) { - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nreturn, FALSE, FALSE, NULL, 0); if (!status) { goto escape; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avmatch.c (revision 41170) @@ -114,5 +114,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -301,5 +301,5 @@ vec[i][0].Nelements = Nfound; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nfields + 1, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nfields + 1, FALSE, FALSE, NULL, 0); free (vec[Nfields]->elements.Int); free (vec[Nfields]); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avperiodomatch.c (revision 41170) @@ -99,5 +99,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; free (vec); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/avselect.c (revision 41170) @@ -132,5 +132,5 @@ CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -327,5 +327,5 @@ vec[NfieldsOut-1] = RADvec; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, NfieldsOut, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, NfieldsOut, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/cimages.c (revision 41170) @@ -0,0 +1,379 @@ +# include "dvoshell.h" +# define BETA 0.41421 + +int wordhash (char *word); + +int cimages (int argc, char **argv) { + + off_t i, Nimage, Nmosaic; + int j, status, InPic, leftside, TimeSelect, ByName; + int WITH_MOSAIC, SOLO_MOSAIC; + time_t tzero, tend; + int n, N, NPTS, Npts, kapa, *foundMosaic; + Vector Xvec, Yvec, Zvec; + double r[8], d[8], x[8], y[8], Rmin, Rmax, Rmid, trange, Radius; + Image *image; + Graphdata graphmode; + char name[256]; + int typehash; + PhotCode *photcode; + int Nsec, photcodeEquiv; + + if (!InitPhotcodes ()) return (FALSE); + if (!style_args (&graphmode, &argc, argv, &kapa)) goto usage; + + WITH_MOSAIC = FALSE; + if ((N = get_argument (argc, argv, "+mosaic"))) { + remove_argument (N, &argc, argv); + WITH_MOSAIC = TRUE; + } + + SOLO_MOSAIC = FALSE; + foundMosaic = NULL; + if ((N = get_argument (argc, argv, "-mosaic"))) { + remove_argument (N, &argc, argv); + SOLO_MOSAIC = TRUE; + WITH_MOSAIC = TRUE; + } + + // int HIDDEN = FALSE; + // if ((N = get_argument (argc, argv, "-hidden"))) { + // remove_argument (N, &argc, argv); + // HIDDEN = TRUE; + // } + + photcode = NULL; + photcodeEquiv = FALSE; + if ((N = get_argument (argc, argv, "-p"))) { + remove_argument (N, &argc, argv); + photcode = GetPhotcodebyName (argv[N]); + if (photcode == NULL) { + gprint (GP_ERR, "photcode %s not found\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + Nsec = GetPhotcodeNsec (photcode->code); + if (Nsec != -1) photcodeEquiv = TRUE; + } + if ((N = get_argument (argc, argv, "-photcode"))) { + remove_argument (N, &argc, argv); + photcode = GetPhotcodebyName (argv[N]); + if (photcode == NULL) { + gprint (GP_ERR, "photcode %s not found\n", argv[N]); + return FALSE; + } + remove_argument (N, &argc, argv); + Nsec = GetPhotcodeNsec (photcode->code); + if (Nsec != -1) photcodeEquiv = TRUE; + } + + ByName = FALSE; + if ((N = get_argument (argc, argv, "-name"))) { + remove_argument (N, &argc, argv); + strcpy (name, argv[N]); + remove_argument (N, &argc, argv); + ByName = TRUE; + } + + Radius = 45; + if ((N = get_argument (argc, argv, "-radius"))) { + remove_argument (N, &argc, argv); + Radius = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + TimeSelect = FALSE; + if ((N = get_argument (argc, argv, "-time"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tzero)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_dtime (argv[N], &trange)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (trange < 0) { + trange = fabs (trange); + tzero -= trange; + } + TimeSelect = TRUE; + } + if ((N = get_argument (argc, argv, "-trange"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tzero)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &tend)) { + gprint (GP_ERR, "syntax error\n"); + goto usage; + } + remove_argument (N, &argc, argv); + trange = tend - tzero; + if (trange < 0) { + trange = fabs (trange); + tzero -= trange; + } + TimeSelect = TRUE; + } + + if (argc != 3) goto usage; + float vmin = atof (argv[1]); + float vmax = atof (argv[2]); + + /* it is not an error for the database not to have any images */ + if ((image = LoadImagesDVO (&Nimage)) == NULL) return (TRUE); + // BuildChipMatch (image, Nimage); + + if (SOLO_MOSAIC && photcode) { + ALLOCATE(foundMosaic, int, Nimage); + memset(foundMosaic, 0, Nimage*sizeof(int)); + } + + Rmin = graphmode.coords.crval1 - 180.0; + Rmax = graphmode.coords.crval1 + 180.0; + Rmid = 0.5*(Rmin + Rmax); + + int DistortImage = wordhash ("-DIS"); + int ChipImage = wordhash ("-WRP"); + int TriangleUp = wordhash ("TRP-"); + int TriangleDn = wordhash ("TRM-"); + int TrianglePts = wordhash ("TRI-"); + + NPTS = 200; + SetVector (&Xvec, OPIHI_FLT, NPTS); + SetVector (&Yvec, OPIHI_FLT, NPTS); + SetVector (&Zvec, OPIHI_FLT, NPTS); + + Image *mosaic = NULL; + + /******** stage 1 : find the images to plot *********/ + + n = 0; // tracks number of images + N = 0; // tracks number of vertices + for (i = 0; i < Nimage; i++) { + if (ByName && strncmp (image[i].name, name, strlen(name))) continue; + if (TimeSelect && ((image[i].tzero < tzero) || (image[i].tzero+image[i].trate*image[i].NY > tzero + trange))) continue; + + mosaic = image[i].parent; + Nmosaic = (SOLO_MOSAIC && mosaic) ? mosaic - image : -1; + + if (photcode) { + if ( photcodeEquiv && (photcode[0].code != GetPhotcodeEquivCodebyCode(image[i].photcode))) continue; + if (!photcodeEquiv && (photcode[0].code != image[i].photcode)) continue; + } + + Npts = 4; + status = TRUE; + leftside = FALSE; + + typehash = wordhash (&image[i].coords.ctype[4]); + + if (photcode && SOLO_MOSAIC && (Nmosaic >= 0)) { + // mosaic (DIS) images are not currently given a photcode : plot these via the WRP entries + /* DIS images represent a field, not a chip */ + if (typehash != ChipImage) continue; + if (foundMosaic[Nmosaic]) continue; + x[0] = -0.5*mosaic->NX; y[0] = -0.5*mosaic->NY; + x[1] = +0.5*mosaic->NX; y[1] = -0.5*mosaic->NY; + x[2] = +0.5*mosaic->NX; y[2] = +0.5*mosaic->NY; + x[3] = -0.5*mosaic->NX; y[3] = +0.5*mosaic->NY; + for (j = 0; j < Npts; j++) { + status = XY_to_RD (&r[j], &d[j], x[j], y[j], &mosaic->coords); + if (!status) break; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + } + foundMosaic[Nmosaic] = TRUE; + goto plot_points; + } + + /* DIS images represent a field, not a chip */ + if ((typehash == DistortImage) && !WITH_MOSAIC) continue; // do not plot the mosaic images + if ((typehash != DistortImage) && SOLO_MOSAIC) continue; // plot only the mosaic images + + if (typehash == DistortImage) { + x[0] = -0.5*image[i].NX; y[0] = -0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = -0.5*image[i].NY; + x[2] = +0.5*image[i].NX; y[2] = +0.5*image[i].NY; + x[3] = -0.5*image[i].NX; y[3] = +0.5*image[i].NY; + goto got_type; + } + + typehash = wordhash (image[i].coords.ctype); + if (typehash == TriangleUp) { + Npts = 3; + x[0] = 0; y[0] = +0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = -0.5*image[i].NY; + x[2] = -0.5*image[i].NX; y[2] = -0.5*image[i].NY; + goto got_type; + } + if (typehash == TriangleDn) { + Npts = 3; + x[0] = 0; y[0] = -0.5*image[i].NY; + x[1] = +0.5*image[i].NX; y[1] = +0.5*image[i].NY; + x[2] = -0.5*image[i].NX; y[2] = +0.5*image[i].NY; + goto got_type; + } + // For 'TrianglePts' (TRI-), we are absurdly overloaded values otherwise not used + if (typehash == TrianglePts) { + Npts = 3; + x[0] = image[0].dXpixSys; y[0] = image[0].dYpixSys; + x[1] = image[0].dMagSys; y[1] = image[0].nFitAstrom; + x[2] = image[0].photom_map_id; y[2] = image[0].astrom_map_id; + goto got_type; + } + + // default layout + { + x[0] = 0; y[0] = 0; + x[1] = image[i].NX; y[1] = 0; + x[2] = image[i].NX; y[2] = image[i].NY; + x[3] = 0; y[3] = image[i].NY; + } + + got_type: + + /**** project this image to screen display coords ****/ + + // check for boundary overlap? + for (j = 0; j < Npts; j++) { + status = XY_to_RD (&r[j], &d[j], x[j], y[j], &image[i].coords); + if (!status) break; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + } + + // extremely large-scale images with certain projections will have odd boundaries. + // eg, the ASCA images are essentially circles of radius ~60 degrees. plot these as + // octagons with some dummy size. + if (!status) { + int jp, xo, yo; + double rc, dc; + xo = 0.5*image[i].NX; + yo = 0.5*image[i].NY; + // is the image center on the screen? + status = XY_to_RD (&rc, &dc, 0.5*x[2], 0.5*y[2], &image[i].coords); + if (status && (image[i].NX * image[i].coords.cdelt2 > 90)) { + // draw an octagon with radius 45 degrees + double dX = Radius / image[i].coords.cdelt2; + x[0] = xo+dX; y[0] = yo+BETA*dX; + x[1] = xo+BETA*dX; y[1] = yo+dX; + x[2] = xo-BETA*dX; y[2] = yo+dX; + x[3] = xo-dX; y[3] = yo+BETA*dX; + x[4] = xo-dX; y[4] = yo-BETA*dX; + x[5] = xo-BETA*dX; y[5] = yo-dX; + x[6] = xo+BETA*dX; y[6] = yo-dX; + x[7] = xo+dX; y[7] = yo-BETA*dX; + Npts = 8; + j = 0; + for (jp = 0; jp < 8; jp++) { + status = XY_to_RD (&r[j], &d[j], x[jp], y[jp], &image[i].coords); + if (!status) continue; + r[j] = ohana_normalize_angle (r[j]); + while (r[j] < Rmin) { r[j] += 360.0; } + while (r[j] > Rmax) { r[j] -= 360.0; } + if (j == 0) { + leftside = (r[j] < Rmid); + } + if (j > 0) { + if ( leftside && (r[j] > Rmid + 90)) { r[j] -= 360.0; } + if (!leftside && (r[j] < Rmid - 90)) { r[j] += 360.0; } + } + j++; + } + Npts = j; + } else { + continue; + } + } + if (Npts == 0) continue; + + plot_points: + + /**** transform corners to screen coordinates ******/ + + status = FALSE; + for (j = 0; j < Npts; j++) { + status |= RD_to_XY (&Xvec.elements.Flt[N+j], &Yvec.elements.Flt[N+j], r[j], d[j], &graphmode.coords); + Zvec.elements.Flt[N+j] = MIN(1.0, MAX(0.01, (image[i].exptime - vmin) / (vmax - vmin))); + } + if (!status) continue; + // if none of the points are on the visible side of the projection, do not plot the image + + InPic = FALSE; + for (j = 0; j < Npts; j++) { + if ((Xvec.elements.Flt[N+j] >= graphmode.xmin) && + (Xvec.elements.Flt[N+j] <= graphmode.xmax) && + (Yvec.elements.Flt[N+j] >= graphmode.ymin) && + (Yvec.elements.Flt[N+j] <= graphmode.ymax)) + InPic = TRUE; + } + // if (!status && HIDDEN) status = TRUE; + if (!InPic) continue; + + n ++; + N += Npts; + if (N + 16 >= NPTS) { /* need to leave room for 8 point image */ + NPTS += 400; + REALLOCATE (Xvec.elements.Flt, opihi_flt, NPTS); + REALLOCATE (Yvec.elements.Flt, opihi_flt, NPTS); + REALLOCATE (Zvec.elements.Flt, opihi_flt, NPTS); + } + } + + gprint (GP_ERR, "plotting %d images\n", n); + Xvec.Nelements = Yvec.Nelements = Zvec.Nelements = N; + if (N > 0) { + // XXX note that ptype limits us to plot only a single type of + // image in a single pass + graphmode.style = KAPA_PLOT_POLYFILL; /* points */ + graphmode.ptype = Npts; /* connect pairs of points */ + graphmode.etype = 0; + graphmode.size = -1; /* point size determined by Zvec */ + graphmode.color = -1; // use z-vec for color + PlotVectorTriplet (kapa, &Xvec, &Yvec, &Zvec, NULL, &graphmode); +// PlotVectorPair (kapa, &Xvec, &Yvec, NULL, &graphmode); + } + + FREE (Xvec.elements.Flt); + FREE (Yvec.elements.Flt); + FREE (Zvec.elements.Flt); + FreeImagesDVO (image); + FREE (foundMosaic); + return (TRUE); + + + usage: + gprint (GP_ERR, "USAGE: cimage (vmin) (vmax) [options]\n"); + gprint (GP_ERR, " +mosaic : show mosaic outline\n"); + gprint (GP_ERR, " -mosaic : only mosaic outline\n"); + gprint (GP_ERR, " -hidden : (deprecated)\n"); + gprint (GP_ERR, " -name : only names matching (start of name)\n"); + gprint (GP_ERR, " -radius : display all-sky images with given radius octagon\n"); + gprint (GP_ERR, " -time (start) (range) : show images for time and interval\n"); + gprint (GP_ERR, " -trange (start) (stop) : show images within time range\n"); + return (FALSE); +} + + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/dvo_host_utils.c (revision 41170) @@ -233,5 +233,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (ResultFile) { - int status = WriteVectorTableFITS (ResultFile, "RESULT", vec, Nvec, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (ResultFile, "RESULT", NULL, vec, Nvec, FALSE, FALSE, NULL, 0); if (!status) { gprint (GP_ERR, "failed to write result file %s\n", ResultFile); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/images.c (revision 41170) @@ -12,8 +12,9 @@ off_t i, Nimage, Nmosaic; - int j, status, InPic, leftside, *plist, TimeSelect, ByName; + int j, status, InPic, leftside, TimeSelect, ByName; + // int *plist, n, npts; -- what is/was plist? int WITH_MOSAIC, SOLO_MOSAIC; time_t tzero, tend; - int N, NPTS, n, npts, Npts, kapa, *foundMosaic; + int n, N, NPTS, Npts, kapa, *foundMosaic; Vector Xvec, Yvec; double r[8], d[8], x[8], y[8], Rmin, Rmax, Rmid, trange, Radius; @@ -148,5 +149,6 @@ int TrianglePts = wordhash ("TRI-"); - npts = NPTS = 200; + // npts = 200 + NPTS = 200; SetVector (&Xvec, OPIHI_FLT, NPTS); SetVector (&Yvec, OPIHI_FLT, NPTS); @@ -154,6 +156,9 @@ Image *mosaic = NULL; - ALLOCATE (plist, int, NPTS); - n = N = 0; + /******** stage 1 : find the images to plot *********/ + + // ALLOCATE (plist, int, NPTS); + n = 0; // tracks number of images + N = 0; // tracks number of vertices for (i = 0; i < Nimage; i++) { if (ByName && strncmp (image[i].name, name, strlen(name))) continue; @@ -335,11 +340,12 @@ if (!InPic) continue; - plist[n] = i; - n++; - if (n > npts - 1) { - npts += 200; - REALLOCATE (plist, int, npts); - } - N+=2*Npts; + // plist[n] = i; + // n++; + // if (n > npts - 1) { + // npts += 200; + // REALLOCATE (plist, int, npts); + // } + n ++; + N += 2*Npts; if (N + 16 >= NPTS) { /* need to leave room for 8 point image */ NPTS += 400; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/init.c (revision 41170) @@ -14,4 +14,5 @@ int catdir_define PROTO((int, char **)); int ccd PROTO((int, char **)); +int cimages PROTO((int, char **)); int cmatch PROTO((int, char **)); int cmd PROTO((int, char **)); @@ -80,4 +81,5 @@ {1, "catdir", catdir_define,"re-define CATDIR"}, // {1, "ccd", ccd, "plot color-color diagram"}, + {1, "cimages", cimages, "fill image boxes with a color"}, {1, "cmatch", cmatch, "match two catalogs"}, // {1, "cmd", cmd, "plot cmd of stars in current region"}, Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mextract.c (revision 41170) @@ -352,5 +352,5 @@ // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere) if (RESULT_FILE && !SKIP_RESULTS) { - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nreturn, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/dvo/mmatch.c (revision 41170) @@ -148,5 +148,5 @@ // XXX this is now set for both cases... CoordsFile = abspath("coords.fits", 1024); - int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (CoordsFile, "COORDS", NULL, vec, 2, FALSE, FALSE, NULL, 0); if (!status) goto escape; } @@ -349,5 +349,5 @@ vec[Nfields-1] = IDXvec; } - int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nfields, FALSE, FALSE, NULL, 0); + int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", NULL, vec, Nfields, FALSE, FALSE, NULL, 0); if (!status) goto escape; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/data.h (revision 41170) @@ -48,5 +48,6 @@ int Nx; int Ny; - float **buffers; + float **flx; + float **var; } MedImageType; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/deimos.h (revision 41170) @@ -7,5 +7,5 @@ } DeimosResult; -float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *trace, int Nx, int Ny, int row); +float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, Spline *psf_trace, int Nx, int Ny, int row); void deimos_make_kernel (float stilt, int Nx); void deimos_free_kernel (); @@ -13,8 +13,8 @@ // internal to make_model: -float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, int Nx, int Ny, int row); +float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, Spline *psf_trace, int Nx, int Ny, int row); float *deimos_apply_tilt (float *input, int Nx, int Ny); void deimos_apply_profile (Vector *profile, float *out, int Nx, int Ny); void deimos_add_background (opihi_flt *backgnd, float *out, int Nx, int Ny, int row); -float *deimos_apply_trace (Spline *trace, float *input, int Nx, int Ny, int row); +float *deimos_apply_trace (Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, float *input, int Nx, int Ny, int row); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/include/dvomath.h (revision 41170) @@ -169,5 +169,5 @@ /* vector IO functions */ -int WriteVectorTableFITS PROTO((char *filename, char *extname, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile)); +int WriteVectorTableFITS PROTO((char *filename, char *extname, Header *extraheader, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile)); Vector **ReadVectorTableFITS PROTO((char *filename, char *extname, int *Nvec)); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/MedImageOps.c (revision 41170) @@ -32,7 +32,9 @@ free (medimage[0].name); for (i = 0; i < medimage[0].Ninput; i++) { - free (medimage[0].buffers[i]); + free (medimage[0].flx[i]); + FREE (medimage[0].var[i]); } - free (medimage[0].buffers); + free (medimage[0].flx); + free (medimage[0].var); free (medimage); } @@ -75,5 +77,6 @@ medimage->Nx = Nx; medimage->Ny = Ny; - ALLOCATE (medimage->buffers, float *, 1); + ALLOCATE (medimage->flx, float *, 1); + ALLOCATE (medimage->var, float *, 1); medimages[N] = medimage; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/SplineOps.c (revision 41170) @@ -41,7 +41,9 @@ ALLOCATE (spline[0].y2, opihi_flt, spline[0].Nknots); } - memset (spline[0].xk, 0, spline[0].Nknots * sizeof(opihi_flt)); - memset (spline[0].yk, 0, spline[0].Nknots * sizeof(opihi_flt)); - memset (spline[0].y2, 0, spline[0].Nknots * sizeof(opihi_flt)); + if (spline[0].Nknots) { + memset (spline[0].xk, 0, spline[0].Nknots * sizeof(opihi_flt)); + memset (spline[0].yk, 0, spline[0].Nknots * sizeof(opihi_flt)); + memset (spline[0].y2, 0, spline[0].Nknots * sizeof(opihi_flt)); + } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/open_kapa.c (revision 41170) @@ -266,4 +266,5 @@ int close_kapa (char *name) { + struct timespec request, remain; int N; @@ -274,4 +275,24 @@ } DelKapaDevice (name); + + // avoid blocking on waitpid, test every 1 msec, up to 500 msec + request.tv_sec = 0; + request.tv_nsec = 1000000; + + // try to harvest the child PID + int waitstatus = 0; + int result = waitpid (-1, &waitstatus, WNOHANG); + for (int i = 0; (i < 500) && (result == 0); i++) { + nanosleep (&request, &remain); + result = waitpid (-1, &waitstatus, WNOHANG); + } + + if ((result == -1) && (errno != ECHILD)) { + fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno); + } + if (result == 0) { + fprintf (stderr, "child did not exit (close_kapa), timeout"); + } + return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/spline.c (revision 41170) @@ -7,4 +7,7 @@ float dy, dx, *tmp; + // spline is not valid with < 3 points + if (N < 3) return; + ALLOCATE (tmp, float, N); @@ -69,4 +72,7 @@ opihi_flt dy, dx, *tmp; + // spline is not valid with < 3 points + if (N < 3) return; + ALLOCATE (tmp, opihi_flt, N); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/starfuncs.c (revision 41170) @@ -75,4 +75,6 @@ FWHMx = 2.355*sqrt (fabs(x2 / I - x*x)); FWHMy = 2.355*sqrt (fabs(y2 / I - y*y)); + + fprintf (stderr, "Mxx, Myy: %f, %f\n", x2/I - x*x, y2/I - y*y); Sxy = xy / I - x*y; mag = -2.5*log10(I); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.data/style_args.c (revision 41170) @@ -72,4 +72,14 @@ } + if ((graphmode[0].style == KAPA_PLOT_POLYGON) || (graphmode[0].style == KAPA_PLOT_POLYFILL)) { + if ((N = get_argument (*argc, argv, "-npoint"))) { + remove_argument (N, argc, argv); + graphmode[0].ptype = atoi (argv[N]); + remove_argument (N, argc, argv); + } else { + gprint (GP_ERR, "polygon & polyfill styles require number of points argument: -npoint N\n"); + return FALSE; + } + } return (TRUE); } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/VectorIO.c (revision 41170) @@ -162,6 +162,50 @@ } +// insert a new header line, return end pointer +char *gfits_insert_header_line (Header *header, char *endptr, char *newline) { + + if (0) { + char temp1[32], temp2[32]; + memset (temp1, 0, 32); + memset (temp2, 0, 32); + + if (endptr) { + memcpy (temp1, endptr, 8); + } + memcpy (temp2, newline, 8); + fprintf (stderr, "insert: %s : %s\n", temp1, temp2); + } + + /* find the END of the header, if not supplied */ + if (!endptr) { + endptr = gfits_header_field (header, "END", 1); + if (endptr == NULL) return NULL; + } + + /* is there enough space for 1 more line? */ + if (header[0].datasize - (endptr - (header[0].buffer)) < 2*FT_LINE_LENGTH) { + // no, so expand by a full header block (FT_RECORD_SIZE = 32*80 = 2880) + header[0].datasize += FT_RECORD_SIZE; + REALLOCATE (header[0].buffer, char, header[0].datasize); + // re-find the "END" marker, in case new memory block is used + endptr = gfits_header_field (header, "END", 1); + if (endptr == NULL) return NULL; + memset (endptr + FT_LINE_LENGTH, ' ', FT_RECORD_SIZE); + } + + /* push END line back 1 */ + memmove ((endptr + FT_LINE_LENGTH), endptr, FT_LINE_LENGTH); + memset (endptr, ' ', FT_LINE_LENGTH); + + strncpy (endptr, newline, 80); + endptr += FT_LINE_LENGTH; + + return endptr; +} + +static char *rawkeywords[] = {"SIMPLE", "BITPIX", "NAXIS", "PCOUNT", "GCOUNT", "EXTEND", "EXTNAME", "BSCALE", "BZERO", "TFIELDS", "TFORM", "TZERO", "TSCAL", " ", NULL}; + // write a set of vectors to a FITS file (vectors names become fits column names) -int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) { +int WriteVectorTableFITS (char *filename, char *extname, Header *extraheader, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) { Header rawheader; @@ -191,5 +235,5 @@ rawtable.header = &rawheader; if (!WriteVectorTable (&rawtable, extname, vec, Nvec, format, (compress != NULL))) goto escape; - // NOTE : for compression, the table is constructed in native by order + // NOTE : for compression, the table is constructed in native byte order FTable *outtable = &rawtable; @@ -215,4 +259,5 @@ if (!append) { + // generate a blank PHU header Header header; Matrix matrix; @@ -228,4 +273,30 @@ } + if (extraheader) { + // copy keywords which are not the standard or table keywords + char *buf = extraheader->buffer; + char *endptr = NULL; + for (int i = 0; i < extraheader->datasize; i+= FT_LINE_LENGTH, buf += FT_LINE_LENGTH) { + + if (0) { + char temp1[32]; + memset (temp1, 0, 32); + memcpy (temp1, buf, 8); + fprintf (stderr, "buffer: %s\n", temp1); + } + + for (int j = 0; rawkeywords[j] != NULL; j++) { + if (!strncmp (buf, rawkeywords[j], strlen(rawkeywords[j]))) goto skip_insert; + } + endptr = gfits_insert_header_line (outheader, endptr, buf); + if (!endptr) { + gprint (GP_ERR, "failed to update FITS header with extra keywords\n"); + return (FALSE); + } + skip_insert: + continue; + } + } + // write the actual table data gfits_fwrite_Theader (f, outheader); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/multicommand.c (revision 41170) @@ -63,4 +63,6 @@ p = line; + int Nline = strlen(line); + done = FALSE; status = TRUE; @@ -74,82 +76,95 @@ tmpline = strncreate (p, q - p); stripwhite (tmpline); - if (*tmpline) { + myAssert (tmpline, "oops"); - if (bufferPending) { - // flush old messages - ExpectMessage (server, 0.2, &message); - bufferPending = FALSE; + // empty command, free and continue + if (*tmpline == 0) { + free (tmpline); + if (q == line + Nline) { + done = TRUE; + } else { + p = q + 1; + myAssert (p - line <= Nline, "oops"); } + continue; + } - status = command (tmpline, &outline, verbose); + if (bufferPending) { + // flush old messages + ExpectMessage (server, 0.2, &message); + bufferPending = FALSE; + } - if (status == -1) { - if (server) { - // send the command to the server instead - if (!SendMessage (server, "%s", outline)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (32); - default: - gprint (GP_ERR, "server is busy...32\n"); - bufferPending = TRUE; - goto escape; - } + // input tmpline is freed by command + status = command (tmpline, &outline, verbose); + + if (status == -1) { + if (server) { + // send the command to the server instead + if (!SendMessage (server, "%s", outline)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (32); + default: + gprint (GP_ERR, "server is busy...32\n"); + bufferPending = TRUE; + goto escape; } + } - // receive the command exit status - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (33); - default: - gprint (GP_ERR, "server is busy...33\n"); - bufferPending = TRUE; - goto escape; - } - } else { - sscanf (message.buffer, "STATUS %d", &status); - } - - // receive the resulting stderr - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (34); - default: - gprint (GP_ERR, "server is busy...34\n"); - bufferPending = TRUE; - goto escape; - } - } else { - gwrite (message.buffer, 1, message.Nbuffer, GP_ERR); - } - - // receive the resulting stdout - if (ExpectMessage (server, MSG_TIMEOUT, &message)) { - switch (errno) { - case EPIPE: - gprint (GP_ERR, "server connection has died\n"); - exit (35); - default: - gprint (GP_ERR, "server is busy...35\n"); - bufferPending = TRUE; - goto escape; - } - } else { - gwrite (message.buffer, 1, message.Nbuffer, GP_LOG); + // receive the command exit status + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (33); + default: + gprint (GP_ERR, "server is busy...33\n"); + bufferPending = TRUE; + goto escape; } } else { - // if no server is defined, we treat an unknown command as an error - status = FALSE; - } - } - escape: - if (outline != NULL) free (outline); - if (!status && auto_break) done = TRUE; + sscanf (message.buffer, "STATUS %d", &status); + } + + // receive the resulting stderr + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (34); + default: + gprint (GP_ERR, "server is busy...34\n"); + bufferPending = TRUE; + goto escape; + } + } else { + gwrite (message.buffer, 1, message.Nbuffer, GP_ERR); + } + + // receive the resulting stdout + if (ExpectMessage (server, MSG_TIMEOUT, &message)) { + switch (errno) { + case EPIPE: + gprint (GP_ERR, "server connection has died\n"); + exit (35); + default: + gprint (GP_ERR, "server is busy...35\n"); + bufferPending = TRUE; + goto escape; + } + } else { + gwrite (message.buffer, 1, message.Nbuffer, GP_LOG); + } + } else { + // if no server is defined, we treat an unknown command as an error + status = FALSE; + } } + escape: + if (outline != NULL) free (outline); + if (!status && auto_break) done = TRUE; + p = q + 1; } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/lib.shell/startup.c (revision 41170) @@ -107,4 +107,16 @@ LOAD_RC = FALSE; } + if ((N = get_argument (*argc, argv, "--no-rc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } + if ((N = get_argument (*argc, argv, "-norc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } + if ((N = get_argument (*argc, argv, "-no-rc"))) { + remove_argument (N, argc, argv); + LOAD_RC = FALSE; + } ONLY_INPUT = FALSE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/Makefile (revision 41170) @@ -26,8 +26,8 @@ $(SRC)/deimos_getobj.$(ARCH).o \ $(SRC)/deimos_fitobj.$(ARCH).o \ -$(SRC)/deimos_fitalt.$(ARCH).o \ -$(SRC)/deimos_mkalt.$(ARCH).o \ $(SRC)/deimos_mkmodel.$(ARCH).o \ $(SRC)/deimos_arclines.$(ARCH).o \ +$(SRC)/deimos_fitarc.$(ARCH).o \ +$(SRC)/deimos_fitprofile.$(ARCH).o \ $(SRC)/findrowpeaks.$(ARCH).o Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos.c (revision 41170) @@ -4,19 +4,23 @@ int deimos_fitslit (int argc, char **argv); int deimos_mkobj (int argc, char **argv); -int deimos_mkalt (int argc, char **argv); int deimos_getobj (int argc, char **argv); +// int deimos_getalt (int argc, char **argv); int deimos_fitobj (int argc, char **argv); -int deimos_fitalt (int argc, char **argv); +// int deimos_fitalt (int argc, char **argv); int deimos_arclines (int argc, char **argv); +int deimos_fitarc (int argc, char **argv); +int deimos_fitprofile (int argc, char **argv); static Command deimos_commands[] = { - {1, "fitobj", deimos_fitobj, "fit for object parameters"}, - {1, "fitalt", deimos_fitalt, "fit for object parameters using LMM"}, - {1, "getobj", deimos_getobj, "determine crude object parameters"}, - {1, "mkobj", deimos_mkobj, "make a full object image"}, - {1, "mkalt", deimos_mkalt, "make a full object image (uses deimos_make_object, for a test)"}, - {1, "mkslit", deimos_mkslit, "make a slit image"}, - {1, "fitslit", deimos_fitslit, "fit slit image to observed slit flux"}, - {1, "arclines", deimos_arclines, "detect arclines using LSF and STILT"}, + {1, "fitobj", deimos_fitobj, "fit for object parameters using LMM"}, + {1, "fitalt", deimos_fitobj, "fit for object parameters using LMM"}, + {1, "getobj", deimos_getobj, "determine crude object parameters"}, + {1, "getalt", deimos_getobj, "determine crude object parameters"}, + {1, "mkobj", deimos_mkobj, "make a full object image"}, + {1, "mkslit", deimos_mkslit, "make a slit image"}, + {1, "fitslit", deimos_fitslit, "fit slit image to observed slit flux"}, + {1, "arclines", deimos_arclines, "detect arclines using LSF and STILT"}, + {1, "fitarc", deimos_fitarc, "fit arclamp lines"}, + {1, "fitprofile", deimos_fitprofile, "fit slit profile"}, }; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_arclines.c (revision 41170) @@ -53,7 +53,14 @@ deimos_make_LSF_kernel (LSF, stilt, Nx); + // FILE *Fout = NULL; + // loop over the rows for (int iy = 0; iy < Ny; iy++) { float Fsum = 0; + float Ksum = 0; + + // if (iy == 5802) Fout = fopen ("test.5802.dat", "w"); + // if (iy == 5815) Fout = fopen ("test.5815.dat", "w"); + // cross-correlation of buffer values and kernel values for (int ix = 0; ix < Nx; ix++) { @@ -62,7 +69,16 @@ if ((iy + ky) < 0) continue; if ((iy + ky) >= Ny) continue; + if (!isfinite(Fin[ix + (iy + ky)*Nx])) continue; + if (!isfinite(kernel[ix + ko*Nx])) continue; Fsum += Fin[ix + (iy + ky)*Nx] * kernel[ix + ko*Nx]; + Ksum += kernel[ix + ko*Nx]; + // if ((iy == 5815) || (iy == 5802)) { + // fprintf (Fout, "%d %d %d : %f %f\n", iy, ix, ky, Fin[ix + (iy + ky)*Nx], kernel[ix + ko*Nx]); + // } } } + + // if ((iy == 5815) || (iy == 5802)) fclose (Fout); + coord->elements.Flt[iy] = iy; flux->elements.Flt[iy] = Fsum; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitalt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitalt.c (revision 41169) +++ (revision ) @@ -1,447 +1,0 @@ -# include "data.h" -# include "deimos.h" - -# if (0) -int deimos_fitalt (int argc, char **argv) { - OHANA_UNUSED_PARAM(argc); - OHANA_UNUSED_PARAM(argv); - return FALSE; -} - -# else - -/* - this is starting to work OK. some improvements to make - * use Gaussdev to sample - * do not scale down range (or user-set scale-down) - */ - -// internal functions to fitobj -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); -double deimos_LMM_update (float *data, float *model_ref, float *model_obj, float *model_sky, float *model_bck, float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda); -static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue); -opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow); - -int deimos_fitalt (int argc, char **argv) { - - // input parameters: - // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * PSF : point-spread function vector (flux normalized, x-dir) - // * stilt : slit tilt response : 2D kernel? - - // in-out parameters: - // * obj : vector of object flux vs y-coord (starting guess and result) - // * sky : vector of local sky signal vs y-coord - // * bck : vector of extra-slit background flux vs y-coord - - int N; - - Spline *trace = NULL; - Vector *profile = NULL; - - Vector *obj = NULL; - Vector *sky = NULL; - Vector *bck = NULL; - - Vector *psf = NULL; - - Buffer *buffer = NULL; - - float stilt = 0.0; // angle of the slit - ohana_gaussdev_init (); - - int DO_PLUS = FALSE; - if ((N = get_argument (argc, argv, "-plus"))) { - remove_argument (N, &argc, argv); - DO_PLUS = TRUE; - } - - int VERBOSE = FALSE; - if ((N = get_argument (argc, argv, "-v"))) { - remove_argument (N, &argc, argv); - VERBOSE = TRUE; - } - - // a crude noise model valid for both the guess vectors and the - // data buffer: - float gain = 1.0; - if ((N = get_argument (argc, argv, "-gain"))) { - remove_argument (N, &argc, argv); - gain = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float noise = 0.0; - if ((N = get_argument (argc, argv, "-noise"))) { - remove_argument (N, &argc, argv); - noise = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float nsigma = 2.0; - if ((N = get_argument (argc, argv, "-nsigma"))) { - remove_argument (N, &argc, argv); - nsigma = atof(argv[N]); - remove_argument (N, &argc, argv); - } - float lambda = 1.0; - if ((N = get_argument (argc, argv, "-lambda"))) { - remove_argument (N, &argc, argv); - lambda = atof(argv[N]); - remove_argument (N, &argc, argv); - } - - int Niter = 25; - if ((N = get_argument (argc, argv, "-iter"))) { - remove_argument (N, &argc, argv); - Niter = atoi(argv[N]); - remove_argument (N, &argc, argv); - } - int Nrow = 5; - if ((N = get_argument (argc, argv, "-row"))) { - remove_argument (N, &argc, argv); - Nrow = atoi(argv[N]); - remove_argument (N, &argc, argv); - } - - // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-profile"))) { - remove_argument (N, &argc, argv); - if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-psf"))) { - remove_argument (N, &argc, argv); - if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-stilt"))) { - remove_argument (N, &argc, argv); - stilt = atof (argv[N]); - remove_argument (N, &argc, argv); - } else { goto usage; } - - // In-Out parameters: - if ((N = get_argument (argc, argv, "-object"))) { - remove_argument (N, &argc, argv); - if ((obj = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-sky"))) { - remove_argument (N, &argc, argv); - if ((sky = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - if ((N = get_argument (argc, argv, "-backgnd"))) { - remove_argument (N, &argc, argv); - if ((bck = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); - remove_argument (N, &argc, argv); - } else { goto usage; } - - if (argc != 3) goto usage; - - // buffer is the full image, Xref is reference coordinate of the profile in the buffer - if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - - // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) - if (profile->Nelements % 2 == 0) { - gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); - return FALSE; - } - if (psf->Nelements % 2 == 0) { - gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); - return FALSE; - } - - // observed buffer size - int Nx = buffer[0].matrix.Naxis[0]; - int Ny = buffer[0].matrix.Naxis[1]; - float *bufVal = (float *) buffer->matrix.buffer; - - // obj, sky, bck must be consistent with data - if (Ny != obj->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (object)\n"); - return FALSE; - } - if (Ny != sky->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (sky)\n"); - return FALSE; - } - if (Ny != bck->Nelements) { - gprint (GP_ERR, "inconsistent wavelength scales (backgnd)\n"); - return FALSE; - } - - // for the cross-dispersion reference pixel, use the user value or set to Nx/2 if < 0 - deimos_set_cross_ref (atoi(argv[2]), Nx); - deimos_make_kernel (stilt, Nx); - - Vector *objMin = NULL; - Vector *skyMin = NULL; - Vector *bckMin = NULL; - - if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); - if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); - if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); - - // the functions below accept an opihi_flt array for object, sky, background - opihi_flt *objVal = objMin->elements.Flt; - opihi_flt *skyVal = skyMin->elements.Flt; - opihi_flt *bckVal = bckMin->elements.Flt; - - // we make a copy of the input guess vectors and minimize them - for (int iy = 0; iy < Ny; iy++) { - objVal[iy] = obj->elements.Flt[iy]; - skyVal[iy] = sky->elements.Flt[iy]; - bckVal[iy] = bck->elements.Flt[iy]; - } - - float noiseVar = SQ(noise); - ALLOCATE_PTR (objNoise, float, Ny); - ALLOCATE_PTR (skyNoise, float, Ny); - ALLOCATE_PTR (bckNoise, float, Ny); - - // generate noise vectors - for (int iy = 0; iy < Ny; iy++) { - objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy]) / gain, noiseVar)); - skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy]) / gain, noiseVar)); - bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy]) / gain, noiseVar)); - } - - // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. - - // XXX worry about last segment (< Nrow rows) - for (int iter = 0; iter < Niter; iter++) { - - int dRow = Nrow/2; - - // use scaled-noise to define test ranges - // we are going to try each row using a window Nrow wide to measure chisq: - for (int row = dRow; row < Ny - dRow - 1; row++) { - - // I am using a numerical equivalent of Levenberg-Marquardt, but only for a single - // p_m[i] = (obj,sky,bck) triplet [m is the index of obj,sky,bck; i is the row - // index] at a time. I start with a current best guess set of (obj,sky,bck). I - // need to calculate the model at this location, and then at offset locations p_m + - // dp_m. Note that I do not need to calculate chisq for each of these to generate - // the elements of the LMM equations: - - int row_0 = row - dRow; // start row of the subset test range - - // Nrow = 5, dRow = 2, Ny = 100, last start is Ny - 2 - 1 = 97 - // last row_0 is 95 - - // generate subset vectors for the range **centered** on row - opihi_flt *objCurr = deimos_subvector (objVal, row_0, Nrow); - opihi_flt *skyCurr = deimos_subvector (skyVal, row_0, Nrow); - opihi_flt *bckCurr = deimos_subvector (bckVal, row_0, Nrow); - - // current vector value: - // for deimos_make_model, row is the starting point of the subimage - float *model_ref = deimos_make_model (objCurr, skyCurr, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // in the test functions below, dRow is the central pixel of the subset, e.g., obj[row] => objCurr[dRow] - - float dObj = 0.0, dSky = 0.0, dBck = 0.0; - - // delta obj (save dObj) - opihi_flt *objtest = deimos_make_test (objCurr, objNoise[row], dRow, Nrow, &dObj); - float *model_obj = deimos_make_model (objtest, skyCurr, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // delta sky (save dSky) - opihi_flt *skytest = deimos_make_test (skyCurr, skyNoise[row], dRow, Nrow, &dSky); - float *model_sky = deimos_make_model (objCurr, skytest, bckCurr, psf, profile, trace, Nx, Nrow, row_0); - - // delta bck (save dBck) - opihi_flt *bcktest = deimos_make_test (bckCurr, bckNoise[row], dRow, Nrow, &dBck); - float *model_bck = deimos_make_model (objCurr, skyCurr, bcktest, psf, profile, trace, Nx, Nrow, row_0); - - // dObj, dSky, dBck carry the delta used derivatives in and the delta applied to the values out - float chisq = deimos_LMM_update (bufVal, model_ref, model_obj, model_sky, model_bck, &dObj, &dSky, &dBck, Nx, Nrow, row_0, lambda); - - if (1) { - if (DO_PLUS) { - objVal[row] += dObj; - skyVal[row] += dSky; - bckVal[row] += dBck; - } else { - objVal[row] -= dObj; - skyVal[row] -= dSky; - bckVal[row] -= dBck; - } - } - - free (objCurr); - free (skyCurr); - free (bckCurr); - - free (model_ref); - - free (model_obj); - free (model_sky); - free (model_bck); - - free (objtest); - free (skytest); - free (bcktest); - - if (VERBOSE) fprintf (stderr, "chisq: %f (%f, %f, %f)\n", chisq, dObj, dSky, dBck); - } - - float *model_full = deimos_make_model (objVal, skyVal, bckVal, psf, profile, trace, Nx, Ny, 0); - - int Npts = 0; - float chisq_full = deimos_get_chisq (bufVal, model_full, Nx, Ny, 0, &Npts); - - fprintf (stderr, "** Full chisq: %f (%d) : %f)\n", chisq_full, Npts, chisq_full / (1.0 * Npts)); - } - - return TRUE; - - usage: - gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); - return FALSE; -} - -/****************** fitobj Support Functions *******************/ - -// we generate a new vector with a single element (row) modified based on sigma -opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow) { - - ALLOCATE_PTR (value, opihi_flt, Nrow); - - for (int i = 0; i < Nrow; i++) { - value[i] = fullVec[i + row]; - } - - return value; -} - - -// we generate a new vector with a single element (row) modified based on sigma -static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue) { - - // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) - // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) - // sigma is valid for the element at row - - ALLOCATE_PTR (value, opihi_flt, Nrow); - - for (int i = 0; i < Nrow; i++) { - value[i] = guess[i]; - } - - value[row] = guess[row] + 0.1*sigma; - - *dValue = 0.1*sigma; - return value; -} - - -double deimos_LMM_update (float *data, float *model_ref, - float *model_obj, float *model_sky, float *model_bck, - float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda) { - - // we are going to loop over the images calculating the following value: - - // W = 1 / sigma^2, leave as 1.0 for now - // chisq = sum W * (data - model_ref)^2 - // dF_X = sum W * (data - model_ref) * (model_ref - model_X) / dX - // d2F_XY = sum W * (model_ref - model_X) * (model_ref - model_Y) / (dX * dY) - - double chisq = 0.0; - double dF_obj = 0.0, dF_sky = 0.0, dF_bck = 0.0; - double d2F_obj_obj = 0.0, d2F_obj_sky = 0.0, d2F_obj_bck = 0.0, d2F_sky_sky = 0.0, d2F_sky_bck = 0.0, d2F_bck_bck = 0.0; - - float W = 1.0; - - for (int iy = 0; iy < Ny; iy++) { - for (int ix = 0; ix < Nx; ix++) { - int pix = ix + iy*Nx; - int pix_data = ix + (iy + row)*Nx; - - // if any of the pixel values are NAN, skip the point: - if (!isfinite(data[pix_data])) continue; - if (!isfinite(model_ref[pix])) continue; - if (!isfinite(model_obj[pix])) continue; - if (!isfinite(model_sky[pix])) continue; - if (!isfinite(model_bck[pix])) continue; - - float df = W * (data[pix_data] - model_ref[pix]); - - chisq += SQ(df); - - float df_obj = (model_ref[pix] - model_obj[pix]) / *dObj; - float df_sky = (model_ref[pix] - model_sky[pix]) / *dSky; - float df_bck = (model_ref[pix] - model_bck[pix]) / *dBck; - - dF_obj += df*df_obj; - dF_sky += df*df_sky; - dF_bck += df*df_bck; - - d2F_obj_obj += W * SQ(df_obj); - d2F_sky_sky += W * SQ(df_sky); - d2F_bck_bck += W * SQ(df_bck); - - d2F_obj_sky += W * df_obj * df_sky; - d2F_obj_bck += W * df_obj * df_bck; - d2F_sky_bck += W * df_sky * df_bck; - } - } - - // these are the elements of Ax = B - ALLOCATE_PTR (A, double *, 3); - ALLOCATE_PTR (B, double *, 3); - for (int i = 0; i < 3; i++) { - ALLOCATE (A[i], double, 3); - ALLOCATE (B[i], double, 1); - } - - A[0][0] = d2F_obj_obj * (1.0 + lambda); - A[1][1] = d2F_sky_sky * (1.0 + lambda); - A[2][2] = d2F_bck_bck * (1.0 + lambda); - - A[0][1] = A[1][0] = d2F_obj_sky; - A[1][2] = A[2][1] = d2F_sky_bck; - A[0][2] = A[2][0] = d2F_obj_bck; - - B[0][0] = dF_obj; - B[1][0] = dF_sky; - B[2][0] = dF_bck; - - dgaussjordan (A, B, 3, 1); - - *dObj = B[0][0]; - *dSky = B[1][0]; - *dBck = B[2][0]; - - return chisq; -} - -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { - - int npts = 0; - float chisq = 0; - - for (int iy = 0; iy < Ny; iy++) { - - for (int ix = 0; ix < Nx; ix++) { - - int pix_buffer = ix + (iy + row)*Nx; - int pix_model = ix + iy*Nx; - - if (!isfinite(buffer[pix_buffer])) continue; - if (!isfinite(model[pix_model])) continue; - chisq += SQ(buffer[pix_buffer] - model[pix_model]); - npts ++; - } - } - *Npts = npts; - return chisq; -} -# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitarc.c (revision 41170) @@ -0,0 +1,237 @@ +# include "data.h" +# define NPAR 4 + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); + +opihi_flt fgaussOD (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar); + +void gaussian_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar); +void gaussian_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC); + +double deimos_weight_cauchy (double x); + +// use these globals to carry into sigmoid_fit +static int VERBOSE = FALSE; +static int QUIET = FALSE; +static int MaxIterations = 10; + +/* fitarc buffer Xo Yo Nx Ny + + * for each col, fit the y-dir profile with a gauss + * fit the C0 values to a line to get the angle + */ + +int deimos_fitarc (int argc, char **argv) { + + // input parameters: + // * xprofile : extracted slit window coordinates (vector) + // * xprofile : extracted slit window profile (vector) + // * Xs, Xe : scale coordinates of window boundaries + + // output parameters: + // * sigmoid_left_center, etc : coordinates of window boundaries + + int N; + Buffer *buffer = NULL; + + QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + MaxIterations = 10; + if ((N = get_argument (argc, argv, "-max-iterations"))) { + remove_argument (N, &argc, argv); + MaxIterations = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + Buffer *residbuf = NULL; + if ((N = get_argument (argc, argv, "-resid"))) { + remove_argument (N, &argc, argv); + residbuf = SelectBuffer (argv[N], ANYBUFFER, TRUE); + if (!residbuf) goto usage; + remove_argument (N, &argc, argv); + } + + Buffer *varbuf = NULL; + if ((N = get_argument (argc, argv, "-varbuf"))) { + remove_argument (N, &argc, argv); + varbuf = SelectBuffer (argv[N], OLDBUFFER, TRUE); + if (!varbuf) goto usage; + remove_argument (N, &argc, argv); + } + + if (argc != 6) goto usage; + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + int nx = buffer[0].matrix.Naxis[0]; + int ny = buffer[0].matrix.Naxis[1]; + + // supplied guess for boundaries + int Xs = atof(argv[2]); + int Ys = atof(argv[3]); + int Nx = atof(argv[4]); + int Ny = atof(argv[5]); + + if (residbuf) { + ResetBuffer (residbuf, nx, ny, -32, 0.0, 1.0); + } + + // validate Xs, Ys, Nx, Ny + // Xs + Nx <= nx + + // loop over the cols + + ALLOCATE_PTR (xcol, double, Ny); + ALLOCATE_PTR (ycol, double, Ny); + ALLOCATE_PTR (ocol, double, Ny); // fitted value + + float *bufVal = (float *) buffer->matrix.buffer; + + for (int ix = Xs; ix < Xs + Nx; ix++) { + + // loop over the pixels in this column to get the x and y vectors + // find the peak at the same time + + double xmax = -HUGE_VAL; + double ymax = -HUGE_VAL; + int Nc = 0; + + for (int iy = 0; iy < Ny; iy++) { + + float yval = bufVal[ix + (iy + Ys)*Nx]; + if (!isfinite(yval)) continue; + + xcol[Nc] = iy + Ys; + ycol[Nc] = yval; + + if (yval > ymax) { + ymax = yval; + xmax = iy + Ys; + } + Nc ++; + } + + // now do a Gauss fit given C0 = xmax, C1 = 2, C2 = ymax, C3 = 0 + + // temporary storage vectors + opihi_flt par[NPAR]; + par[0] = xmax; + par[1] = 2; + par[2] = ymax; + par[3] = 0; + gaussian_fit_irls (xcol, ycol, NULL, ocol, Nc, par, NPAR); + + fprintf (stderr, "%d : %f %f : %f %f\n", ix, par[0], par[2], par[1], par[3]); + set_variable ("C0", par[0]); + set_variable ("C1", par[1]); + set_variable ("C2", par[2]); + set_variable ("C3", par[3]); + + if (residbuf) { + float *residVal = (float *) residbuf->matrix.buffer; + for (int iy = 0; iy < Ny; iy++) { + float yval = fgaussOD (iy + Ys, par, NPAR, NULL); + // residVal[ix + (iy + Ys)*Nx] = bufVal[ix + (iy + Ys)*Nx] - yval; + residVal[ix + (iy + Ys)*Nx] = yval; + } + } + } + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos fitarc buff Xo Yo Nx Ny\n"); + gprint (GP_ERR, " [-resid buffer] : output residual image\n"); + gprint (GP_ERR, " [-varbuf buffer] : input variance image\n"); + return FALSE; +} + +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 + +// xprofile : x-coordinate +// fprofile : flux (counts) +// wprofile : weight (true poisson error) +// oprofile : output profile (not used?) +void gaussian_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar) { + + // buffer to store result from each iteration + ALLOCATE_PTR (par_old, opihi_flt, Npar); + + // define initial weights as 1.0 + ALLOCATE_PTR (wt, opihi_flt, Npts); + ALLOCATE_PTR (wtemp, opihi_flt, Npts); + for (int i = 0; i < Npts; i++) { + wtemp[i] = wprofile ? (fabs(wprofile[i]) > 1e-5 ? wprofile[i] : 1e-5) : 1.0; + wt[i] = 1.0 / SQ(wtemp[i]); + } + + // initial fit with ordinary least-squares + // otemp is the fitted value of the function at xtemp + gaussian_fit (xprofile, fprofile, wt, NULL, Npts, par, Npar, fgaussOD); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < MaxIterations); iterations++) { + + for (int i = 0; i < Npts; i++) { + wt[i] = deimos_weight_cauchy ((fprofile[i] - oprofile[i]) / wtemp[i]) / SQ(wtemp[i]); + } + + gaussian_fit (xprofile, fprofile, wt, oprofile, Npts, par, Npar, fgaussOD); + + // save this solution + for (int i = 0; i < Npar; i++) { + par_old[i] = par[i]; + } + + converged = TRUE; + for (int i = 0; i < Npts; i++) { + if ((fabs(par[i] - par_old[i]) > FIT_TOLERANCE * fabs(par[i])) && + (fabs(par[i] - par_old[i]) > FLT_TOLERANCE)) + converged = FALSE; + } + } + + for (int i = 0; i < Npts; i++) { + oprofile[i] = fgaussOD (xprofile[i], par, Npar, NULL); + } + + FREE (par_old); + FREE (wt); +} + +void gaussian_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC) { + + double ochisq = mrqinit (xprofile, fprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + double dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + double chisq = mrqmin (xprofile, fprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + dchisq = ochisq - chisq; + ochisq = chisq; + if (VERBOSE) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (VERBOSE) gprint (GP_ERR, "%d iterations\n", Niter); + + if (!QUIET) gprint (GP_LOG, "gauss @ %f (%f) : top %f bottom %f\n", par[0], par[1], par[2], par[3]); + + if (oprofile) { + for (int i = 0; i < Npts; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + } + + mrqfree (Npar); +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.c (revision 41170) @@ -1,13 +1,4 @@ # include "data.h" # include "deimos.h" - -# if (0) -int deimos_fitobj (int argc, char **argv) { - OHANA_UNUSED_PARAM(argc); - OHANA_UNUSED_PARAM(argv); - return FALSE; -} - -# else /* @@ -18,19 +9,19 @@ // internal functions to fitobj -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); -int *deimos_sort_result (DeimosResult *result, int Nresult); -int deimos_get_random_sample (int Nresult); -static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput); - -int USE_GAUSS_DEV = FALSE; +static float deimos_get_chisq (float *buffer, float *vardata, float *model, int Nx, int Ny, int row, int *Npts); +double deimos_LMM_update (float *data, float *vardata, float *model_ref, float *model_obj, float *model_sky, float *model_bck, float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda); +static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue); +opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow); int deimos_fitobj (int argc, char **argv) { // input parameters: - // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * PSF : point-spread function vector (flux normalized, x-dir) - // * stilt : slit tilt response : 2D kernel? + // * buffer : 2D image of slit (full frame or cutout?) + // * slit_trace_red : spline fit of slit central x pos vs y-coord + // * slit_trace_blu : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of slit central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? // in-out parameters: @@ -41,17 +32,48 @@ int N; - Spline *trace = NULL; - Vector *profile = NULL; - - Vector *obj = NULL; - Vector *sky = NULL; - Vector *bck = NULL; - - Vector *psf = NULL; - - Buffer *buffer = NULL; - - float stilt = 0.0; // angle of the slit + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; + + Vector *profile = NULL; + Vector *psf = NULL; + + Vector *obj = NULL; + Vector *sky = NULL; + Vector *bck = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + ohana_gaussdev_init (); + + int DO_PLUS = FALSE; + if ((N = get_argument (argc, argv, "-plus"))) { + remove_argument (N, &argc, argv); + DO_PLUS = TRUE; + } + + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + + int SKIPFIT = FALSE; + if ((N = get_argument (argc, argv, "-skipfit"))) { + remove_argument (N, &argc, argv); + SKIPFIT = TRUE; + } + + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } // a crude noise model valid for both the guess vectors and the @@ -75,11 +97,10 @@ remove_argument (N, &argc, argv); } - float scale = 0.75; - if ((N = get_argument (argc, argv, "-scale"))) { - remove_argument (N, &argc, argv); - scale = atof(argv[N]); - remove_argument (N, &argc, argv); - } - + float lambda = 1.0; + if ((N = get_argument (argc, argv, "-lambda"))) { + remove_argument (N, &argc, argv); + lambda = atof(argv[N]); + remove_argument (N, &argc, argv); + } int Niter = 25; @@ -89,10 +110,4 @@ remove_argument (N, &argc, argv); } - int Ntry = 5; - if ((N = get_argument (argc, argv, "-try"))) { - remove_argument (N, &argc, argv); - Ntry = atoi(argv[N]); - remove_argument (N, &argc, argv); - } int Nrow = 5; if ((N = get_argument (argc, argv, "-row"))) { @@ -102,26 +117,19 @@ } - USE_GAUSS_DEV = FALSE; - if ((N = get_argument (argc, argv, "-gaussdev"))) { - remove_argument (N, &argc, argv); - USE_GAUSS_DEV = TRUE; - } - - int SAVE_ALL_VECTORS = FALSE; - if ((N = get_argument (argc, argv, "-save-all-vectors"))) { - remove_argument (N, &argc, argv); - SAVE_ALL_VECTORS = TRUE; - } - - int SAVE_MIN_VECTORS = FALSE; - if ((N = get_argument (argc, argv, "-save-min-vectors"))) { - remove_argument (N, &argc, argv); - SAVE_MIN_VECTORS = TRUE; - } - // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } else { goto usage; } @@ -158,4 +166,20 @@ remove_argument (N, &argc, argv); } else { goto usage; } + + // save the model + Buffer *model = NULL; + if ((N = get_argument (argc, argv, "-model"))) { + remove_argument (N, &argc, argv); + if ((model = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + // save the model + Buffer *varim = NULL; + if ((N = get_argument (argc, argv, "-variance"))) { + remove_argument (N, &argc, argv); + if ((varim = SelectBuffer (argv[N], OLDBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } if (argc != 3) goto usage; @@ -179,4 +203,10 @@ float *bufVal = (float *) buffer->matrix.buffer; + if (varim) { + if (Nx != varim[0].matrix.Naxis[0]) { gprint (GP_ERR, "inconsistent sizes for image and varim\n"); return FALSE; } + if (Ny != varim[0].matrix.Naxis[1]) { gprint (GP_ERR, "inconsistent sizes for image and varim\n"); return FALSE; } + } + float *varVal = varim ? (float *) varim->matrix.buffer : NULL; + // obj, sky, bck must be consistent with data if (Ny != obj->Nelements) { @@ -193,240 +223,141 @@ } - // for the cross-dispersion reference pixle, use the user value or set to Nx/2 if < 0 + if (model) { + if (!CreateBuffer (model, Nx, Ny, -32, 1.0, 0.0)) { gprint (GP_ERR, "error generating output model buffer\n"); return FALSE; } + } + + // for the cross-dispersion reference pixel, use the user value or set to Nx/2 if < 0 deimos_set_cross_ref (atoi(argv[2]), Nx); deimos_make_kernel (stilt, Nx); - // the functions below accept an opihi_flt array for object, sky, background - opihi_flt *objVal = obj->elements.Flt; - opihi_flt *skyVal = sky->elements.Flt; - opihi_flt *bckVal = bck->elements.Flt; - - int NRESULT = Niter + Ntry*Niter + 1; - ALLOCATE_PTR (result, DeimosResult, NRESULT); - - Vector **objOut = NULL; - Vector **skyOut = NULL; - Vector **bckOut = NULL; Vector *objMin = NULL; Vector *skyMin = NULL; Vector *bckMin = NULL; - // Save all result vectors for each of obj, sky, bck: - if (SAVE_ALL_VECTORS) { - ALLOCATE (objOut, Vector *, NRESULT); - ALLOCATE (skyOut, Vector *, NRESULT); - ALLOCATE (bckOut, Vector *, NRESULT); - for (int i = 0; i < NRESULT; i++) { - char name[64]; - snprintf (name, 64, "obj_%04d", i); if ((objOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objOut[i], OPIHI_FLT, Ny); - snprintf (name, 64, "sky_%04d", i); if ((skyOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyOut[i], OPIHI_FLT, Ny); - snprintf (name, 64, "bck_%04d", i); if ((bckOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckOut[i], OPIHI_FLT, Ny); - for (int iy = 0; iy < Ny; iy++) { - objOut[i]->elements.Flt[iy] = 0.0; - skyOut[i]->elements.Flt[iy] = 0.0; - bckOut[i]->elements.Flt[iy] = 0.0; + if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); + if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); + if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); + + // the functions below accept an opihi_flt array for object, sky, background + opihi_flt *objVal = objMin->elements.Flt; + opihi_flt *skyVal = skyMin->elements.Flt; + opihi_flt *bckVal = bckMin->elements.Flt; + + // we make a copy of the input guess vectors and minimize them + for (int iy = 0; iy < Ny; iy++) { + objVal[iy] = obj->elements.Flt[iy]; + skyVal[iy] = sky->elements.Flt[iy]; + bckVal[iy] = bck->elements.Flt[iy]; + } + + float noiseVar = SQ(noise); + ALLOCATE_PTR (objNoise, float, Ny); + ALLOCATE_PTR (skyNoise, float, Ny); + ALLOCATE_PTR (bckNoise, float, Ny); + + // generate noise vectors + for (int iy = 0; iy < Ny; iy++) { + objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy]) / gain, noiseVar)); + skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy]) / gain, noiseVar)); + bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy]) / gain, noiseVar)); + } + + // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. + + // XXX worry about last segment (< Nrow rows) + for (int iter = 0; iter < Niter; iter++) { + + int dRow = Nrow/2; + + // use scaled-noise to define test ranges + // we are going to try each row using a window Nrow wide to measure chisq: + for (int row = dRow; !SKIPFIT && (row < Ny - dRow - 1); row++) { + + // I am using a numerical equivalent of Levenberg-Marquardt, but only for a single + // p_m[i] = (obj,sky,bck) triplet [m is the index of obj,sky,bck; i is the row + // index] at a time. I start with a current best guess set of (obj,sky,bck). I + // need to calculate the model at this location, and then at offset locations p_m + + // dp_m. Note that I do not need to calculate chisq for each of these to generate + // the elements of the LMM equations: + + int row_0 = row - dRow; // start row of the subset test range + + // Nrow = 5, dRow = 2, Ny = 100, last start is Ny - 2 - 1 = 97 + // last row_0 is 95 + + // generate subset vectors for the range **centered** on row + opihi_flt *objCurr = deimos_subvector (objVal, row_0, Nrow); + opihi_flt *skyCurr = deimos_subvector (skyVal, row_0, Nrow); + opihi_flt *bckCurr = deimos_subvector (bckVal, row_0, Nrow); + + // current vector value: + // for deimos_make_model, row is the starting point of the subimage + float *model_ref = deimos_make_model (objCurr, skyCurr, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // in the test functions below, dRow is the central pixel of the subset, e.g., obj[row] => objCurr[dRow] + + float dObj = 0.0, dSky = 0.0, dBck = 0.0; + + // delta obj (save dObj) + opihi_flt *objtest = deimos_make_test (objCurr, objNoise[row], dRow, Nrow, &dObj); + float *model_obj = deimos_make_model (objtest, skyCurr, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // delta sky (save dSky) + opihi_flt *skytest = deimos_make_test (skyCurr, skyNoise[row], dRow, Nrow, &dSky); + float *model_sky = deimos_make_model (objCurr, skytest, bckCurr, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // delta bck (save dBck) + opihi_flt *bcktest = deimos_make_test (bckCurr, bckNoise[row], dRow, Nrow, &dBck); + float *model_bck = deimos_make_model (objCurr, skyCurr, bcktest, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Nrow, row_0); + + // dObj, dSky, dBck carry the delta used derivatives in and the delta applied to the values out + float chisq = deimos_LMM_update (bufVal, varVal, model_ref, model_obj, model_sky, model_bck, &dObj, &dSky, &dBck, Nx, Nrow, row_0, lambda); + + if (FALSE && (row > 10) && (row < 20)) { + fprintf (stderr, "%d %d : %f %f %f\n", iter, row, dObj, dSky, dBck); } - } - } - if (SAVE_MIN_VECTORS) { - if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); - if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); - if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); - for (int iy = 0; iy < Ny; iy++) { - objMin->elements.Flt[iy] = 0.0; - skyMin->elements.Flt[iy] = 0.0; - bckMin->elements.Flt[iy] = 0.0; - } - } - Vector *chisqVect; - if ((chisqVect = SelectVector ("chisqOut", ANYVECTOR, TRUE)) == NULL) { return FALSE; } - ResetVector (chisqVect, OPIHI_FLT, NRESULT); - for (int i = 0; i < NRESULT; i++) { chisqVect->elements.Flt[i] = 0.0; } - - // XXX temp hack: save chisq subset vectors for each row pass - int Npass = Ny / Nrow; - ALLOCATE_PTR (chiPass, Vector *, Npass); - for (int pass = 0; pass < Npass; pass++) { - char name[64]; - snprintf (name, 64, "chiPass_%03d", pass); - if ((chiPass[pass] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return FALSE; } ResetVector (chiPass[pass], OPIHI_FLT, NRESULT); - } - - float noiseVar = SQ(noise); - ALLOCATE_PTR (objNoise, float, Nrow); - ALLOCATE_PTR (skyNoise, float, Nrow); - ALLOCATE_PTR (bckNoise, float, Nrow); - - // work on segments of Nrow at a time - // XXX worry about last segment (< Nrow rows) - int pass = 0; - for (int row = 0; row < Ny - Nrow; row += Nrow, pass++) { - - int Nresult = 0; - - // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. - - // If I just use the value in a given bin to define the range, random noise for the - // faint end could have bad results (range ~ 0.0). For that matter, if I am in a - // region with large dynamic range (e.g., on an emission or absorption line), then we - // likely have large errors in how the guess follows the truth. - - /** old concept: - float objRange = 0, skyRange = 0, bckRange = 0; - for (int iy = 0; iy < Nrow; iy++) { - objRange = MAX(objVal[iy + row], objRange); - skyRange = MAX(skyVal[iy + row], skyRange); - bckRange = MAX(bckVal[iy + row], bckRange); - } - **/ - - // new concept: start with 2-sigma range, then ramp down below - for (int iy = 0; iy < Nrow; iy++) { - objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy + row]) / gain, noiseVar)); - skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy + row]) / gain, noiseVar)); - bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy + row]) / gain, noiseVar)); - } - - // first test is the input guess - if (1) { - opihi_flt *objtest = deimos_make_test (objVal, NULL, row, Nrow, TRUE); - opihi_flt *skytest = deimos_make_test (skyVal, NULL, row, Nrow, TRUE); - opihi_flt *bcktest = deimos_make_test (bckVal, NULL, row, Nrow, TRUE); - - // generate the model based on the test values - // objtest, skytest, bcktest are segments of the full parameter vectors - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - // XXX need to add in the per-pixel error or variance - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - fprintf (stderr, "guess: chisq: %f, Npts: %d\n", chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; - } - - // save the results - result[Nresult].chisq = chisq / (float) Npts; - result[Nresult].obj = objtest; - result[Nresult].sky = skytest; - result[Nresult].bck = bcktest; - Nresult ++; - } - - // run Niter iterations - for (int iter = 0; iter < Niter; iter ++) { - - // generate a set of test values for obj, sky, backgnd based on the current guess - opihi_flt *objtest = deimos_make_test (objVal, objNoise, row, Nrow, TRUE); - opihi_flt *skytest = deimos_make_test (skyVal, skyNoise, row, Nrow, TRUE); - opihi_flt *bcktest = deimos_make_test (bckVal, bckNoise, row, Nrow, TRUE); - - for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { - fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", - objVal[itmp + row], skyVal[itmp + row], bckVal[itmp + row], - objtest[itmp], skytest[itmp], bcktest[itmp]); - } - - // generate the model based on the test values - // objtest, skytest, bcktest are segments of the full parameter vectors - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - // XXX need to add in the per-pixel error or variance - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - // fprintf (stderr, "iter: %d, chisq: %f, Npts: %d\n", iter, chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; - } - - // save the results - result[Nresult].chisq = chisq / (float) Npts; - result[Nresult].obj = objtest; - result[Nresult].sky = skytest; - result[Nresult].bck = bcktest; - Nresult ++; - } - - for (int try = 0; try < Ntry; try ++) { - // generate an index of results sorted by chisq - int *IDX = deimos_sort_result (result, Nresult); - - int Nnewset = 0; - - // XXX not sure this scaling factor concept is good - for (int iy = 0; iy < Nrow; iy++) { - objNoise[iy] = scale*objNoise[iy]; - skyNoise[iy] = scale*skyNoise[iy]; - bckNoise[iy] = scale*bckNoise[iy]; - } - - // run Niter iterations, selecting guess values from the result set - for (int iter = 0; iter < Niter; iter ++) { - int bin = deimos_get_random_sample (Nresult); - int entry = IDX[bin]; - // fprintf (stderr, "try: %d : %d, %f\n", bin, entry, result[entry].chisq); - - // generate a set of test values for obj, sky, backgnd based on the current guess - opihi_flt *objtest = deimos_make_test (result[entry].obj, objNoise, row, Nrow, FALSE); - opihi_flt *skytest = deimos_make_test (result[entry].sky, skyNoise, row, Nrow, FALSE); - opihi_flt *bcktest = deimos_make_test (result[entry].bck, bckNoise, row, Nrow, FALSE); - - for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { - fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", - result[entry].obj[itmp], result[entry].sky[itmp], result[entry].bck[itmp], - objtest[itmp], skytest[itmp], bcktest[itmp]); - } - - // generate the model based on the test values - float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, Nx, Nrow, row); - - // calculate the current chisq - int Npts = 0; - float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); - // fprintf (stderr, "try: %d, iter: %d, chisq: %f, Npts: %d\n", try, iter, chisq, Npts); - free (model); - - if (pass < Npass) { - chiPass[pass]->elements.Flt[Nresult + Nnewset] = chisq / (float) Npts; - } - - // save the results - result[Nresult + Nnewset].chisq = chisq / (float) Npts; - result[Nresult + Nnewset].obj = objtest; - result[Nresult + Nnewset].sky = skytest; - result[Nresult + Nnewset].bck = bcktest; - Nnewset ++; - } - Nresult += Nnewset; - } - - int *IDX = deimos_sort_result (result, Nresult); - - // ENDING: save the best result and info about the allowed chisq region - // int *index = sort_result (result, Nresult); - // copy result[index[0]].obj,sky,bck to obj,sky,bck - for (int i = 0; i < Nresult; i++) { - for (int iy = 0; iy < Nrow; iy++) { - if (SAVE_ALL_VECTORS) { - objOut[i]->elements.Flt[iy + row] = result[IDX[i]].obj[iy]; - skyOut[i]->elements.Flt[iy + row] = result[IDX[i]].sky[iy]; - bckOut[i]->elements.Flt[iy + row] = result[IDX[i]].bck[iy]; + + if (1) { + if (DO_PLUS) { + objVal[row] += dObj; + skyVal[row] += dSky; + bckVal[row] += dBck; + } else { + objVal[row] -= dObj; + skyVal[row] -= dSky; + bckVal[row] -= dBck; } } - chisqVect->elements.Flt[i] += result[IDX[i]].chisq; + + free (objCurr); + free (skyCurr); + free (bckCurr); + + free (model_ref); + + free (model_obj); + free (model_sky); + free (model_bck); + + free (objtest); + free (skytest); + free (bcktest); + + if (VERBOSE) fprintf (stderr, "chisq: %f (%f, %f, %f)\n", chisq, dObj, dSky, dBck); } - if (SAVE_MIN_VECTORS) { - for (int iy = 0; iy < Nrow; iy++) { - objMin->elements.Flt[iy + row] = result[IDX[0]].obj[iy]; - skyMin->elements.Flt[iy + row] = result[IDX[0]].sky[iy]; - bckMin->elements.Flt[iy + row] = result[IDX[0]].bck[iy]; - } + + float *model_full = deimos_make_model (objVal, skyVal, bckVal, psf, profile, slit_trace_red, slit_trace_blu, redlimit, psf_trace, Nx, Ny, 0); + + int Npts = 0; + float chisq_full = deimos_get_chisq (bufVal, varVal, model_full, Nx, Ny, 0, &Npts); + + if (model) { + free (model[0].matrix.buffer); + model[0].matrix.buffer = (char *) model_full; + } else { + free (model_full); } + + fprintf (stderr, "** Full chisq: %f (%d) : %f)\n", chisq_full, Npts, chisq_full / (1.0 * Npts)); } @@ -434,5 +365,6 @@ usage: - gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -slit-trace-red (spline) -slit-trace-blu (spline) -psf-trace (spline) -profile vector -psf vector -stilt (angle)\n"); + gprint (GP_ERR, "OPTIONS: [-model image] [-variance image] [-gain value] [-noise value] [-nsigma value] [-lambda value] [-iter N] [-row N] [-redlimit Npixel] [-skipfit] [-v] [-plus]\n"); return FALSE; } @@ -440,38 +372,120 @@ /****************** fitobj Support Functions *******************/ -static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput) { - - // we have a vector, guess, with some length >= row + Nrow. - // generate a set of random values in the vicinty of guess[row] to guess[row+Nrow] - // return vector is of length Nrow +// we generate a new vector with a single element (row) modified based on sigma +opihi_flt *deimos_subvector (opihi_flt *fullVec, int row, int Nrow) { + + ALLOCATE_PTR (value, opihi_flt, Nrow); + + for (int i = 0; i < Nrow; i++) { + value[i] = fullVec[i + row]; + } + + return value; +} + + +// we generate a new vector with a single element (row) modified based on sigma +static opihi_flt *deimos_make_test (opihi_flt *guess, float sigma, int row, int Nrow, float *dValue) { // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) - // sigma is always only Nrow long + // sigma is valid for the element at row ALLOCATE_PTR (value, opihi_flt, Nrow); - int start_pix = fullInput ? row : 0; - int stop_pix = fullInput ? row + Nrow : Nrow; - - int iv = 0; - for (int i = start_pix; i < stop_pix; i++, iv++) { - if (!sigma) { - value[iv] = guess[i]; - continue; - } - if (USE_GAUSS_DEV) { - value[iv] = ohana_gaussdev_rnd(guess[i], sigma[iv]); - continue; - } - - // uniform distribution: - value[iv] = sigma[iv]*(drand48() - 0.5) + guess[i]; - } + for (int i = 0; i < Nrow; i++) { + value[i] = guess[i]; + } + + value[row] = guess[row] + 0.1*sigma; + + *dValue = 0.1*sigma; return value; } -static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { +double deimos_LMM_update (float *data, float *vardata, float *model_ref, + float *model_obj, float *model_sky, float *model_bck, + float *dObj, float *dSky, float *dBck, int Nx, int Ny, int row, float lambda) { + + // we are going to loop over the images calculating the following value: + + // W = 1 / sigma^2, leave as 1.0 for now + // chisq = sum W * (data - model_ref)^2 + // dF_X = sum W * (data - model_ref) * (model_ref - model_X) / dX + // d2F_XY = sum W * (model_ref - model_X) * (model_ref - model_Y) / (dX * dY) + + double chisq = 0.0; + double dF_obj = 0.0, dF_sky = 0.0, dF_bck = 0.0; + double d2F_obj_obj = 0.0, d2F_obj_sky = 0.0, d2F_obj_bck = 0.0, d2F_sky_sky = 0.0, d2F_sky_bck = 0.0, d2F_bck_bck = 0.0; + + for (int iy = 0; iy < Ny; iy++) { + for (int ix = 0; ix < Nx; ix++) { + int pix = ix + iy*Nx; + int pix_data = ix + (iy + row)*Nx; + + // if any of the pixel values are NAN, skip the point: + if (!isfinite(data[pix_data])) continue; + if (!isfinite(model_ref[pix])) continue; + if (!isfinite(model_obj[pix])) continue; + if (!isfinite(model_sky[pix])) continue; + if (!isfinite(model_bck[pix])) continue; + + if (vardata && !isfinite(vardata[pix_data])) continue; + float Var = vardata ? vardata[pix_data] : 1.0; + float W = (fabs(Var) < 0.01) ? 100.0 : 1.0 / Var; + + float df = W * (data[pix_data] - model_ref[pix]); + + chisq += SQ(df); + + float df_obj = (model_ref[pix] - model_obj[pix]) / *dObj; + float df_sky = (model_ref[pix] - model_sky[pix]) / *dSky; + float df_bck = (model_ref[pix] - model_bck[pix]) / *dBck; + + dF_obj += df*df_obj; + dF_sky += df*df_sky; + dF_bck += df*df_bck; + + d2F_obj_obj += W * SQ(df_obj); + d2F_sky_sky += W * SQ(df_sky); + d2F_bck_bck += W * SQ(df_bck); + + d2F_obj_sky += W * df_obj * df_sky; + d2F_obj_bck += W * df_obj * df_bck; + d2F_sky_bck += W * df_sky * df_bck; + } + } + + // these are the elements of Ax = B + ALLOCATE_PTR (A, double *, 3); + ALLOCATE_PTR (B, double *, 3); + for (int i = 0; i < 3; i++) { + ALLOCATE (A[i], double, 3); + ALLOCATE (B[i], double, 1); + } + + A[0][0] = d2F_obj_obj * (1.0 + lambda); + A[1][1] = d2F_sky_sky * (1.0 + lambda); + A[2][2] = d2F_bck_bck * (1.0 + lambda); + + A[0][1] = A[1][0] = d2F_obj_sky; + A[1][2] = A[2][1] = d2F_sky_bck; + A[0][2] = A[2][0] = d2F_obj_bck; + + B[0][0] = dF_obj; + B[1][0] = dF_sky; + B[2][0] = dF_bck; + + dgaussjordan (A, B, 3, 1); + + *dObj = B[0][0]; + *dSky = B[1][0]; + *dBck = B[2][0]; + + return chisq; +} + +static float deimos_get_chisq (float *buffer, float *vardata, float *model, int Nx, int Ny, int row, int *Npts) { int npts = 0; @@ -487,5 +501,12 @@ if (!isfinite(buffer[pix_buffer])) continue; if (!isfinite(model[pix_model])) continue; - chisq += SQ(buffer[pix_buffer] - model[pix_model]); + + if (vardata && !isfinite(vardata[pix_buffer])) continue; + float Var = vardata ? vardata[pix_buffer] : 1.0; + float W = (fabs(Var) < 0.01) ? 100.0 : 1.0 / Var; + + float dchisq = SQ(buffer[pix_buffer] - model[pix_model]) * W; + if (!isfinite(dchisq)) continue; + chisq += dchisq; npts ++; } @@ -494,42 +515,2 @@ return chisq; } - -int *deimos_sort_result (DeimosResult *result, int Nresult) { - - ALLOCATE_PTR (IDX, int, Nresult); - ALLOCATE_PTR (chisq, float, Nresult); - - for (int i = 0; i < Nresult; i++) { - IDX[i] = i; - chisq[i] = result[i].chisq; - } - - sort_float_index (chisq, IDX, Nresult); - free (chisq); - - return IDX; -} - -static float DEIMOS_A = 2.0; // value for random sample - -int deimos_get_random_sample (int Nresult) { - - // choose a bin from the range 0 - Nresult, with front-loaded weighting. - - // bin = Nresult * (exp(q) - exp(-A)) / (exp(A) - exp(-A)) - // q = 2A(x - 1/2) : random number between -A and +A - // x : random number between 0 and 1 - - // f(q;A) = (exp(q) - exp(-A)) / (exp(A) - exp(-A)) - // f(q;A) ranges from 0 to +1 as x ranges from 0 to +1 - - float x = drand48(); - float q = 2*DEIMOS_A*(x - 0.5); - float f = (exp(q) - exp(-DEIMOS_A)) / (exp(DEIMOS_A) - exp(-DEIMOS_A)); - // XXX optimization : precalculate and save the elements of this function - - int bin = floor(Nresult * f); - - return bin; -} -# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitobj.v0.c (revision 41170) @@ -0,0 +1,538 @@ +# include "data.h" +# include "deimos.h" + +# if (0) +int deimos_fitobj (int argc, char **argv) { + OHANA_UNUSED_PARAM(argc); + OHANA_UNUSED_PARAM(argv); + return FALSE; +} + +# else + +/* + this is starting to work OK. some improvements to make + * use Gaussdev to sample + * do not scale down range (or user-set scale-down) + */ + +// internal functions to fitobj +static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts); +int *deimos_sort_result (DeimosResult *result, int Nresult); +int deimos_get_random_sample (int Nresult); +static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput); + +int USE_GAUSS_DEV = FALSE; + +int deimos_fitobj (int argc, char **argv) { + + // input parameters: + // * buffer : 2D image of slit (full frame or cutout?) + // * trace : spline fit of slit central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? + + // in-out parameters: + // * obj : vector of object flux vs y-coord (starting guess and result) + // * sky : vector of local sky signal vs y-coord + // * bck : vector of extra-slit background flux vs y-coord + + int N; + + Spline *trace = NULL; + Vector *profile = NULL; + + Vector *obj = NULL; + Vector *sky = NULL; + Vector *bck = NULL; + + Vector *psf = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + ohana_gaussdev_init (); + + fprintf (stderr, "use deimos fitalt instead\n"); + return FALSE; + + // a crude noise model valid for both the guess vectors and the + // data buffer: + float gain = 1.0; + if ((N = get_argument (argc, argv, "-gain"))) { + remove_argument (N, &argc, argv); + gain = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float noise = 0.0; + if ((N = get_argument (argc, argv, "-noise"))) { + remove_argument (N, &argc, argv); + noise = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float nsigma = 2.0; + if ((N = get_argument (argc, argv, "-nsigma"))) { + remove_argument (N, &argc, argv); + nsigma = atof(argv[N]); + remove_argument (N, &argc, argv); + } + float scale = 0.75; + if ((N = get_argument (argc, argv, "-scale"))) { + remove_argument (N, &argc, argv); + scale = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + + int Niter = 25; + if ((N = get_argument (argc, argv, "-iter"))) { + remove_argument (N, &argc, argv); + Niter = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int Ntry = 5; + if ((N = get_argument (argc, argv, "-try"))) { + remove_argument (N, &argc, argv); + Ntry = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + int Nrow = 5; + if ((N = get_argument (argc, argv, "-row"))) { + remove_argument (N, &argc, argv); + Nrow = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + + USE_GAUSS_DEV = FALSE; + if ((N = get_argument (argc, argv, "-gaussdev"))) { + remove_argument (N, &argc, argv); + USE_GAUSS_DEV = TRUE; + } + + int SAVE_ALL_VECTORS = FALSE; + if ((N = get_argument (argc, argv, "-save-all-vectors"))) { + remove_argument (N, &argc, argv); + SAVE_ALL_VECTORS = TRUE; + } + + int SAVE_MIN_VECTORS = FALSE; + if ((N = get_argument (argc, argv, "-save-min-vectors"))) { + remove_argument (N, &argc, argv); + SAVE_MIN_VECTORS = TRUE; + } + + // Input parameters: + if ((N = get_argument (argc, argv, "-trace"))) { + remove_argument (N, &argc, argv); + if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-profile"))) { + remove_argument (N, &argc, argv); + if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf"))) { + remove_argument (N, &argc, argv); + if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-stilt"))) { + remove_argument (N, &argc, argv); + stilt = atof (argv[N]); + remove_argument (N, &argc, argv); + } else { goto usage; } + + // In-Out parameters: + if ((N = get_argument (argc, argv, "-object"))) { + remove_argument (N, &argc, argv); + if ((obj = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-sky"))) { + remove_argument (N, &argc, argv); + if ((sky = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-backgnd"))) { + remove_argument (N, &argc, argv); + if ((bck = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if (argc != 3) goto usage; + + // buffer is the full image, Xref is reference coordinate of the profile in the buffer + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + + // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) + if (profile->Nelements % 2 == 0) { + gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); + return FALSE; + } + if (psf->Nelements % 2 == 0) { + gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); + return FALSE; + } + + // observed buffer size + int Nx = buffer[0].matrix.Naxis[0]; + int Ny = buffer[0].matrix.Naxis[1]; + float *bufVal = (float *) buffer->matrix.buffer; + + // obj, sky, bck must be consistent with data + if (Ny != obj->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (object)\n"); + return FALSE; + } + if (Ny != sky->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (sky)\n"); + return FALSE; + } + if (Ny != bck->Nelements) { + gprint (GP_ERR, "inconsistent wavelength scales (backgnd)\n"); + return FALSE; + } + + // for the cross-dispersion reference pixle, use the user value or set to Nx/2 if < 0 + deimos_set_cross_ref (atoi(argv[2]), Nx); + deimos_make_kernel (stilt, Nx); + + // the functions below accept an opihi_flt array for object, sky, background + opihi_flt *objVal = obj->elements.Flt; + opihi_flt *skyVal = sky->elements.Flt; + opihi_flt *bckVal = bck->elements.Flt; + + int NRESULT = Niter + Ntry*Niter + 1; + ALLOCATE_PTR (result, DeimosResult, NRESULT); + + Vector **objOut = NULL; + Vector **skyOut = NULL; + Vector **bckOut = NULL; + Vector *objMin = NULL; + Vector *skyMin = NULL; + Vector *bckMin = NULL; + + // Save all result vectors for each of obj, sky, bck: + if (SAVE_ALL_VECTORS) { + ALLOCATE (objOut, Vector *, NRESULT); + ALLOCATE (skyOut, Vector *, NRESULT); + ALLOCATE (bckOut, Vector *, NRESULT); + for (int i = 0; i < NRESULT; i++) { + char name[64]; + snprintf (name, 64, "obj_%04d", i); if ((objOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objOut[i], OPIHI_FLT, Ny); + snprintf (name, 64, "sky_%04d", i); if ((skyOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyOut[i], OPIHI_FLT, Ny); + snprintf (name, 64, "bck_%04d", i); if ((bckOut[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckOut[i], OPIHI_FLT, Ny); + for (int iy = 0; iy < Ny; iy++) { + objOut[i]->elements.Flt[iy] = 0.0; + skyOut[i]->elements.Flt[iy] = 0.0; + bckOut[i]->elements.Flt[iy] = 0.0; + } + } + } + if (SAVE_MIN_VECTORS) { + if ((objMin = SelectVector ("objMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (objMin, OPIHI_FLT, Ny); + if ((skyMin = SelectVector ("skyMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (skyMin, OPIHI_FLT, Ny); + if ((bckMin = SelectVector ("bckMin", ANYVECTOR, TRUE)) == NULL) { return (FALSE); } ResetVector (bckMin, OPIHI_FLT, Ny); + for (int iy = 0; iy < Ny; iy++) { + objMin->elements.Flt[iy] = 0.0; + skyMin->elements.Flt[iy] = 0.0; + bckMin->elements.Flt[iy] = 0.0; + } + } + Vector *chisqVect; + if ((chisqVect = SelectVector ("chisqOut", ANYVECTOR, TRUE)) == NULL) { return FALSE; } + ResetVector (chisqVect, OPIHI_FLT, NRESULT); + for (int i = 0; i < NRESULT; i++) { chisqVect->elements.Flt[i] = 0.0; } + + // XXX temp hack: save chisq subset vectors for each row pass + int Npass = Ny / Nrow; + ALLOCATE_PTR (chiPass, Vector *, Npass); + for (int pass = 0; pass < Npass; pass++) { + char name[64]; + snprintf (name, 64, "chiPass_%03d", pass); + if ((chiPass[pass] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) { return FALSE; } ResetVector (chiPass[pass], OPIHI_FLT, NRESULT); + } + + float noiseVar = SQ(noise); + ALLOCATE_PTR (objNoise, float, Nrow); + ALLOCATE_PTR (skyNoise, float, Nrow); + ALLOCATE_PTR (bckNoise, float, Nrow); + + // work on segments of Nrow at a time + // XXX worry about last segment (< Nrow rows) + int pass = 0; + for (int row = 0; row < Ny - Nrow; row += Nrow, pass++) { + + int Nresult = 0; + + // obj, sky, bck are initial guesses. Also use these values to define a range for the initial guesses. + + // If I just use the value in a given bin to define the range, random noise for the + // faint end could have bad results (range ~ 0.0). For that matter, if I am in a + // region with large dynamic range (e.g., on an emission or absorption line), then we + // likely have large errors in how the guess follows the truth. + + /** old concept: + float objRange = 0, skyRange = 0, bckRange = 0; + for (int iy = 0; iy < Nrow; iy++) { + objRange = MAX(objVal[iy + row], objRange); + skyRange = MAX(skyVal[iy + row], skyRange); + bckRange = MAX(bckVal[iy + row], bckRange); + } + **/ + + // new concept: start with 2-sigma range, then ramp down below + for (int iy = 0; iy < Nrow; iy++) { + objNoise[iy] = nsigma*sqrt(hypot(fabs(objVal[iy + row]) / gain, noiseVar)); + skyNoise[iy] = nsigma*sqrt(hypot(fabs(skyVal[iy + row]) / gain, noiseVar)); + bckNoise[iy] = nsigma*sqrt(hypot(fabs(bckVal[iy + row]) / gain, noiseVar)); + } + + // first test is the input guess + if (1) { + opihi_flt *objtest = deimos_make_test (objVal, NULL, row, Nrow, TRUE); + opihi_flt *skytest = deimos_make_test (skyVal, NULL, row, Nrow, TRUE); + opihi_flt *bcktest = deimos_make_test (bckVal, NULL, row, Nrow, TRUE); + + // generate the model based on the test values + // objtest, skytest, bcktest are segments of the full parameter vectors + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + // XXX need to add in the per-pixel error or variance + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + fprintf (stderr, "guess: chisq: %f, Npts: %d\n", chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; + } + + // save the results + result[Nresult].chisq = chisq / (float) Npts; + result[Nresult].obj = objtest; + result[Nresult].sky = skytest; + result[Nresult].bck = bcktest; + Nresult ++; + } + + // run Niter iterations + for (int iter = 0; iter < Niter; iter ++) { + + // generate a set of test values for obj, sky, backgnd based on the current guess + opihi_flt *objtest = deimos_make_test (objVal, objNoise, row, Nrow, TRUE); + opihi_flt *skytest = deimos_make_test (skyVal, skyNoise, row, Nrow, TRUE); + opihi_flt *bcktest = deimos_make_test (bckVal, bckNoise, row, Nrow, TRUE); + + for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { + fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", + objVal[itmp + row], skyVal[itmp + row], bckVal[itmp + row], + objtest[itmp], skytest[itmp], bcktest[itmp]); + } + + // generate the model based on the test values + // objtest, skytest, bcktest are segments of the full parameter vectors + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + // XXX need to add in the per-pixel error or variance + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + // fprintf (stderr, "iter: %d, chisq: %f, Npts: %d\n", iter, chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult] = chisq / (float) Npts; + } + + // save the results + result[Nresult].chisq = chisq / (float) Npts; + result[Nresult].obj = objtest; + result[Nresult].sky = skytest; + result[Nresult].bck = bcktest; + Nresult ++; + } + + for (int try = 0; try < Ntry; try ++) { + // generate an index of results sorted by chisq + int *IDX = deimos_sort_result (result, Nresult); + + int Nnewset = 0; + + // XXX not sure this scaling factor concept is good + for (int iy = 0; iy < Nrow; iy++) { + objNoise[iy] = scale*objNoise[iy]; + skyNoise[iy] = scale*skyNoise[iy]; + bckNoise[iy] = scale*bckNoise[iy]; + } + + // run Niter iterations, selecting guess values from the result set + for (int iter = 0; iter < Niter; iter ++) { + int bin = deimos_get_random_sample (Nresult); + int entry = IDX[bin]; + // fprintf (stderr, "try: %d : %d, %f\n", bin, entry, result[entry].chisq); + + // generate a set of test values for obj, sky, backgnd based on the current guess + opihi_flt *objtest = deimos_make_test (result[entry].obj, objNoise, row, Nrow, FALSE); + opihi_flt *skytest = deimos_make_test (result[entry].sky, skyNoise, row, Nrow, FALSE); + opihi_flt *bcktest = deimos_make_test (result[entry].bck, bckNoise, row, Nrow, FALSE); + + for (int itmp = 0; FALSE && (itmp < Nrow); itmp++) { + fprintf (stderr, "val: %f %f %f, tst: %f %f %f\n", + result[entry].obj[itmp], result[entry].sky[itmp], result[entry].bck[itmp], + objtest[itmp], skytest[itmp], bcktest[itmp]); + } + + // generate the model based on the test values + float *model = deimos_make_model (objtest, skytest, bcktest, psf, profile, trace, NULL, Nx, Nrow, row); + + // calculate the current chisq + int Npts = 0; + float chisq = deimos_get_chisq (bufVal, model, Nx, Nrow, row, &Npts); + // fprintf (stderr, "try: %d, iter: %d, chisq: %f, Npts: %d\n", try, iter, chisq, Npts); + free (model); + + if (pass < Npass) { + chiPass[pass]->elements.Flt[Nresult + Nnewset] = chisq / (float) Npts; + } + + // save the results + result[Nresult + Nnewset].chisq = chisq / (float) Npts; + result[Nresult + Nnewset].obj = objtest; + result[Nresult + Nnewset].sky = skytest; + result[Nresult + Nnewset].bck = bcktest; + Nnewset ++; + } + Nresult += Nnewset; + } + + int *IDX = deimos_sort_result (result, Nresult); + + // ENDING: save the best result and info about the allowed chisq region + // int *index = sort_result (result, Nresult); + // copy result[index[0]].obj,sky,bck to obj,sky,bck + for (int i = 0; i < Nresult; i++) { + for (int iy = 0; iy < Nrow; iy++) { + if (SAVE_ALL_VECTORS) { + objOut[i]->elements.Flt[iy + row] = result[IDX[i]].obj[iy]; + skyOut[i]->elements.Flt[iy + row] = result[IDX[i]].sky[iy]; + bckOut[i]->elements.Flt[iy + row] = result[IDX[i]].bck[iy]; + } + } + chisqVect->elements.Flt[i] += result[IDX[i]].chisq; + } + if (SAVE_MIN_VECTORS) { + for (int iy = 0; iy < Nrow; iy++) { + objMin->elements.Flt[iy + row] = result[IDX[0]].obj[iy]; + skyMin->elements.Flt[iy + row] = result[IDX[0]].sky[iy]; + bckMin->elements.Flt[iy + row] = result[IDX[0]].bck[iy]; + } + } + } + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos fitobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + return FALSE; +} + +/****************** fitobj Support Functions *******************/ + +static opihi_flt *deimos_make_test (opihi_flt *guess, float *sigma, int row, int Nrow, int fullInput) { + + // we have a vector, guess, with some length >= row + Nrow. + // generate a set of random values in the vicinty of guess[row] to guess[row+Nrow] + // return vector is of length Nrow + + // if fullInput is TRUE, the guess vector runs from 0 to Ny (full wavelength range) + // if fullInput is FALSE, the guess vector runs from row to row + Nrow (subset range) + // sigma is always only Nrow long + + ALLOCATE_PTR (value, opihi_flt, Nrow); + + int start_pix = fullInput ? row : 0; + int stop_pix = fullInput ? row + Nrow : Nrow; + + int iv = 0; + for (int i = start_pix; i < stop_pix; i++, iv++) { + if (!sigma) { + value[iv] = guess[i]; + continue; + } + if (USE_GAUSS_DEV) { + value[iv] = ohana_gaussdev_rnd(guess[i], sigma[iv]); + continue; + } + + // uniform distribution: + value[iv] = sigma[iv]*(drand48() - 0.5) + guess[i]; + } + return value; +} + + +static float deimos_get_chisq (float *buffer, float *model, int Nx, int Ny, int row, int *Npts) { + + int npts = 0; + float chisq = 0; + + for (int iy = 0; iy < Ny; iy++) { + + for (int ix = 0; ix < Nx; ix++) { + + int pix_buffer = ix + (iy + row)*Nx; + int pix_model = ix + iy*Nx; + + if (!isfinite(buffer[pix_buffer])) continue; + if (!isfinite(model[pix_model])) continue; + chisq += SQ(buffer[pix_buffer] - model[pix_model]); + npts ++; + } + } + *Npts = npts; + return chisq; +} + +int *deimos_sort_result (DeimosResult *result, int Nresult) { + + ALLOCATE_PTR (IDX, int, Nresult); + ALLOCATE_PTR (chisq, float, Nresult); + + for (int i = 0; i < Nresult; i++) { + IDX[i] = i; + chisq[i] = result[i].chisq; + } + + sort_float_index (chisq, IDX, Nresult); + free (chisq); + + return IDX; +} + +static float DEIMOS_A = 2.0; // value for random sample + +int deimos_get_random_sample (int Nresult) { + + // choose a bin from the range 0 - Nresult, with front-loaded weighting. + + // bin = Nresult * (exp(q) - exp(-A)) / (exp(A) - exp(-A)) + // q = 2A(x - 1/2) : random number between -A and +A + // x : random number between 0 and 1 + + // f(q;A) = (exp(q) - exp(-A)) / (exp(A) - exp(-A)) + // f(q;A) ranges from 0 to +1 as x ranges from 0 to +1 + + float x = drand48(); + float q = 2*DEIMOS_A*(x - 0.5); + float f = (exp(q) - exp(-DEIMOS_A)) / (exp(DEIMOS_A) - exp(-DEIMOS_A)); + // XXX optimization : precalculate and save the elements of this function + + int bin = floor(Nresult * f); + + return bin; +} +# endif Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitprofile.c (revision 41170) @@ -0,0 +1,299 @@ +# include "data.h" + +typedef opihi_flt FitFunc (opihi_flt, opihi_flt *, int, opihi_flt *); +typedef enum {MODE_UPPER, MODE_LOWER} SigmoidMode; + +void sigmoid_fit (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC); +void sigmoid_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int NptsAll, opihi_flt *par, int Npar, SigmoidMode mode, double limit); + +opihi_flt fsigmoidDeimos (opihi_flt, opihi_flt *, int, opihi_flt *); +opihi_flt rsigmoidDeimos (opihi_flt, opihi_flt *, int, opihi_flt *); + +double deimos_weight_cauchy (double x); + +// use these globals to carry into sigmoid_fit +static int VERBOSE = FALSE; +static int QUIET = FALSE; +static int MaxIterations = 10; + +int deimos_fitprofile (int argc, char **argv) { + + // fit simple slit window profile model to vector + // I am fitting a sigmoid at both edges + + // input parameters: + // * xprofile : extracted slit window coordinates (vector) + // * xprofile : extracted slit window profile (vector) + // * Xs, Xe : scale coordinates of window boundaries + + // output parameters: + // * sigmoid_left_center, etc : coordinates of window boundaries + + int N; + Vector *xprofile = NULL; + Vector *fprofile = NULL; + Vector *wprofile = NULL; + + QUIET = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + QUIET = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + MaxIterations = 10; + if ((N = get_argument (argc, argv, "-max-iterations"))) { + remove_argument (N, &argc, argv); + MaxIterations = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 6) goto usage; + if ((xprofile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((fprofile = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((wprofile = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if (xprofile->Nelements != fprofile->Nelements) goto usage; + if (xprofile->Nelements != wprofile->Nelements) goto usage; + + // supplied guess for boundaries + double Xs = atof(argv[4]); + double Xe = atof(argv[5]); + + // output vectors of fits + Vector *Lprofile = SelectVector ("sigmoid_left", ANYVECTOR, TRUE); + if (Lprofile == NULL) return (FALSE); + MatchVector (Lprofile, xprofile, OPIHI_FLT); + Vector *Rprofile = SelectVector ("sigmoid_right", ANYVECTOR, TRUE); + if (Rprofile == NULL) return (FALSE); + MatchVector (Rprofile, xprofile, OPIHI_FLT); + + // first, use supplied Xs & Xe guess to make a guess for BckL, Sky, BckR + + float Sky_V = 0.0, BckL_V = 0.0, BckR_V = 0.0; + float Sky_N = 0.0, BckL_N = 0.0, BckR_N = 0.0; + for (int i = 0; i < xprofile->Nelements; i++) { + if (!isfinite(wprofile->elements.Flt[i])) continue; + if (!isfinite(fprofile->elements.Flt[i])) continue; + float wt = 1 / SQ(wprofile->elements.Flt[i]); + if (xprofile->elements.Flt[i] < Xs) { + BckL_V += fprofile->elements.Flt[i]*wt; + BckL_N += wt; + continue; + } + if (xprofile->elements.Flt[i] > Xe) { + BckR_V += fprofile->elements.Flt[i]*wt; + BckR_N += wt; + continue; + } + Sky_V += fprofile->elements.Flt[i]*wt; + Sky_N += wt; + } + float Sky = Sky_V / Sky_N; + float BckL = BckL_V / BckL_N; + float BckR = BckR_V / BckR_N; + + // temporary storage vectors + opihi_flt par[4]; + + float Xmid = 0.5*(Xs + Xe); + + int Npar = 4; + par[0] = Xs; + par[1] = 1; + par[2] = Sky; + par[3] = BckL; + sigmoid_fit_irls (xprofile->elements.Flt, fprofile->elements.Flt, wprofile->elements.Flt, Lprofile->elements.Flt, xprofile->Nelements, par, Npar, MODE_LOWER, Xmid); + set_variable ("sigmoid_left_center", par[0]); + set_variable ("sigmoid_left_sigma", par[1]); + set_variable ("sigmoid_left_top", par[2]); + set_variable ("sigmoid_left_bottom", par[3]); + + par[0] = Xe; + par[1] = 1; + par[2] = Sky; + par[3] = BckR; + sigmoid_fit_irls (xprofile->elements.Flt, fprofile->elements.Flt, wprofile->elements.Flt, Rprofile->elements.Flt, xprofile->Nelements, par, Npar, MODE_UPPER, Xmid); + set_variable ("sigmoid_right_center", par[0]); + set_variable ("sigmoid_right_sigma", par[1]); + set_variable ("sigmoid_right_top", par[2]); + set_variable ("sigmoid_right_bottom", par[3]); + + return TRUE; + +usage: + gprint (GP_ERR, "USAGE: deimos fitprofile xprof fprof wprof Xs Xe\n"); + return FALSE; +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt rsigmoidDeimos (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + if (dpar) { + dpar[0] = par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + } + + return (f); +} + +/* pars: x_o, sigma, top, bottom */ +opihi_flt fsigmoidDeimos (opihi_flt x, opihi_flt *par, int Npar, opihi_flt *dpar) { + OHANA_UNUSED_PARAM(Npar); + + double z = (x - par[0])/par[1]; + double r = exp (-z); + double s = 1 + r; + double q = 1 / s; + double f = par[2]*q + par[3]; + + // df/dp0 = df/dq * dq/dr * dr/dz * dz/dp0 + // df/dq = par[2] + // dq/ds = -s^-2 = -q/s + // ds/dr = 1 + // dr/dz = -r + // dz/dp0 = -1 / par[1] + // dz/dp1 = -z / par[1] + + // df/dp0 = -par[2]*(-q/s)*r*(-1/par[1]) = par[2]*q*r/(s*par[1]) + // df/dp1 = -par[2]*(-q/s)*r*(-z/par[1]) = par[2]*q*r*z/(s*par[1]) + + if (dpar) { + dpar[0] = -par[2]*q*r/(s*par[1]); + dpar[1] = dpar[0]*z; + dpar[2] = q; + dpar[3] = 1; + } + + return (f); +} + +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 + +// wprofile is the true poisson error +void sigmoid_fit_irls (opihi_flt *xprofile, opihi_flt *fprofile, opihi_flt *wprofile, opihi_flt *oprofile, int NptsAll, opihi_flt *par, int Npar, SigmoidMode mode, double limit) { + + FitFunc *FUNC = (mode == MODE_LOWER) ? fsigmoidDeimos : rsigmoidDeimos; + + ALLOCATE_PTR (par_old, opihi_flt, Npar); + + // allocate these as global static in this file? + ALLOCATE_PTR (xtemp, opihi_flt, NptsAll); + ALLOCATE_PTR (ytemp, opihi_flt, NptsAll); + ALLOCATE_PTR (wtemp, opihi_flt, NptsAll); + ALLOCATE_PTR (otemp, opihi_flt, NptsAll); + + // extract the points on the right side and fit + int Npts = 0; + for (int i = 0; i < NptsAll; i++) { + if (!isfinite(wprofile[i])) continue; + if (!isfinite(fprofile[i])) continue; + int keep = (mode == MODE_UPPER) ^ (xprofile[i] < limit); + if (keep) { + xtemp[Npts] = xprofile[i]; + ytemp[Npts] = fprofile[i]; + wtemp[Npts] = (wprofile[i] == 0.0) ? 1.0 : wprofile[i]; // crude + Npts ++; + } + } + + // define initial weights as 1.0 + ALLOCATE_PTR (wt, opihi_flt, Npts); + for (int i = 0; i < Npts; i++) { + wt[i] = 1.0 / SQ(wtemp[i]); + } + + // initial fit with ordinary least-squares + // otemp is the fitted value of the function at xtemp + sigmoid_fit (xtemp, ytemp, wt, otemp, Npts, par, Npar, FUNC); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < MaxIterations); iterations++) { + + for (int i = 0; i < Npts; i++) { + wt[i] = deimos_weight_cauchy ((ytemp[i] - otemp[i]) / wtemp[i]) / SQ(wtemp[i]); + } + + sigmoid_fit (xtemp, ytemp, wt, otemp, Npts, par, Npar, FUNC); + + // save this solution + for (int i = 0; i < Npar; i++) { + par_old[i] = par[i]; + } + + converged = TRUE; + for (int i = 0; i < Npts; i++) { + if ((fabs(par[i] - par_old[i]) > FIT_TOLERANCE * fabs(par[i])) && + (fabs(par[i] - par_old[i]) > FLT_TOLERANCE)) + converged = FALSE; + } + } + + for (int i = 0; i < NptsAll; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + + FREE (xtemp); + FREE (ytemp); + FREE (wtemp); + FREE (otemp); + + FREE (par_old); + FREE (wt); +} + +void sigmoid_fit (opihi_flt *xprofile, opihi_flt *yprofile, opihi_flt *wprofile, opihi_flt *oprofile, int Npts, opihi_flt *par, int Npar, FitFunc *FUNC) { + + double ochisq = mrqinit (xprofile, yprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + double dchisq = ochisq + 2*Npts; + + int Niter; + for (Niter = 0; (Niter < 20) && ((dchisq > 0.1*(Npts - Npar)) || (dchisq <= 0.0)); Niter++) { + double chisq = mrqmin (xprofile, yprofile, wprofile, Npts, par, Npar, FUNC, VERBOSE); + dchisq = ochisq - chisq; + ochisq = chisq; + if (VERBOSE) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (VERBOSE) gprint (GP_ERR, "%d iterations\n", Niter); + + if (!QUIET) gprint (GP_LOG, "sigmoid @ %f (%f) : top %f bottom %f\n", par[0], par[1], par[2], par[3]); + + for (int i = 0; i < Npts; i++) { + oprofile[i] = FUNC (xprofile[i], par, Npar, NULL); + } + + mrqfree (Npar); +} + +double deimos_weight_cauchy (double x) { + double r = x / 2.385; + return (1.0 / (1.0 + SQ(r))); +} + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_fitslit.c (revision 41170) @@ -1,9 +1,11 @@ # include "data.h" +# define IRLS_TOLERANCE 1e-4 + +float weight_cauchy_square_flt (float x2); +static void fitflux (float *Fwind, float *Fprof, float *weight, int Nx, double *flux, double *sky); int deimos_fitslit (int argc, char **argv) { - // fitslit (window) (slit) (flux) (sky) - - // int N; + // fitslit (window) (vari) (slit) (flux) (sky) Vector *flux = NULL; @@ -11,9 +13,25 @@ Buffer *wind = NULL; + Buffer *vari = NULL; Buffer *slit = NULL; - if (argc != 5) { - gprint (GP_ERR, "USAGE: deimos fitslit (window) (slit) (flux) (sky)\n"); - gprint (GP_ERR, " inputs: window (observed 2D flux), slit (model 2D flux)\n"); + int N; + int Niter = 10; + if ((N = get_argument (argc, argv, "-irls-iter"))) { + remove_argument (N, &argc, argv); + Niter = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + Buffer *model = NULL; + if ((N = get_argument (argc, argv, "-model"))) { + remove_argument (N, &argc, argv); + if ((model = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + + if (argc != 6) { + gprint (GP_ERR, "USAGE: deimos fitslit (window) (variance) (slit) (flux) (sky)\n"); + gprint (GP_ERR, " inputs: window (observed 2D flux), variance (on 2D flux), slit (model 2D flux)\n"); gprint (GP_ERR, " outputs: flux (best-fit 1D flux), sky (best-fit 1D background)\n"); return FALSE; @@ -22,9 +40,8 @@ // XXX I probably should rename FindSpline as SelectSpline and give it the same behavior if ((wind = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); - if ((slit = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); - if ((flux = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); - if ((sky = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); - - // XXX confirm slit and wind have same dimensions + if ((vari = SelectBuffer (argv[2], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((slit = SelectBuffer (argv[3], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((flux = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((sky = SelectVector (argv[5], ANYVECTOR, TRUE)) == NULL) return (FALSE); // define the output window @@ -32,42 +49,117 @@ int Ny = wind[0].matrix.Naxis[1]; + // confirm wind, vari, slit have same dimensions if (Nx != slit[0].matrix.Naxis[0]) { gprint (GP_ERR, "size mismatch in fitslit for window and slit\n"); return FALSE; } if (Ny != slit[0].matrix.Naxis[1]) { gprint (GP_ERR, "size mismatch in fitslit for window and slit\n"); return FALSE; } + if (Nx != vari[0].matrix.Naxis[0]) { gprint (GP_ERR, "size mismatch in fitslit for window and variance\n"); return FALSE; } + if (Ny != vari[0].matrix.Naxis[1]) { gprint (GP_ERR, "size mismatch in fitslit for window and variance\n"); return FALSE; } ResetVector (flux, OPIHI_FLT, Ny); ResetVector (sky, OPIHI_FLT, Ny); + if (model) { + if (!CreateBuffer (model, Nx, Ny, -32, 1.0, 0.0)) { gprint (GP_ERR, "error generating output model buffer\n"); return FALSE; } + } + float *Fwind = (float *) wind[0].matrix.buffer; + float *Fvari = (float *) vari[0].matrix.buffer; float *Fprof = (float *) slit[0].matrix.buffer; + float *FmodOut = model ? (float *) model[0].matrix.buffer : NULL; + + ALLOCATE_PTR (weight, float, Nx); + ALLOCATE_PTR (rawwgt, float, Nx); // loop over the rows for (int iy = 0; iy < Ny; iy++) { - // calculate elements of the chi-square for this row: + double F = 0.0, S = 0.0; - double R = 0.0, P = 0.0, P2 = 0.0, f = 0.0, fp = 0.0; - double wt = 1.0; - + // set weight based on the variance for (int ix = 0; ix < Nx; ix++) { - - float fxy = Fwind[ix + iy*Nx]; - float pxy = Fprof[ix + iy*Nx]; - - R += wt; - P += pxy*wt; - P2 += pxy*pxy*wt; - f += fxy*wt; - fp += fxy*pxy*wt; + float vari = Fvari[ix + iy*Nx]; + weight[ix] = !isfinite(vari) || (fabs(vari) < 1e-6) ? 1.0 : 1.0 / vari; + rawwgt[ix] = weight[ix]; } - double det = R*P2 - P*P; - double S = (P2*f - P*fp) / det; - double F = (R*fp - P*f) / det; + // calculate elements of the chi-square for this row: + fitflux (&Fwind[iy*Nx], &Fprof[iy*Nx], weight, Nx, &F, &S); + + // do an IRLS loop here to reject outliers + int converged = FALSE; + for (int iter = 0; (iter < Niter) && !converged; iter++) { + + double Flast = F; + double Slast = S; + + // calculate weight modification based on distances (squared). + // use modifier to calculate new weighted mean + for (int ix = 0; ix < Nx; ix++) { + + // F & S are model parameters; compare the model and observed values for each pixel + float Fmodel = S + F*Fprof[ix + iy*Nx]; + float dV = (Fwind[ix + iy*Nx] - Fmodel); + float d2 = SQ(dV) * rawwgt[ix]; + + float Mod = weight_cauchy_square_flt (d2); + weight[ix] = Mod * rawwgt[ix]; + } + fitflux (&Fwind[iy*Nx], &Fprof[iy*Nx], weight, Nx, &F, &S); + + float dF = fabs(F - Flast); + float dS = fabs(S - Slast); + + if ((dF < F * IRLS_TOLERANCE) && (dS < S * IRLS_TOLERANCE)) converged = TRUE; + } + + if (FmodOut) { + for (int ix = 0; ix < Nx; ix++) { + // F & S are model parameters; compare the model and observed values for each pixel + float Fmodel = S + F*Fprof[ix + iy*Nx]; + float dV = (Fwind[ix + iy*Nx] - Fmodel); + float d2 = SQ(dV) * rawwgt[ix]; + float Mod = weight_cauchy_square_flt (d2); + if (Mod < 0.1) { + FmodOut[ix + iy*Nx] = NAN; + } else { + FmodOut[ix + iy*Nx] = Fmodel; + } + } + } flux->elements.Flt[iy] = F; sky->elements.Flt[iy] = S; } - return TRUE; } +// weight is 1 / variance +// Fwind, Fprof are pointers to the start of a single row +// Fwind is the observed flux in the window +// Fprof is the slit profile +void fitflux (float *Fwind, float *Fprof, float *weight, int Nx, double *flux, double *sky) { + + // calculate elements of the chi-square fit for this row: + + double R = 0.0, P = 0.0, P2 = 0.0, f = 0.0, fp = 0.0; + + for (int ix = 0; ix < Nx; ix++) { + + double fxy = Fwind[ix]; + double pxy = Fprof[ix]; + + // skip NAN / Inf pixels in either window or slit + if (!isfinite(fxy)) continue; + if (!isfinite(pxy)) continue; + + double wt = (weight == NULL) ? 1.0 : weight[ix]; + R += wt; + P += pxy*wt; + P2 += pxy*pxy*wt; + f += fxy*wt; + fp += fxy*pxy*wt; + } + + double det = R*P2 - P*P; + *sky = (P2*f - P*fp) / det; + *flux = (R*fp - P*f) / det; +} Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.c (revision 41170) @@ -3,29 +3,17 @@ // temporary storage used in local functions: -float *tmpv = NULL; -float *tmpx = NULL; - -// pass in a pointer to the row (&buffV[iy*Nx]) -float get_segment_median (float *buffV, float Xs, float Xe, float *Xref) { - - // get the pixels in the left portion below profile: - // XXX: worry about the effect of a fractional offset - for (int ix = Xs + 0.5; ix < Xe + 0.5; ix++) { - int i = ix - Xs; - tmpv[i] = buffV[ix]; - tmpx[i] = ix; - } - int Nv = Xe - Xs; - - // generate a statistic from these pixels: mean, median, etc? - fsort (tmpv, Nv); - int Nv2 = Nv/2; // Nv == 1, Nv2 = 0, Nv == 2, Nv2 = 1, Nv == 3, Nv2 = 1, Nv == 4, Nv == 2 - float V = (Nv % 2) ? tmpv[Nv2] : 0.5*(tmpv[Nv2-1] + tmpv[Nv2]); - - // calculate average coordinate (not flux-weighted...) - // XXX : if we allow (and exclude) NAN-valued pixels, we can calculate this more explicitly - *Xref = 0.5*(Xe + Xs); - - return V; +static float *tmpv = NULL; +static float *tmpx = NULL; + +float get_median_flt (float *val, int Nval) { + + if (Nval <= 0) return NAN; + + int Nval2 = Nval / 2; + + fsort (val, Nval); + float value = (Nval % 2) ? val[Nval2] : 0.5*(val[Nval2-1] + val[Nval2]); + + return value; } @@ -37,7 +25,12 @@ // not used : LSF -- not needed (measurement per disp pixel) + // This version uses the median of the pixels in the image which are + // in regions where the slit profile is < 15% (as opposed to the + // getobj version which attempts to fit both background and sky regions + // input parameters: // * buffer : 2D image of slit (full frame or cutout?) - // * trace : spline fit of slit central x pos vs y-coord + // * slit_trace : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of psf central x pos vs y-coord // * profile : slit window profile (vector) // * PSF : point-spread function vector (flux normalized, x-dir) @@ -51,5 +44,7 @@ int N; - Spline *trace = NULL; + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; Vector *profile = NULL; @@ -64,14 +59,28 @@ float stilt = 0.0; // angle of the slit - int SaveSegments = FALSE; - if ((N = get_argument (argc, argv, "-save-segments"))) { - remove_argument (N, &argc, argv); - SaveSegments = TRUE; - } + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } // Input parameters: - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } else { goto usage; } @@ -137,47 +146,4 @@ ResetVector (backgnd, OPIHI_FLT, Ny); - // I need to generate a sky-left and sky-right vector in a region on the slit portion of - // the profile, but off the object. Then each of these vectors can be shifted up and down - // based on stilt. Use these sky vectors to predict the sky under the PSF - - // *** scan the profile to determine the background and sky cut points *** - - // values below in the profile coordinate system need to be shifted by (Xref - Nprofile/2) - // to match the image coordinate system - - int B_s_l = 0, B_e_l = -1; // left background window - int B_s_r = -1, B_e_r = Nprofile - 1; // right background window - int S_left = -1, S_right = -1; // top portion of profile - for (int i = 0; i < Nprofile; i++) { - if ((B_e_l < 0) && (profile->elements.Flt[i] > 0.10)) B_e_l = i - 1; - if ((B_e_l >= 0) && (S_left < 0) && (profile->elements.Flt[i] > 0.95)) S_left = i; - if ((S_left >= 0) && (S_right < 0) && (profile->elements.Flt[i] < 0.95)) S_right = i - 1; - if ((S_right >= 0) && (profile->elements.Flt[i] < 0.10)) { B_s_r = i; break; } - } - if (B_e_l == B_s_l) B_e_l ++; - if (B_e_r == B_s_r) B_s_r --; - - // S_left to S_right defines are window at the top, choose 0.05 - 0.15, 0.85 - 0.95 - int N_top = S_right - S_left + 1; - - // if N_top < 20, these jumps might be too small - // if N_top < 20, these jumps might be too small - int S_s_l = S_left + 0.05*N_top; - int S_e_l = S_left + 0.15*N_top; - if (S_e_l == S_s_l) S_e_l ++; - int S_s_r = S_left + 0.85*N_top; - int S_e_r = S_left + 0.95*N_top; - if (S_e_r == S_s_r) S_s_r --; - - gprint (GP_ERR, "windows: %.1f - %.1f, %.1f - %.1f, %.1f - %.1f, %.1f - %.1f\n", - B_s_l + Xref - Xprofile, B_e_l + Xref - Xprofile, - S_s_l + Xref - Xprofile, S_e_l + Xref - Xprofile, - S_s_r + Xref - Xprofile, S_e_r + Xref - Xprofile, - B_s_r + Xref - Xprofile, B_e_r + Xref - Xprofile); - - // OPEN QUESTIONS: - // NOTE: if the range of pixels in the windows is too large, then the tilt will wash out the signal. - // this pass is a guess, so perhaps that does not matter, but there is clearly a trade-off here. - // generate two buffers to store the flux and coordinate for the background, sky, object pixels: ALLOCATE (tmpv, float, Nx); @@ -186,75 +152,9 @@ float *buffV = (float *) buffer[0].matrix.buffer; - // use these to store the flux vectors - ALLOCATE_PTR (bckF_lf, float, Ny); - ALLOCATE_PTR (bckF_rt, float, Ny); - ALLOCATE_PTR (skyF_lf, float, Ny); - ALLOCATE_PTR (skyF_rt, float, Ny); - - // use these to store the x-coord vectors - ALLOCATE_PTR (bckX_lf, float, Ny); - ALLOCATE_PTR (bckX_rt, float, Ny); - ALLOCATE_PTR (skyX_lf, float, Ny); - ALLOCATE_PTR (skyX_rt, float, Ny); - float dxMax = 0; // maximum distance between sky regions - Vector *skyLsV = NULL, *skyLeV = NULL, *skyLfV = NULL, *skyRsV = NULL, *skyReV = NULL, *skyRfV = NULL; - Vector *bckLsV = NULL, *bckLeV = NULL, *bckLfV = NULL, *bckRsV = NULL, *bckReV = NULL, *bckRfV = NULL; - if (SaveSegments) { - skyLsV = SelectVector ("skyLs", ANYVECTOR, TRUE); ResetVector (skyLsV, OPIHI_FLT, Ny); - skyLeV = SelectVector ("skyLe", ANYVECTOR, TRUE); ResetVector (skyLeV, OPIHI_FLT, Ny); - skyLfV = SelectVector ("skyLf", ANYVECTOR, TRUE); ResetVector (skyLfV, OPIHI_FLT, Ny); - skyRsV = SelectVector ("skyRs", ANYVECTOR, TRUE); ResetVector (skyRsV, OPIHI_FLT, Ny); - skyReV = SelectVector ("skyRe", ANYVECTOR, TRUE); ResetVector (skyReV, OPIHI_FLT, Ny); - skyRfV = SelectVector ("skyRf", ANYVECTOR, TRUE); ResetVector (skyRfV, OPIHI_FLT, Ny); - bckLsV = SelectVector ("bckLs", ANYVECTOR, TRUE); ResetVector (bckLsV, OPIHI_FLT, Ny); - bckLeV = SelectVector ("bckLe", ANYVECTOR, TRUE); ResetVector (bckLeV, OPIHI_FLT, Ny); - bckLfV = SelectVector ("bckLf", ANYVECTOR, TRUE); ResetVector (bckLfV, OPIHI_FLT, Ny); - bckRsV = SelectVector ("bckRs", ANYVECTOR, TRUE); ResetVector (bckRsV, OPIHI_FLT, Ny); - bckReV = SelectVector ("bckRe", ANYVECTOR, TRUE); ResetVector (bckReV, OPIHI_FLT, Ny); - bckRfV = SelectVector ("bckRf", ANYVECTOR, TRUE); ResetVector (bckRfV, OPIHI_FLT, Ny); - } - - // loop over the y positions - for (int iy = 0; iy < Ny; iy++) { - - // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dXtrace = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); - - // Xref is the reference coordinate of the profile on the image - // Xprofile is the center of the profile. - // dXtrace follows the x-dir variations of the slit profile - float Xtrace = Xref - Xprofile + dXtrace; - - bckF_lf[iy] = get_segment_median(&buffV[iy*Nx], B_s_l + Xtrace, B_e_l + Xtrace, &bckX_lf[iy]); - bckF_rt[iy] = get_segment_median(&buffV[iy*Nx], B_s_r + Xtrace, B_e_r + Xtrace, &bckX_rt[iy]); - skyF_lf[iy] = get_segment_median(&buffV[iy*Nx], S_s_l + Xtrace, S_e_l + Xtrace, &skyX_lf[iy]); - skyF_rt[iy] = get_segment_median(&buffV[iy*Nx], S_s_r + Xtrace, S_e_r + Xtrace, &skyX_rt[iy]); - dxMax = MAX (dxMax, skyX_rt[iy] - skyX_lf[iy]); // this is a constant value unless we ignore some of the pixels - // assert skyX_rt > skyX_lf ? - - if (SaveSegments) { - skyLsV->elements.Flt[iy] = S_s_l + Xtrace; - skyLeV->elements.Flt[iy] = S_e_l + Xtrace; - skyLfV->elements.Flt[iy] = skyF_lf[iy]; - skyRsV->elements.Flt[iy] = S_s_r + Xtrace; - skyReV->elements.Flt[iy] = S_e_r + Xtrace; - skyRfV->elements.Flt[iy] = skyF_rt[iy]; - bckLsV->elements.Flt[iy] = B_s_l + Xtrace; - bckLeV->elements.Flt[iy] = B_e_l + Xtrace; - bckLfV->elements.Flt[iy] = bckF_lf[iy]; - bckRsV->elements.Flt[iy] = B_s_r + Xtrace; - bckReV->elements.Flt[iy] = B_e_r + Xtrace; - bckRfV->elements.Flt[iy] = bckF_rt[iy]; - } - } - // extract sky-subtracted values for the psf fit, using the two sky vectors to predict the sky - float dyMax = dxMax*sin(stilt*RAD_DEG); // dxMax is based on the profile width, calculated above - - // find nominal dy_lf, dy_rt offsets for the sky vectors. - int dy_lf = -dyMax / 2; - int dy_rt = +dyMax / 2; + float sin_stilt = sin(stilt*RAD_DEG); + float dyMax = dxMax*sin_stilt; // dxMax is based on the profile width, calculated above // we are fitting this PSF to the data: @@ -273,32 +173,99 @@ } - - // Ys, Ye based on the dyMax value above (skip the first few lines to avoid running out of bounds on sky) for (int iy = fabs(dyMax); iy < Ny - fabs(dyMax); iy++) { - // X_lf & X_rt are calculated in image x-coords - float S_lf = skyF_lf[iy + dy_lf]; - float S_rt = skyF_rt[iy + dy_rt]; - float X_lf = skyX_lf[iy + dy_lf]; - float X_rt = skyX_rt[iy + dy_rt]; - // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dXtrace = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dXtraceSLIT = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + float dXtracePSF = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); // Xref is the reference coordinate of the profile on the image // Xpsf is the center of the PSF. // dXtrace follows the x-dir variations of the slit profile (or PSF?) - float Xtrace = Xref - Xpsf + dXtrace; - - // include the PSF trace and restrict the fit below to portions where the profile is > 0.95 + + // XtracePSF is the offset between the buffer ix and the psf elements at this location + float XtracePSF = Xref - Xpsf + dXtraceSLIT + dXtracePSF; + + // XtraceSLIT is the offset between the buffer ix and the slit profile elements at this location + float XtraceSLIT = Xref - Xprofile + dXtraceSLIT; + + // measure the background outside the slit + + // select pixels from the image which are in the background regions (profile < 0.15) + int Nbck = 0; + int Nsky = 0; + for (int ix = 0; ix < Nx; ix ++) { + int islit = ix - XtraceSLIT - 0.5; // slit window coordinate + int ipsf = ix - XtracePSF - 0.5; // psf model coordinate + + if (islit < 0) continue; + if (islit >= Nprofile) continue; + + // coordinate of the PSF center: + int dxpsf = ipsf - Xpsf; + int dypsf = dxpsf * sin_stilt; + + int iY = iy + dypsf; + + if (iy == 6000) { + fprintf (stderr, "bck : %d %d %d : %f : %f\n", ix, islit, ipsf, buffV[ix + iY*Nx], profile->elements.Flt[islit]); + } + + // pixels in the off-slit background: + if (profile->elements.Flt[islit] < 0.15) { + tmpv[Nbck] = buffV[ix + iY*Nx]; + if (!isfinite(tmpv[Nbck])) continue; + Nbck ++; + continue; + } + + // the psf window can be smaller than the slit window + if (ipsf < 0) continue; + if (ipsf >= Npsf) continue; + + // pixels in the on-slit sky: + if ((profile->elements.Flt[islit] > 0.95) && (psfV[ipsf] < 0.15)) { + tmpx[Nsky] = buffV[ix + iY*Nx]; + if (!isfinite(tmpx[Nsky])) continue; + Nsky ++; + } + } + backgnd->elements.Flt[iy] = get_median_flt (tmpv, Nbck); + float So = get_median_flt (tmpx, Nsky); + sky->elements.Flt[iy] = So; // now I need to find the limits of the PSF in the buffer at this iy location - for (int ipsf = 0; ipsf < Npsf; ipsf ++) { - - int ix = ipsf + Xtrace + 0.5; // image coordinate (nearest pixel center) - float So = S_lf + (ix - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak - - tmpv[ipsf] = buffV[ix + iy*Nx] - So; - tmpx[ipsf] = ipsf; + int Ntmp = 0; + for (int ix = 0; ix < Nx; ix ++) { + int islit = ix - XtraceSLIT - 0.5; // slit window coordinate + int ipsf = ix - XtracePSF - 0.5; // psf model coordinate + + if (islit < 0) continue; + if (islit >= Nprofile) continue; + + // the psf window can be smaller than the slit window + if (ipsf < 0) continue; + if (ipsf >= Npsf) continue; + + // coordinate of the PSF center: + int dxpsf = ipsf - Xpsf; + int dypsf = dxpsf * sin_stilt; + + int iY = iy + dypsf; + + // filter on the PSF value? + if (profile->elements.Flt[islit] < 0.95) continue; + + if (iy == 6000) { + fprintf (stderr, "%d %d %d : %f %f : %f : %f\n", ix, islit, ipsf, buffV[ix + iY*Nx] - So, psfV[ipsf], (buffV[ix + iY*Nx] - So)*psfV[ipsf], profile->elements.Flt[islit]); + } + + tmpv[Ntmp] = buffV[ix + iY*Nx] - So; + tmpx[Ntmp] = psfV[ipsf]; + + if (!isfinite(tmpv[Ntmp])) continue; + if (!isfinite(tmpx[Ntmp])) continue; + + Ntmp ++; } @@ -306,31 +273,16 @@ // now fit the PSF to tmpv by calculating a normalization float S1 = 0.0, S2 = 0.0; - for (int ipsf = 0; ipsf < Npsf; ipsf++) { + for (int ipsf = 0; ipsf < Ntmp; ipsf++) { if (iy == 6000) { - fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], psfV[ipsf], tmpv[ipsf]*psfV[ipsf]); - } - S1 += tmpv[ipsf]*psfV[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) - S2 += psfV[ipsf]*psfV[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], tmpx[ipsf], tmpv[ipsf]*tmpx[ipsf]); + } + S1 += tmpv[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + S2 += tmpx[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) } - - int Xo = Xref - dXtrace; // center of PSF in image x-coordinates - float So = S_lf + (Xo - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak object->elements.Flt[iy] = S1 / S2; - sky->elements.Flt[iy] = So; - backgnd->elements.Flt[iy] = 0.5*(bckF_lf[iy] + bckF_rt[iy]); } free (tmpv); free (tmpx); - - free (bckF_lf); - free (bckF_rt); - free (skyF_lf); - free (skyF_rt); - - free (bckX_lf); - free (bckX_rt); - free (skyX_lf); - free (skyX_rt); return TRUE; Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_getobj.v0.c (revision 41170) @@ -0,0 +1,383 @@ +# include "data.h" + +// temporary storage used in local functions: + +static float *tmpv = NULL; +static float *tmpx = NULL; + +// pass in a pointer to the row (&buffV[iy*Nx]) +float get_segment_median (float *buffV, float Xs, float Xe, float *Xref) { + + // get the pixels in the left portion below profile: + // XXX: worry about the effect of a fractional offset + for (int ix = Xs + 0.5; ix < Xe + 0.5; ix++) { + int i = ix - Xs; + tmpv[i] = buffV[ix]; + tmpx[i] = ix; + } + int Nv = Xe - Xs; + + // generate a statistic from these pixels: mean, median, etc? + fsort (tmpv, Nv); + int Nv2 = Nv/2; // Nv == 1, Nv2 = 0, Nv == 2, Nv2 = 1, Nv == 3, Nv2 = 1, Nv == 4, Nv == 2 + float V = (Nv % 2) ? tmpv[Nv2] : 0.5*(tmpv[Nv2-1] + tmpv[Nv2]); + + // calculate average coordinate (not flux-weighted...) + // XXX : if we allow (and exclude) NAN-valued pixels, we can calculate this more explicitly + *Xref = 0.5*(Xe + Xs); + + return V; +} + +int deimos_getobj (int argc, char **argv) { + + // determine rough object model parameters based on quick stats + // input : buffer PSF trace profile stilt + // output : object sky background + // not used : LSF -- not needed (measurement per disp pixel) + + // input parameters: + // * buffer : 2D image of slit (full frame or cutout?) + // * slit_trace : spline fit of slit central x pos vs y-coord + // * psf_trace : spline fit of psf central x pos vs y-coord + // * profile : slit window profile (vector) + // * PSF : point-spread function vector (flux normalized, x-dir) + // * stilt : slit tilt response : 2D kernel? + + // output parameters: + // * object : vector of object flux vs y-coord + // * sky : vector of local sky signal vs y-coord + // * background : vector of extra-slit background flux vs y-coord + + int N; + + Spline *slit_trace = NULL; + Spline *psf_trace = NULL; + Vector *profile = NULL; + + Vector *object = NULL; + Vector *sky = NULL; + Vector *backgnd = NULL; + + Vector *psf = NULL; + + Buffer *buffer = NULL; + + float stilt = 0.0; // angle of the slit + + int SaveSegments = FALSE; + if ((N = get_argument (argc, argv, "-save-segments"))) { + remove_argument (N, &argc, argv); + SaveSegments = TRUE; + } + + // Input parameters: + if ((N = get_argument (argc, argv, "-slit-trace"))) { + remove_argument (N, &argc, argv); + if ((slit_trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-profile"))) { + remove_argument (N, &argc, argv); + if ((profile = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-psf"))) { + remove_argument (N, &argc, argv); + if ((psf = SelectVector (argv[N], OLDVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-stilt"))) { + remove_argument (N, &argc, argv); + stilt = atof (argv[N]); + remove_argument (N, &argc, argv); + } else { goto usage; } + + // Output parameters: + if ((N = get_argument (argc, argv, "-object"))) { + remove_argument (N, &argc, argv); + if ((object = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-sky"))) { + remove_argument (N, &argc, argv); + if ((sky = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + if ((N = get_argument (argc, argv, "-backgnd"))) { + remove_argument (N, &argc, argv); + if ((backgnd = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } else { goto usage; } + + if (argc != 3) goto usage; + + // buffer is the full image, Xref is reference coordinate of the profile in the buffer + if ((buffer = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + float Xref = atof(argv[2]); + + // profile and PSF are defined with central reference pixel mapped to Xref (+ trace) + int Nprofile = profile->Nelements; + if (Nprofile % 2 == 0) { + gprint (GP_ERR, "slit profile vector must have an odd number of pixels\n"); + return FALSE; + } + int Xprofile = Nprofile / 2; + + int Npsf = psf->Nelements; + if (Npsf % 2 == 0) { + gprint (GP_ERR, "PSF vector must have an odd number of pixels\n"); + return FALSE; + } + int Xpsf = Npsf / 2; + + // we will generate output vectors (object, sky, background) of length Ny + int Nx = buffer[0].matrix.Naxis[0]; + int Ny = buffer[0].matrix.Naxis[1]; + ResetVector (object, OPIHI_FLT, Ny); + ResetVector (sky, OPIHI_FLT, Ny); + ResetVector (backgnd, OPIHI_FLT, Ny); + + // I need to generate a sky-left and sky-right vector in a region on the slit portion of + // the profile, but off the object. Then each of these vectors can be shifted up and down + // based on stilt. Use these sky vectors to predict the sky under the PSF + + // *** scan the profile to determine the background and sky cut points *** + + // values below in the profile coordinate system need to be shifted by (Xref - Nprofile/2) + // to match the image coordinate system + + int B_s_l = 0, B_e_l = -1; // left background window + int B_s_r = -1, B_e_r = Nprofile - 1; // right background window + int S_left = -1, S_right = -1; // top portion of profile + for (int i = 0; i < Nprofile; i++) { + if ((B_e_l < 0) && (profile->elements.Flt[i] > 0.10)) B_e_l = i - 1; + if ((B_e_l >= 0) && (S_left < 0) && (profile->elements.Flt[i] > 0.95)) S_left = i; + if ((S_left >= 0) && (S_right < 0) && (profile->elements.Flt[i] < 0.95)) S_right = i - 1; + if ((S_right >= 0) && (profile->elements.Flt[i] < 0.10)) { B_s_r = i; break; } + } + if (B_e_l == B_s_l) B_e_l ++; + if (B_e_r == B_s_r) B_s_r --; + + // S_left to S_right defines are window at the top, choose 0.05 - 0.15, 0.85 - 0.95 + int N_top = S_right - S_left + 1; + + // if N_top < 20, these jumps might be too small + // if N_top < 20, these jumps might be too small + int S_s_l = S_left + 0.05*N_top; + int S_e_l = S_left + 0.15*N_top; + if (S_e_l == S_s_l) S_e_l ++; + int S_s_r = S_left + 0.85*N_top; + int S_e_r = S_left + 0.95*N_top; + if (S_e_r == S_s_r) S_s_r --; + + gprint (GP_ERR, "windows: %.1f - %.1f, %.1f - %.1f, %.1f - %.1f, %.1f - %.1f\n", + B_s_l + Xref - Xprofile, B_e_l + Xref - Xprofile, + S_s_l + Xref - Xprofile, S_e_l + Xref - Xprofile, + S_s_r + Xref - Xprofile, S_e_r + Xref - Xprofile, + B_s_r + Xref - Xprofile, B_e_r + Xref - Xprofile); + + // OPEN QUESTIONS: + // NOTE: if the range of pixels in the windows is too large, then the tilt will wash out the signal. + // this pass is a guess, so perhaps that does not matter, but there is clearly a trade-off here. + + // generate two buffers to store the flux and coordinate for the background, sky, object pixels: + ALLOCATE (tmpv, float, Nx); + ALLOCATE (tmpx, float, Nx); + + float *buffV = (float *) buffer[0].matrix.buffer; + + // use these to store the flux vectors + ALLOCATE_PTR (bckF_lf, float, Ny); + ALLOCATE_PTR (bckF_rt, float, Ny); + ALLOCATE_PTR (skyF_lf, float, Ny); + ALLOCATE_PTR (skyF_rt, float, Ny); + + // use these to store the x-coord vectors + ALLOCATE_PTR (bckX_lf, float, Ny); + ALLOCATE_PTR (bckX_rt, float, Ny); + ALLOCATE_PTR (skyX_lf, float, Ny); + ALLOCATE_PTR (skyX_rt, float, Ny); + + float dxMax = 0; // maximum distance between sky regions + + Vector *skyLsV = NULL, *skyLeV = NULL, *skyLfV = NULL, *skyRsV = NULL, *skyReV = NULL, *skyRfV = NULL; + Vector *bckLsV = NULL, *bckLeV = NULL, *bckLfV = NULL, *bckRsV = NULL, *bckReV = NULL, *bckRfV = NULL; + if (SaveSegments) { + skyLsV = SelectVector ("skyLs", ANYVECTOR, TRUE); ResetVector (skyLsV, OPIHI_FLT, Ny); + skyLeV = SelectVector ("skyLe", ANYVECTOR, TRUE); ResetVector (skyLeV, OPIHI_FLT, Ny); + skyLfV = SelectVector ("skyLf", ANYVECTOR, TRUE); ResetVector (skyLfV, OPIHI_FLT, Ny); + skyRsV = SelectVector ("skyRs", ANYVECTOR, TRUE); ResetVector (skyRsV, OPIHI_FLT, Ny); + skyReV = SelectVector ("skyRe", ANYVECTOR, TRUE); ResetVector (skyReV, OPIHI_FLT, Ny); + skyRfV = SelectVector ("skyRf", ANYVECTOR, TRUE); ResetVector (skyRfV, OPIHI_FLT, Ny); + bckLsV = SelectVector ("bckLs", ANYVECTOR, TRUE); ResetVector (bckLsV, OPIHI_FLT, Ny); + bckLeV = SelectVector ("bckLe", ANYVECTOR, TRUE); ResetVector (bckLeV, OPIHI_FLT, Ny); + bckLfV = SelectVector ("bckLf", ANYVECTOR, TRUE); ResetVector (bckLfV, OPIHI_FLT, Ny); + bckRsV = SelectVector ("bckRs", ANYVECTOR, TRUE); ResetVector (bckRsV, OPIHI_FLT, Ny); + bckReV = SelectVector ("bckRe", ANYVECTOR, TRUE); ResetVector (bckReV, OPIHI_FLT, Ny); + bckRfV = SelectVector ("bckRf", ANYVECTOR, TRUE); ResetVector (bckRfV, OPIHI_FLT, Ny); + } + + // loop over the y positions + // examine the slit window profile to measure background and slit sky + for (int iy = 0; iy < Ny; iy++) { + + // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + float dXtrace = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + + // Xref is the reference coordinate of the profile on the image + // Xprofile is the center of the profile. + // dXtrace follows the x-dir variations of the slit profile + float Xtrace = Xref - Xprofile + dXtrace; + + bckF_lf[iy] = get_segment_median(&buffV[iy*Nx], B_s_l + Xtrace, B_e_l + Xtrace, &bckX_lf[iy]); + bckF_rt[iy] = get_segment_median(&buffV[iy*Nx], B_s_r + Xtrace, B_e_r + Xtrace, &bckX_rt[iy]); + skyF_lf[iy] = get_segment_median(&buffV[iy*Nx], S_s_l + Xtrace, S_e_l + Xtrace, &skyX_lf[iy]); + skyF_rt[iy] = get_segment_median(&buffV[iy*Nx], S_s_r + Xtrace, S_e_r + Xtrace, &skyX_rt[iy]); + dxMax = MAX (dxMax, skyX_rt[iy] - skyX_lf[iy]); // this is a constant value unless we ignore some of the pixels + // assert skyX_rt > skyX_lf ? + + if (SaveSegments) { + skyLsV->elements.Flt[iy] = S_s_l + Xtrace; + skyLeV->elements.Flt[iy] = S_e_l + Xtrace; + skyLfV->elements.Flt[iy] = skyF_lf[iy]; + skyRsV->elements.Flt[iy] = S_s_r + Xtrace; + skyReV->elements.Flt[iy] = S_e_r + Xtrace; + skyRfV->elements.Flt[iy] = skyF_rt[iy]; + bckLsV->elements.Flt[iy] = B_s_l + Xtrace; + bckLeV->elements.Flt[iy] = B_e_l + Xtrace; + bckLfV->elements.Flt[iy] = bckF_lf[iy]; + bckRsV->elements.Flt[iy] = B_s_r + Xtrace; + bckReV->elements.Flt[iy] = B_e_r + Xtrace; + bckRfV->elements.Flt[iy] = bckF_rt[iy]; + } + } + + // extract sky-subtracted values for the psf fit, using the two sky vectors to predict the sky + float dyMax = dxMax*sin(stilt*RAD_DEG); // dxMax is based on the profile width, calculated above + + // find nominal dy_lf, dy_rt offsets for the sky vectors. + int dy_lf = -dyMax / 2; + int dy_rt = +dyMax / 2; + + // we are fitting this PSF to the data: + opihi_flt *psfV = psf->elements.Flt; + + // init the regions which are outside the valid region + for (int iy = 0; iy < fabs(dyMax); iy++) { + object->elements.Flt[iy] = 0; + sky->elements.Flt[iy] = 0; + backgnd->elements.Flt[iy] = 0; + } + for (int iy = Ny - fabs(dyMax); iy < Ny; iy++) { + object->elements.Flt[iy] = 0; + sky->elements.Flt[iy] = 0; + backgnd->elements.Flt[iy] = 0; + } + + + // Ys, Ye based on the dyMax value above (skip the first few lines to avoid running out of bounds on sky) + for (int iy = fabs(dyMax); iy < Ny - fabs(dyMax); iy++) { + + // X_lf & X_rt are calculated in image x-coords + float S_lf = skyF_lf[iy + dy_lf]; + float S_rt = skyF_rt[iy + dy_rt]; + float X_lf = skyX_lf[iy + dy_lf]; + float X_rt = skyX_rt[iy + dy_rt]; + + // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + float dXtraceSLIT = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); + float dXtracePSF = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); + + // Xref is the reference coordinate of the profile on the image + // Xpsf is the center of the PSF. + // dXtrace follows the x-dir variations of the slit profile (or PSF?) + float XtracePSF = Xref - Xpsf + dXtraceSLIT + dXtracePSF; + float XtraceSLIT = Xref - Xprofile + dXtraceSLIT; + + // include the PSF trace and restrict the fit below to portions where the profile is > 0.95 + + // now I need to find the limits of the PSF in the buffer at this iy location + int Ntmp = 0; + for (int ipsf = 0; ipsf < Npsf; ipsf ++) { + + int ix = ipsf + XtracePSF + 0.5; // image coordinate (nearest pixel center) + int islit = ix - XtraceSLIT - 0.5; // image coordinate (nearest pixel center) + + float So = S_lf + (ix - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak + + if (islit < 0) continue; + if (islit >= profile->Nelements) continue; + + if (iy == 6000) { + fprintf (stderr, "%d : %f %f : %f : %f\n", ipsf, buffV[ix + iy*Nx] - So, psfV[ipsf], (buffV[ix + iy*Nx] - So)*psfV[ipsf], profile->elements.Flt[islit]); + } + + if (profile->elements.Flt[islit] < 0.95) continue; + + tmpv[Ntmp] = buffV[ix + iy*Nx] - So; + tmpx[Ntmp] = psfV[ipsf]; + Ntmp ++; + } + + // I now have a pair of vectors (tmpx, tmpv) which are aligned to the PSF at this location + // now fit the PSF to tmpv by calculating a normalization + float S1 = 0.0, S2 = 0.0; + for (int ipsf = 0; ipsf < Ntmp; ipsf++) { + if (iy == 6000) { + fprintf (stderr, "%d : %f %f : %f\n", ipsf, tmpv[ipsf], tmpx[ipsf], tmpv[ipsf]*tmpx[ipsf]); + } + S1 += tmpv[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + S2 += tmpx[ipsf]*tmpx[ipsf]; // if we have an errorbar, include / SQ(sigv[i]) + } + + int Xo = Xref - dXtraceSLIT - dXtracePSF; // center of PSF in image x-coordinates + float So = S_lf + (Xo - X_lf)*(S_rt - S_lf)/(X_rt - X_lf); // interpolated sky under PSF peak + object->elements.Flt[iy] = S1 / S2; + sky->elements.Flt[iy] = So; + backgnd->elements.Flt[iy] = 0.5*(bckF_lf[iy] + bckF_rt[iy]); + } + + free (tmpv); + free (tmpx); + + free (bckF_lf); + free (bckF_rt); + free (skyF_lf); + free (skyF_rt); + + free (bckX_lf); + free (bckX_rt); + free (skyX_lf); + free (skyX_rt); + + return TRUE; + + usage: + gprint (GP_ERR, "USAGE: deimos getobj (buffer) (Xref) -object vector -sky vector -backgnd vector -trace spline -profile vector -psf vector -stilt (angle)\n"); + // can I make default values for trace (0.0), profile (?), psf (fraction of profile?), stilt (0.0) + return FALSE; +} + + /* + // convolve the object flux at this y-coord with PSF and add sky + for (int ix = 0; i < Nx; i++) { + opihi_flt g = s = 0; + for (int n = -Npsf; n <= Npsf; n++) { + if (ix + n < 0) continue; + if (ix + n >= Nx) continue; + opihi_flt value = objV ? objV[ix+n] : 1.0; + s += psfV[n]*value; + g += psfV[n]; + } + out[ix + iy*Nx] = Npsf ? s / g : (objV ? objV[ix+n] : 1.0); + if (skyV) { + out[ix + iy*Nx] += skyV[iy] + } + */ + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkalt.c (revision 41170) @@ -1,4 +1,6 @@ # include "data.h" # include "deimos.h" + +XXX: deprecated // Use this to test the deimos_make_object function used by deimos_fitobj @@ -92,5 +94,5 @@ // generate the model based on the test values - float *model = deimos_make_model (objV, skyV, bckV, psf, profile, trace, Nx, Ny, 0); + float *model = deimos_make_model (objV, skyV, bckV, psf, profile, trace, NULL, Nx, Ny, 0); ResetBuffer (output, Nx, Ny, -32, 0.0, 1.0); Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkmodel.c (revision 41170) @@ -86,11 +86,12 @@ // generate a subimage of the full model for the range row - row + Ny // obj,sky,bck are subset vectors corresponding to the range row to row+Ny -float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *trace, int Nx, int Ny, int row) { - - float *raw = deimos_make_straight_image (obj, sky, psf, Nx, Ny, row); +// the slit_trace_* and psf_trace refer to the full image starting at row +float *deimos_make_model (opihi_flt *obj, opihi_flt *sky, opihi_flt *bck, Vector *psf, Vector *profile, Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, Spline *psf_trace, int Nx, int Ny, int row) { + + float *raw = deimos_make_straight_image (obj, sky, psf, psf_trace, Nx, Ny, row); float *new = deimos_apply_tilt (raw, Nx, Ny); deimos_apply_profile (profile, new, Nx, Ny); deimos_add_background (bck, new, Nx, Ny, row); - float *out = deimos_apply_trace (trace, new, Nx, Ny, row); + float *out = deimos_apply_trace (slit_trace_red, slit_trace_blu, redlimit, new, Nx, Ny, row); free (raw); @@ -104,5 +105,5 @@ // *** generate a raw image with no slit tilt and no trace offsets. apply the PSF to the // object flux, add in the sky flux -float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, int Nx, int Ny, int row) { +float *deimos_make_straight_image (opihi_flt *obj, opihi_flt *sky, Vector *psf, Spline *psf_trace, int Nx, int Ny, int row) { OHANA_UNUSED_PARAM(row); @@ -113,5 +114,5 @@ // Xref is a global supplied by the user (defaults to Nx/2 if < 0) - int Xoff = (int)(Xref - Npsf); + int XoffRef = (int)(Xref - Npsf); // Nx : width of output window @@ -123,7 +124,26 @@ // we are generating the image for just the row range row to row+Ny + // note that obj & sky are subset vectors for just this range of pixels opihi_flt objVy = obj[iy]; // if we shift to using the row offset: obj[iy + row] opihi_flt skyVy = sky[iy]; // if we shift to using the row offset: sky[iy + row] + // integral and fractional pixel offsets of PSF (none, by default) + int dxi = 0; + float dxf = 0.0, dxr = 1.0; + int Xoff = XoffRef; + + if (psf_trace) { + // the psf_trace is referenced against the full image, so we need to add row to iy: + float dx = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy + row); + dxi = floor(dx); + dxf = dx - dxi; // -1.7 -> +0.3, -0.5 -> +0.5, +0.5 ->+0.5, +1.7 -> +0.7 + dxr = 1 - dxf; + Xoff += dxi; + // fprintf (stderr, "%d: %f : %d\n", iy + row, dx, Xoff); + } + + // if fractional offset is small, do not interpolate + int doInterp = fabs(dxf) < 1e-5 ? FALSE : TRUE; + // flux = obj * PSF + sky for (int ix = 0; ix < Nx; ix++) { @@ -136,5 +156,20 @@ // only add in the flux if we are in range of the PSF if ((n >= 0) && (n < NpsfFull)) { - value += objVy * psfV[n]; + if (doInterp) { + if ((n > 0) && (n < NpsfFull)) { + if (isfinite(psfV[n]) && isfinite(psfV[n-1])) { + value += objVy*psfV[n]*dxr + objVy*psfV[n-1]*dxf; + } + } + if (n == 0) { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]*dxr; + } + } + } else { + if (isfinite(psfV[n])) { + value += objVy * psfV[n]; + } + } } out[ix + iy*Nx] = value; @@ -219,5 +254,5 @@ // shift pixels in x based on the trace -float *deimos_apply_trace (Spline *trace, float *input, int Nx, int Ny, int row) { +float *deimos_apply_trace (Spline *slit_trace_red, Spline *slit_trace_blu, float redlimit, float *input, int Nx, int Ny, int row) { ALLOCATE_PTR (output, float, Nx*Ny); @@ -227,5 +262,7 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center // NOTE: spline is evaluated at full dispersion coordinate - float dx = -spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy + row); + int iy_full = iy + row; + Spline *slit_trace = (iy_full < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = -spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy_full); // extract the integer pixel offset and the fractional offset Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkobj.c (revision 41170) @@ -6,33 +6,60 @@ // input parameters: - // * trace : spline fit of slit central x pos vs y-coord - // * profile : slit window profile (vector) - // * object : vector of object flux vs y-coord - // * sky : vector of local sky signal vs y-coord - // * background : vector of extra-slit background flux vs y-coord - // * PSF : point-spread function vector (flux normalized, x-dir) - // * LSF : sline-spread function vector (flux normalized, y-dir) - // * stilt : slit tilt response : 2D kernel? + // * slit_trace_red : spline fit of slit central x pos vs y-coord for red chip + // * slit_trace_blu : spline fit of slit central x pos vs y-coord for blu chip + // * psf_trace : spline fit of psf central x pos vs y-coord + // * profile : slit window profile (vector) + // * object : vector of object flux vs y-coord + // * sky : vector of local sky signal vs y-coord + // * backgnd : vector of extra-slit background flux vs y-coord + // * PSF : point-spread function vector (flux normalized, x-dir) + // * LSF : sline-spread function vector (flux normalized, y-dir) + // * stilt : slit tilt response : 2D kernel? + + // * redlimit : pixel break between red and blu chips + // * Nwave : number of wavelength pixels in output image int N; // if any of these are not defined, they will have assumed identity values - Spline *trace = NULL; - Vector *profile = NULL; - Vector *object = NULL; - Vector *sky = NULL; - Vector *backgnd = NULL; - - Vector *psf = NULL; - Vector *lsf = NULL; + Spline *slit_trace_red = NULL; + Spline *slit_trace_blu = NULL; + Spline *psf_trace = NULL; + Vector *profile = NULL; + Vector *object = NULL; + Vector *sky = NULL; + Vector *backgnd = NULL; + + Vector *psf = NULL; + Vector *lsf = NULL; // add tilt later - Buffer *output = NULL; + Buffer *output = NULL; float stilt = 0.0; // angle of the slit - if ((N = get_argument (argc, argv, "-trace"))) { - remove_argument (N, &argc, argv); - if ((trace = FindSpline (argv[N])) == NULL) return (FALSE); + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-slit-trace-red"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_red = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-slit-trace-blu"))) { + remove_argument (N, &argc, argv); + if ((slit_trace_blu = FindSpline (argv[N])) == NULL) return (FALSE); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-psf-trace"))) { + remove_argument (N, &argc, argv); + if ((psf_trace = FindSpline (argv[N])) == NULL) return (FALSE); remove_argument (N, &argc, argv); } @@ -81,6 +108,24 @@ } + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + if (argc != 3) { - gprint (GP_ERR, "USAGE: deimos mkobj (buffer) (Ncross) [-Nwave N] [-object vector] [-sky vector] [-backgnd vector] [-trace spline] [-profile vector]\n"); + gprint (GP_ERR, "USAGE: deimos mkobj (buffer) (Ncross) [-object vector] [-sky vector] [-backgnd vector] [-slit-trace-red spline] [-slit-trace-blu spline] [-psf-trace spline] [-profile vector] [-psf vector] [-lsf vector] [-stilt angle] [-Nwave Npixel] [-redlimit pixel]\n"); + gprint (GP_ERR, " slit_trace_red : spline fit of slit central x pos vs y-coord for red chip\n"); + gprint (GP_ERR, " slit_trace_blu : spline fit of slit central x pos vs y-coord for blu chip\n"); + gprint (GP_ERR, " psf_trace : spline fit of psf central x pos vs y-coord\n"); + gprint (GP_ERR, " profile : slit window profile (vector)\n"); + gprint (GP_ERR, " object : vector of object flux vs y-coord \n"); + gprint (GP_ERR, " sky : vector of local sky signal vs y-coord\n"); + gprint (GP_ERR, " backgnd : vector of extra-slit background flux vs y-coord\n"); + gprint (GP_ERR, " PSF : point-spread function vector (flux normalized, x-dir)\n"); + gprint (GP_ERR, " LSF : sline-spread function vector (flux normalized, y-dir)\n"); + gprint (GP_ERR, " stilt : slit tilt response : 2D kernel? \n"); + gprint (GP_ERR, " redlimit : pixel break between red and blu chips\n"); + gprint (GP_ERR, " Nwave : number of wavelength pixels in output image\n"); return FALSE; } @@ -123,10 +168,11 @@ // if we are appying a trace offset spline, we need to generate an output window which // is Nx + the full swing of the trace, then window back down - if (trace) { + if (slit_trace_red && slit_trace_blu) { float dXmin = +1000; float dXmax = -1000; for (int iy = 0; iy < Ny; iy++) { // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dx = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); dXmin = MIN (dx, dXmin); dXmax = MAX (dx, dXmax); @@ -181,4 +227,21 @@ if (psf) Npof -= Npsf; + // integral and fractional pixel offsets of PSF (none, by default) + int dxi = 0; + float dxf = 0.0, dxr = 1.0; + + if (psf_trace) { + float dx = spline_apply_dbl (psf_trace->xk, psf_trace->yk, psf_trace->y2, psf_trace->Nknots, iy); + dxi = floor(dx); + dxf = dx - dxi; // -1.7 -> +0.3, -0.5 -> +0.5, +0.5 ->+0.5, +1.7 -> +0.7 + dxr = 1 - dxf; + Npof += dxi; + // fprintf (stderr, "%d: %f : %d\n", iy, dx, Npof); + if (VERBOSE && (iy % 200 == 0)) { gprint (GP_ERR, "%d : %f : %d : %f %f\n", iy, dx, dxi, dxf, dxr); } + } + + // if fractional offset is small, do not interpolate + int doInterp = fabs(dxf) < 1e-5 ? FALSE : TRUE; + // flux = obj * PSF + sky for (int ix = 0; ix < Nx; ix++) { @@ -191,5 +254,20 @@ // only add in the flux if we are in range of the PSF if ((n >= 0) && (n < NpsfFull)) { - value += objVy * psfV[n]; + if (doInterp) { + if ((n > 0) && (n < NpsfFull)) { + if (isfinite(psfV[n]) && isfinite(psfV[n-1])) { + value += objVy*psfV[n]*dxr + objVy*psfV[n-1]*dxf; + } + } + if (n == 0) { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]*dxr; + } + } + } else { + if (isfinite(psfV[n])) { + value += objVy*psfV[n]; + } + } } } else { @@ -329,5 +407,5 @@ // shift pixels in x based on the trace - if (trace) { + if (slit_trace_red && slit_trace_blu) { ALLOCATE_PTR (outTraceBuffer, char, NxBase*Ny*sizeof(float)); float *outTrace = (float *) outTraceBuffer; @@ -337,5 +415,6 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center - float dx = -spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); + Spline *slit_trace = (iy < redlimit) ? slit_trace_red : slit_trace_blu; + float dx = -spline_apply_dbl (slit_trace->xk, slit_trace->yk, slit_trace->y2, slit_trace->Nknots, iy); // extract the integer pixel offset and the fractional offset Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/deimos_mkslit.c (revision 41170) @@ -7,14 +7,21 @@ // input parameters: // slit profile response : vector of fractional flux vs x-coord - // trace : spline fit of slit central x pos vs y-coord - // flux : vector of input signal vs y-coord + // trace_ref,trace_blu : spline fit of slit central x pos vs y-coord + // flux (optional) : vector of input signal vs y-coord + // sky (optional) : vector of input signal vs y-coord + // fluxbins (optional) : size of output image in y-direction + // NOTE: one of flux, sky, or fluxbins must be defined + + // output : buff (an image with Nx defined by profile and Ny defined + // by flux, sky, or fluxbins) int N; - Vector *profile = NULL; - Vector *flux = NULL; - Vector *sky = NULL; - Spline *trace = NULL; - Buffer *buff = NULL; + Vector *profile = NULL; + Vector *flux = NULL; + Vector *sky = NULL; + Spline *trace_red = NULL; + Spline *trace_blu = NULL; + Buffer *buff = NULL; // user-specified flux value (otherwise assumed to be 1) @@ -40,6 +47,24 @@ } - if (argc != 4) { - gprint (GP_ERR, "USAGE: deimos mkslit (profile) (trace) (buffer) [-flux vector] [-fluxbins N] [-sky vector]\n"); + // either flux or fluxbins must be specified to define output size + int profilebins = 0; + if ((N = get_argument (argc, argv, "-profilebins"))) { + remove_argument (N, &argc, argv); + profilebins = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + // for a red vs blu spline, we need to specify the split point + // XXX this is REALLY ad-hoc for Deimos. not sure how to make + // this more generic (need to define the ranges somewhere) + int redlimit = 4096; + if ((N = get_argument (argc, argv, "-redlimit"))) { + remove_argument (N, &argc, argv); + redlimit = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: deimos mkslit (profile) (trace_red) (trace_blu) (buffer) [-flux vector] [-fluxbins N] [-sky vector]\n"); return FALSE; } @@ -51,10 +76,12 @@ // XXX I probably should rename FindSpline as SelectSpline and give it the same behavior - if ((profile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); - if ((trace = FindSpline (argv[2])) == NULL) return (FALSE); - if ((buff = SelectBuffer (argv[3], ANYBUFFER, TRUE)) == NULL) return (FALSE); + if ((profile = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((trace_red = FindSpline (argv[2])) == NULL) return (FALSE); + if ((trace_blu = FindSpline (argv[3])) == NULL) return (FALSE); + if ((buff = SelectBuffer (argv[4], ANYBUFFER, TRUE)) == NULL) return (FALSE); // define the output window - int Nx = 2*profile[0].Nelements + 1; + // if no profile half-width is specified, use supplied profile + int Nx = profilebins ? 2*profilebins + 1 : 2*profile[0].Nelements + 1; int Ny = flux ? flux[0].Nelements : fluxbins; @@ -73,4 +100,5 @@ // evaluate the trace spline at this y-coord to find the x-coord offset of the profile center + Spline *trace = (iy < redlimit) ? trace_red : trace_blu; float dx = spline_apply_dbl (trace->xk, trace->yk, trace->y2, trace->Nknots, iy); @@ -90,5 +118,5 @@ // set the sky in the entire window - out[ix + iy*Nx] = sky ? sky->elements.Flt[iy] : 0.0; + out[ix + iy*Nx] = NAN; // equivalent coord in the profile: @@ -98,4 +126,5 @@ // a default value: + float vsky = sky ? sky->elements.Flt[iy] : 0.0; float vout = NAN; @@ -121,5 +150,5 @@ if (flux) vout *= flux->elements.Flt[iy]; - out[ix + iy*Nx] += vout; + out[ix + iy*Nx] = vsky + vout; } } Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitarc.sh (revision 41170) @@ -0,0 +1,91 @@ + +macro test_01 + if ($0 != 2) + echo "USAGE: test_01 (offset)" + break + end + + # make a gaussian profile + create x 0 31 + set yo = 105.1*exp(-0.5*(x - 12.2)^2/2.2^2) + gaussdev dg yo[] 0 1 + # set dg = zero(yo) + set y = yo + dg*sqrt(yo) + + # fitarc version + dimenup y im 1 31 + deimos fitarc im 0 0 1 31 -resid imf -max-iterations 0 + dimendown imf yf1 + + echo $C0 $C2 $C1 $C3 + + $C0 = $C0 + $1 + vgauss -apply x yf2 + + # dy1 : model from fitarc + # dy2 : rebuilt model + # NOTE: vgauss and fitarc output results match exactly + # asymmetry is a result of the fitted values + + set dy1 = y - yf1 + set dy2 = y - yf2 + + lim -n 1 x dy2; clear; box; + + plot x dy1 -c red -x hist + plot x dy2 -c blue -x hist + + lim -n 2 x y; clear; box; + + plot x y -c black -x hist -lw 4 + + plot x yf1 -c red -x hist -lw 3 + plot x yf2 -c blue -x hist -lw 2 +end + +macro test_02 + if ($0 != 2) + echo "USAGE: test_02 (offset)" + break + end + + # make a gaussian profile + create x 0 31 + set yo = 105.1*exp(-0.5*(x - 12.2)^2/2.2^2) + gaussdev dg yo[] 0 1 + set dg = zero(yo) + set y = yo + dg*sqrt(yo) + + # vgauss version + $C0 = 15 + $C1 = 2 + $C2 = 100 + $C3 = 0 + vgauss x y con yf1 -q + echo $C0 $C2 $C1 $C3 + $C0 = $C0 + $1 + vgauss -apply x yf3 + + set dy1 = y - yf1 + set dy3 = y - yf3 + + # fitarc version + dimenup y im 1 31 + deimos fitarc im 0 0 1 31 -resid dim -max-iterations 0 + dimendown dim yf2 + set dy2 = y - yf2 + + lim -n 1 x dy2; clear; box; + + plot x dy1 -c red -x hist + plot x dy2 -c blue -x hist + plot x dy3 -c black -x hist + + lim -n 2 x y; clear; box; + + plot x y -c black -x hist -lw 3 + + plot x yf2 -c blue -x hist -lw 3 + plot x yf1 -c red -x hist -lw 2 + plot x yf3 -c blue60 -x hist +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitprofile.sh (revision 41170) @@ -0,0 +1,28 @@ + +macro memtest1 + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set fprofile = profile_left + profile_mid + profile_right + set wprofile = 0.02*fprofile + + $i = 0 + deimos fitprofile -q xprof fprofile wprofile $Xpmin $Xpmax + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos fitprofile -q xprof fprofile wprofile $Xpmin $Xpmax + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_fitslit.sh (revision 41170) @@ -0,0 +1,31 @@ + +macro memtest1 + # make a delta-function profile + create x 0 11 + set yg = exp(-0.25*(x - 5)^2) + + # flux vs wavelength is flat & short + create w 0 100 + set f = zero(w) + 1 + + delete -q Xn Yn + vlist Xn 0 100 + vlist Yn 0 2 + + spline create trace Xn Yn + spline apply trace w dX + $i = 0 + + # run once to ensure the output is create + deimos mkslit yg trace trace outbuff -flux f + mgaussdev noise outbuff[][0] outbuff[0][] 0.0 0.2 + + set obs = 2*outbuff + noise + deimos fitslit obs outbuff outf outs + + memory check + for i 0 1000 + deimos mkslit yg trace trace outbuff -flux f + end + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkalt.sh (revision 41170) @@ -0,0 +1,55 @@ + +macro memtest1 + # generate LSF: + create x 0 7 + set lsf = exp(-0.5*(x-3)^2) + + # generate PSF: + create xp 0 9 + set psf = exp(-0.5*(xp-5)^2/1.5^2) + + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set profile = profile_left + profile_mid + profile_right + + # generate object flux: + create wave 0 100 + set flux = 100 + 0.5*wave + + # generate sky flux: + set sky = 200 + zero(wave) + + # off-slit background: + set backgnd = 300 + zero(wave) + + # slit trace + vlist Xn 0 100 + vlist Yn 0 4 + spline create slit_trace_in Xn Yn + + # psf trace + vlist Xn 0 25 50 75 100 + vlist Yn 0 2 4 2 0 + spline create psf_trace_in Xn Yn + + $i = 0 + deimos mkalt out 31 -psf psf -object flux -sky sky -stilt 20.0 -profile profile -backgnd backgnd -trace slit_trace_in + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos mkalt out 31 -psf psf -object flux -sky sky -stilt 20.0 -profile profile -backgnd backgnd -trace slit_trace_in + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkobj.sh (revision 41170) @@ -0,0 +1,55 @@ + +macro memtest1 + # generate LSF: + create x 0 7 + set lsf = exp(-0.5*(x-3)^2) + + # generate PSF: + create xp 0 9 + set psf = exp(-0.5*(xp-5)^2/1.5^2) + + # generate slit profile: + create xprof 0 17 + $Xpmin = 3 + $Xpmax = 13 + set profile_left = (xprof <= $Xpmin) ? exp(-0.5*(xprof - $Xpmin)^2/1.0^2) : zero(xprof) + set profile_right = (xprof >= $Xpmax) ? exp(-0.5*(xprof - $Xpmax)^2/1.0^2) : zero(xprof) + set profile_mid = (xprof > $Xpmin) && (xprof < $Xpmax) ? zero(xprof) + 1 : zero(xprof) + set profile = profile_left + profile_mid + profile_right + + # generate object flux: + create wave 0 100 + set flux = 100 + 0.5*wave + + # generate sky flux: + set sky = 200 + zero(wave) + + # off-slit background: + set backgnd = 300 + zero(wave) + + # slit trace + vlist Xn 0 100 + vlist Yn 0 4 + spline create slit_trace_in Xn Yn + + # psf trace + vlist Xn 0 25 50 75 100 + vlist Yn 0 2 4 2 0 + spline create psf_trace_in Xn Yn + + $i = 0 + deimos mkobj out 31 -psf psf -object flux -sky sky -stilt 20.0 -lsf lsf -profile profile -backgnd backgnd -slit-trace slit_trace_in -psf-trace psf_trace_in + + # tv -n 0 out 250 750 + # center 15 50 20 + # resize 850 2050 + # tvcolor rainbow + # break + + memory check + for i 0 1000 + deimos mkobj out 31 -psf psf -object flux -sky sky -stilt 20.0 -lsf lsf -profile profile -backgnd backgnd -slit-trace slit_trace_in -psf-trace psf_trace_in + end + memory check +end + Index: branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh (revision 41170) +++ branches/eam_branches/ipp-20191011/Ohana/src/opihi/mana/test/deimos_mkslit.sh (revision 41170) @@ -0,0 +1,28 @@ + + +macro memtest1 + # make a delta-function profile + create x 0 11 + set yg = exp(-0.25*(x - 5)^2) + + # flux vs wavelength is flat & short + create w 0 100 + set f = zero(w) + 1 + + delete -q Xn Yn + vlist Xn 0 100 + vlist Yn 0 2 + + spline create trace Xn Yn + spline apply trace w dX + $i = 0 + + # run once to ensure the output is create + deimos mkslit yg trace trace outbuff -flux f + + memory check + for i 0 1000 + deimos mkslit yg trace trace outbuff -flux f + end + memory check +end Index: branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/reload_catalogs.c (revision 41170) @@ -62,4 +62,8 @@ catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } catalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog Index: branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c =================================================================== --- branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c (revision 41169) +++ branches/eam_branches/ipp-20191011/Ohana/src/relphot/src/relphot_objects.c (revision 41170) @@ -73,4 +73,8 @@ catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -162,5 +166,11 @@ // we can optionally convert output format here // but it would be better to define a dvo crawler program to do this - // catalog.catformat = DVO_FORMAT_PS1_V1; + // catalog.catformat = DVO_FORMAT_PS1_V1; + + // modify the output format as desired (ignore current format on disk) + if (UPDATE_CATFORMAT) { + catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } + SetProtect (TRUE); if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } Index: branches/eam_branches/ipp-20191011/backups/README =================================================================== --- branches/eam_branches/ipp-20191011/backups/README (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/README (revision 41170) @@ -0,0 +1,13 @@ +README + +This module contains files that perform backups of various services used by +the IPP. Each file has it's own backup class, which is inherited from the base +class. To perform a backup there needs to a config that matches the expected +schema. + +Run 'deploy_backups.py' to deploy the relevant scripts and utils to the machines +that the services live on. + +Check a config is in place for each backup. + +If automated backup is desired run the script periodically in a crontab. Index: branches/eam_branches/ipp-20191011/backups/backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/backup.py (revision 41170) @@ -0,0 +1,301 @@ +import configparser +import datetime +import os +import os.path as path +import socket +import subprocess +from configparser import ConfigParser + +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing + +DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] +) +COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) +TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) +RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, SpecialSchemaProcessing.commma_separated) + ] +) + +MYSQLDUMP_SCHEMA_SECTION = ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) + + +def get_backup_schema() -> ConfigSchema: + backup_schema = ConfigSchema(None) + backup_schema.add_section(DEFAULT_SCHEMA_SECTION) + backup_schema.add_section(COPY_SCHEMA_SECTION) + backup_schema.add_section(TAR_SCHEMA_SECTION) + backup_schema.add_section(RSYNC_SCHEMA_SECTION) + backup_schema.add_section(MYSQLDUMP_SCHEMA_SECTION) + return backup_schema + + +class Backup(): + """Base class for creating backups, contains all the generic versions of +functions for copying, taring etc. + """ + + def __init__(self, config_file=None, schema: ConfigSchema=None): + if config_file is None: + raise errs.ValidationError("ValidationError: Backup class must be given a config") + if schema is None: + schema = get_backup_schema() + self.load_config(config_file, schema) + + def load_config(self, config_class_or_filepath, schema): + """Accepts a filepath to a config file, or """ + + config = ConfigParser() + expected_class = configparser.ConfigParser + if config_class_or_filepath is None: + raise errs.ValidationError("ValidationError: config settings or filepath to config must be given") + elif isinstance(config_class_or_filepath, expected_class): + config = config_class_or_filepath + elif not path.exists(config_class_or_filepath): + raise errs.ValidationError(f"ValidationError: Config filepath does not exist (or incorrect class given, expected: {expected_class})") + config.read(config_class_or_filepath) + + self.read_config_and_schema(config, schema) + + def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): + parsed_config = ParsedConfig(config, schema) + self.default_dict = parsed_config.get_section('DEFAULT') + self.hostname_is_valid() + self.maybe_make_backup_dirs() + + section = 'COPY' + self.should_run_copy = parsed_config.has_section(section) + if self.should_run_copy: + self.copy_dict = parsed_config.get_section(section) + + section = 'TAR' + self.should_run_tar = parsed_config.has_section(section) + if self.should_run_tar: + self.tar_dict = parsed_config.get_section(section) + + section = 'RSYNC' + self.should_run_rsync = parsed_config.has_section(section) + if self.should_run_rsync: + self.rsync_dict = parsed_config.get_section(section) + + section = 'MYSQLDUMP' + self.should_run_mysqldump = parsed_config.has_section(section) + if self.should_run_mysqldump: + self.mysqldump_dict = parsed_config.get_section(section) + + def run_backup_processes(self): + self._write_begin_log() + if self.should_run_copy: + self._run_copy() + if self.should_run_tar: + self._run_tar() + if self.should_run_rsync: + self._run_rsync() + if self.should_run_mysqldump: + self._run_mysqldump() + self._write_end_log() + + def hostname_is_valid(self) -> bool: + if self.default_dict is None or 'source_machine' not in self.default_dict.keys(): + raise ValueError('No source machine specified') + + actual_hostname = socket.gethostname() + return actual_hostname == self.default_dict['source_machine'] + + def target_backup_paths_ok(self, raise_error=False): + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + if raise_error: + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + return False + return True + + def maybe_make_backup_dirs(self): + if self.default_dict is None or 'make_dirs' not in self.default_dict.keys(): + return + if self.default_dict['make_dirs']: + for backup_path in self.default_dict['backup_paths']: + if not os.path.exists(backup_path): + os.makedirs(backup_path) + sub_utils.chmod_wrapper(['777', backup_path]) + + def _run_copy(self): + if self.should_run_copy is None or False: + return 0 + self.target_backup_paths_ok(raise_error=True) + + for bp in self.default_dict['backup_paths']: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.copy_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_tar(self): + if self.should_run_tar is None or False: + return 0 + + backup_paths = self.default_dict['backup_paths'] + + filename = self.tar_dict['target_filename'] + if 'prefix_date' in self.tar_dict.keys() and self.tar_dict['prefix_date']: + filename = f"{get_date()}_{filename}" + filename = path.join(backup_paths[0], filename) # may need some tar suffix magic + + add_args = [] if 'additional_args' not in self.tar_dict.keys() else self.tar_dict['additional_args'] + all_args = add_args + [filename] + self.tar_dict['sources'] + + try: + sub_utils.tar_wrapper(all_args) + sub_utils.chmod_wrapper(['774', filename]) + # copy to backups + if len(backup_paths) == 1: + return 0 + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([ filename, backup_paths[d]]) + except errs.SubprocessError as e: + raise errs.SubprocessError() from e + return 0 + + def _run_rsync(self): + if self.should_run_rsync is None or False: + return 0 + + self.target_backup_paths_ok(raise_error=True) + backup_paths = self.default_dict['backup_paths'] + + sources = self.rsync_dict['sources'] + for i in range(0, len(sources)): + source = sources[i] + if not path.exists(source): + raise errs.ValidationError(f"ValidationError: target path does not exist: {source}") + + for bp in backup_paths: + try: + additional_args = [] + if 'additional_args' in self.rsync_dict.keys(): + additional_args = self.rsync_dict['additional_args'] + + source_and_target_args = [source, bp] + + # sub-dirs option allows items to be backed up to subdirs within + # the source for better naming + if 'sub-dirs' in self.rsync_dict: + sub_dirs = self.rsync_dict['sub-dirs'] + if len(sub_dirs) != len(sources): + raise errs.ValidationError(f"ValidationError: if using " + + "subdirs the number of subdirs must equal the number of sources") + bp_with_subdir = os.path.join(bp, sub_dirs[i]) + if not os.path.exists(bp_with_subdir): + os.makedirs(bp_with_subdir) + source_and_target_args = [source, bp_with_subdir] + + args = source_and_target_args + additional_args + sub_utils.local_rsync_wrapper(args) + + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: rsync\n" + f" from: {source_and_target_args[0]}\n" + f" to: {source_and_target_args[1]}\n" + ) from e + return 0 + + def _run_mysqldump(self): + + backup_paths = self.default_dict['backup_paths'] + + mysqldump_args = [ 'mysqldump' + , '-u', self.mysqldump_dict['user'] + , f"-p{self.mysqldump_dict['password']}" + , '--databases', self.mysqldump_dict['db_name'] + ] + if 'additional_args' in self.mysqldump_dict.keys(): + mysqldump_args = mysqldump_args + self.mysqldump_dict['additional_args'] + + dump_filename = self.mysqldump_dict['target_filename'] + if 'prefix_date' in self.mysqldump_dict.keys() and self.mysqldump_dict['prefix_date']: + dump_filename = f"{get_date()}_{dump_filename}" + dump_filename = path.join(backup_paths[0], dump_filename) + + if path.exists(dump_filename): + raise errs.ValidationError(f"ValidationError: mysqldump already exists {dump_filename}") + + with open(dump_filename, "w+") as df: + mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) + dump_result = mysqldump_process.communicate() + + # Get the piped std err. See if it contains the word error + stdo, stde = dump_result + if stde is not None: + decoded_stderr = stde.decode() + if decoded_stderr.casefold().find('error'.casefold()) > -1: + raise errs.SubprocessError(f"SubprocessError: Errors detected in mysqldump:\n{decoded_stderr}") + + # copy the dump to other locations + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([dump_filename, backup_paths[d]]) + return 0 + + def _write_begin_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if path.exists(log_file): + os.remove(log_file) + with open(log_file, "w+") as lf: + lf.write(f'Run started at: {datetime.datetime.now().strftime("%Y_%b_%d_%H:%M")}\n') + + def _write_end_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if not path.exists(log_file): + return + with open(log_file, "a") as lf: + lf.write(f'Last successful backup completed at : {datetime.datetime.now().strftime("%Y_%b_%d_%H:%M")}\n') + + +def get_date(custom_date=None, custom_format="%Y_%m_%d"): + # Format is currently: 2019-10-08 + if custom_date is not None: + return custom_date.strftime(custom_format) + return datetime.date.today().strftime(custom_format) Index: branches/eam_branches/ipp-20191011/backups/backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/backup_test.py (revision 41170) @@ -0,0 +1,777 @@ +import os +import pytest +import stat +import socket +from configparser import ConfigParser +from pathlib import Path + +import backup as bckup +import utils.errors as errs +from backup import Backup +from utils.config_parse_helper import ConfigSchema + +TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') + + +def make_default_config( tmpdir: str + , backup_targs: [str] = ['bck1', 'bck2'] + , verbose: bool = False + ) -> ConfigParser: + config = ConfigParser() + + # make the paths: + for b in backup_targs: + os.makedirs(os.path.join(tmpdir, b)) + + # Get the correct formats for the config + backup_paths = ','.join([os.path.join(tmpdir, b) for b in backup_targs]) + source_machine = socket.gethostname() + config['DEFAULT'] = \ + { 'backup_paths' : backup_paths + , 'source_machine' : source_machine + , 'verbose' : verbose + } + return config + + +class TestArguments(object): + + def test_default_config_file_load_default(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # Default + assert backy.default_dict['backup_paths'] == \ + [ "/export/ippops4.0/atlassian_backups" + , "/data/ippops3.0/atlassian_backups" + , "/data/ippops5.0/atlassian_backups" + ] + assert backy.default_dict['source_machine'] == "hostname_of_computer" + assert backy.default_dict['verbose'] is True + + def test_copy_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.copy_dict['sources'] == \ + [ "/some_dir_to_copy_from/" + , "/some_dir/and_specific_file.jpeg" + ] + assert backy.copy_dict['additional_args'] == ['-v'] + + def test_tar_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.tar_dict['sources'] == \ + [ "/this_dir/" + , "/another_dir/" + , "/and_a_couple_files/subdir/file1.txt" + , "/and_a_couple_files/subdir/file2.png" + , "/and_a_couple_files/subdir/file3.pdf" + ] + assert backy.tar_dict['additional_args'] == [ "--verbose" + , "-z" + , "-p" + ] + assert backy.tar_dict['target_filename'] == "cool_tar" + assert backy.tar_dict['prefix_date'] is True + + def test_rsync_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Rsync] + assert backy.rsync_dict['sources'] == [ "/dir/to/sync"] + assert backy.rsync_dict['additional_args'] == \ + [ "--progress" + , "-r" + ] + + def test_mysqldump_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Mysqldump] + assert backy.mysqldump_dict['user'] == "dumper" + assert backy.mysqldump_dict['password'] == "not_a_real_password" + assert backy.mysqldump_dict['db_name'] == "jiradb" + assert backy.mysqldump_dict['additional_args'] == [''] + assert backy.mysqldump_dict['target_filename'] == "itsa_mysqldump" + assert backy.mysqldump_dict['prefix_date'] is True + + def test_loading_only_defaults(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.default_dict['backup_paths'] == \ + [ os.path.join(tmpdir, 'bck1') + , os.path.join(tmpdir, 'bck2') + ] + assert backy.default_dict['source_machine'] == socket.gethostname() + assert backy.default_dict['verbose'] is False + + def test_failure_if_no_defaults_provided(self): + config = ConfigParser() + with pytest.raises(errs.ConfigError) as e_empty: + Backup(config_file=config) + assert "is required but was not found in config" in str(e_empty) + + with pytest.raises(errs.ValidationError) as e_none: + Backup(config_file=None) + assert "ValidationError: Backup class must be given a config" in str(e_none) + + with pytest.raises(errs.ValidationError) as e_nofile: + Backup(config_file="/not/a/real/file.config") + assert "ValidationError: Config filepath does not exist" in str(e_nofile) + + def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.should_run_copy is False + assert backy.should_run_tar is False + assert backy.should_run_rsync is False + assert backy.should_run_mysqldump is False + + def test_hostname_assertion_fails(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + backy.default_dict['source_machine'] = 'not_a_real_hostname' + assert backy.hostname_is_valid() is False + + def test_hostname_assertion_passes(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + actual_hostname = socket.gethostname() + backy.default_dict['source_machine'] = actual_hostname + assert backy.hostname_is_valid() is True + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exist_in_testing(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.target_backup_paths_ok() is False + + def test_raising_an_error_if_a_backup_path_does_not_exist(self): + with pytest.raises(errs.ValidationError) as e: + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + backy.target_backup_paths_ok(raise_error=True) + assert "ValidationError: target path does not exist" in str(e) + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + + backy.default_dict['backup_paths'] = ["/not/real/path"] + assert backy.target_backup_paths_ok() is False + + backy.default_dict['backup_paths'] = [tmpdir] + assert backy.target_backup_paths_ok() is True + + def test_backup_paths_created_if_make_dirs_given(self, tmpdir): + config = ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + bck1_path = os.path.join(tmpdir, 'bck1') + bck1b_path = os.path.join(tmpdir, 'bck1/bobby') + bck1bb_path = os.path.join(tmpdir, 'bck1/bobby/backup') + bck2_path = os.path.join(tmpdir, 'bck2') + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + # backup dirs not created automatically + Backup(config, schema) + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + + # make_dirs included in config - when loaded dirs created. + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + , 'make_dirs' : 'True' + } + backy = Backup(config, schema) + assert backy.default_dict['make_dirs'] is True + assert os.path.exists(bck1_path) + assert os.path.exists(bck1b_path) + assert os.path.exists(bck1bb_path) + assert os.path.exists(bck2_path) + + # Check permissions are 777 too + new_stats = os.stat(bck2_path) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestRunningCopy(): + + def test_simple_copy(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + # make file to be copied + Path(source).touch() + + backy = Backup(config, schema) + assert backy.should_run_copy is True + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_copy_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v --preserve -r" + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + os.makedirs(source1) + Path(source2).touch() + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_copy_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_requires_dash_r_arg_for_copying_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['COPY'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + with pytest.raises(errs.SubprocessError): + backy = Backup(config, schema) + backy._run_copy() + + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + +class TestRunningTar(): + + def test_simply_tar(self, tmpdir): + config = make_default_config(tmpdir, ['bck1']) + source = os.path.join(tmpdir, "folder_to_tar") + os.makedirs(source) + # put a file in it + Path(os.path.join(source, "a_file.txt")) + + config['TAR'] = \ + { 'sources' : f"{source}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_tar1 = os.path.join(tmpdir, 'bck1', 'cool_tar.tar') + print(expected_tar1) + assert not os.path.exists(expected_tar1) + backy._run_tar() + assert os.path.exists(expected_tar1) + + def test_tar_with_multiple_sources_and_date(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "folder_to_tar") + source2 = os.path.join(tmpdir, "another_folder_to_tar") + os.makedirs(source1) + os.makedirs(source2) + # put a file in it + Path(os.path.join(source1, "a_file.txt")) + Path(os.path.join(source2, "other.file")) + + config['TAR'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'True' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_tar is True + + date = bckup.get_date() + + expected_tar1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_tar.tar') + expected_tar2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_tar.tar') + assert not os.path.exists(expected_tar1) + assert not os.path.exists(expected_tar2) + backy._run_tar() + assert os.path.exists(expected_tar1) + assert os.path.exists(expected_tar2) + + +class TestRunningRsync(): + + def test_simple_rsync(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = {'sources' : f"{source}"} + # make file to be copied + Path(source).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_rsync_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source1) + Path(source2).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['RSYNC'] = {'sources' : f"{source}"} + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_requires_dash_r_arg_for_copying_only_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['RSYNC'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + backy = Backup(config, schema) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + def test_rsync_with_delete_after_options(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source_dir = os.path.join(tmpdir, "some_dir") + source_file1 = os.path.join(source_dir, "file1") + source_file2 = os.path.join(source_dir, "file2") + source_file3 = os.path.join(source_dir, "file3") + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source_dir) + Path(source_file1).touch() + Path(source_file2).touch() + Path(source_file3).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file1') + expected2dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file2') + expected3dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file3') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + backy._run_rsync() + print(expected1dir) + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + + # Delete two files, create a new one. + os.remove(source_file2) + os.remove(source_file3) + source_file4 = os.path.join(source_dir, "file4") + Path(source_file4).touch() + expected4dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file4') + + # Rsync will add 4, and 2 and 3 should still exist + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + # with delete after option removed files are cleaned up + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : '-v -z -r --delete-after' + } + backy = Backup(config, schema) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + def test_rsync_with_subdirs_option(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "some_other_dir") + file_in_dir = os.path.join(source1, "and_specific_file.jpeg") + file_in_other_dir = os.path.join(source2, "another_file.jpeg") + os.makedirs(source1) + os.makedirs(source2) + Path(file_in_dir).touch() + Path(file_in_other_dir).touch() + + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-a --progress -v" + , 'sub-dirs' : "subdir_with_better_name1, subdir_with_better_name2" + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_bak1_src1_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir') + expected_bak1_src1_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak1_src2_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir') + expected_bak1_src2_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + expected_bak2_src1_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir') + expected_bak2_src1_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak2_src2_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir') + expected_bak2_src2_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + assert not os.path.exists(expected_bak1_src1_dir) + assert not os.path.exists(expected_bak1_src1_file) + assert not os.path.exists(expected_bak1_src2_dir) + assert not os.path.exists(expected_bak1_src2_file) + assert not os.path.exists(expected_bak2_src1_dir) + assert not os.path.exists(expected_bak2_src1_file) + assert not os.path.exists(expected_bak2_src2_dir) + assert not os.path.exists(expected_bak2_src2_file) + backy._run_rsync() + assert os.path.exists(expected_bak1_src1_dir) + assert os.path.exists(expected_bak1_src1_file) + assert os.path.exists(expected_bak1_src2_dir) + assert os.path.exists(expected_bak1_src2_file) + assert os.path.exists(expected_bak2_src1_dir) + assert os.path.exists(expected_bak2_src1_file) + assert os.path.exists(expected_bak2_src2_dir) + assert os.path.exists(expected_bak2_src2_file) + +@pytest.mark.skip(reason="Requires a fake database to be setup") +class TestMysqlBackup(object): + """ The mySQL dump tests require the machine on which they are +running to have a mysql database running with a user called +'dumper' and a password set to 'not_a_real_password'. + +In production, a proper password should be set on the server and in the config +file. DO NOT USE THE ABOVE IN PRODUCTION + +To change the password: +UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; +""" + + def test_mysql_dump_is_ok(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_mysqldump is True + + date = bckup.get_date() + + expected_dump1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_dump.sql') + expected_dump2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_dump.sql') + assert not os.path.exists(expected_dump1) + assert not os.path.exists(expected_dump2) + result = backy._run_mysqldump() + assert os.path.exists(expected_dump1) + assert os.path.exists(expected_dump2) + + assert result == 0 + + def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'incorrect_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + with pytest.raises(errs.SubprocessError): + backy._run_mysqldump() + + +# class TestRunningAll(object): + +# def test_running_copy_tar_and_rsyc(self): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + + +class TestSimpleLogWriting(object): + + def test_log_writing_success(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + backy.run_backup_processes() + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + # Read file + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + def test_log_writes_only_attempted_if_failure_encountered(self, tmpdir): + + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + # fails due to missing source file in copy + with pytest.raises(errs.ValidationError): + backy.run_backup_processes() + + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] Index: branches/eam_branches/ipp-20191011/backups/confluence_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/confluence_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/confluence_backup.py (revision 41170) @@ -0,0 +1,107 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') + +CONFLUENCE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class ConfluenceBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_confluence_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + confluence_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(confluence_format) + return datetime.date.today().strftime(confluence_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For confluence backup there should only be one source: + source = self.copy_dict['sources'][0] + + confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_xml_filepaths = glob.glob(confluence_wildcard_search) + if len(confluence_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No confluence backups found of form: {confluence_wildcard_search}") + + for jxf in confluence_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + cb = ConfluenceBackup(config, schema) + cb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/confluence_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/confluence_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/confluence_backup_test.py (revision 41170) @@ -0,0 +1,366 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backup as bckup +import confluence_backup as cbck_mod + +from confluence_backup import ConfluenceBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after --exclude exclude_me_if_you_like' + , 'sub-dirs' : 'conf_subdir1, conf_subdir2' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_confluence_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +ConfluenceBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + confluence_backup_path = os.path.join(tmpdir, "confluence_backup_path") + confluence_attachments_path = os.path.join(tmpdir, "confluence_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(confluence_backup_path) + os.makedirs(confluence_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(confluence_backup_path) + assert os.path.exists(confluence_attachments_path) + + # Create backup files that will be copied + confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip" + confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) + Path(confluence_backup_filepath).touch() + assert os.path.exists(confluence_backup_filepath) + + # Create some files that will be tar'd + confluence_attachment_1_name = "attachment_1.zip" + confluence_attachment_2_name = "attachment_2.txt" + confluence_attachment_1_filepath = os.path.join(confluence_attachments_path, confluence_attachment_1_name) + confluence_attachment_2_filepath = os.path.join(confluence_attachments_path, confluence_attachment_2_name) + Path(confluence_attachment_1_filepath).touch() + Path(confluence_attachment_2_filepath).touch() + assert os.path.exists(confluence_attachment_1_filepath) + assert os.path.exists(confluence_attachment_2_filepath) + + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + exluded_file = os.path.join(rsync_source_dir1, "exclude_me_if_you_like") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + Path(exluded_file).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + assert os.path.exists(exluded_file) + + +class TestArguments(object): + + def assert_defaults(self, jb: ConfluenceBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/confluence_backups' + , '/data/ippops3.0/backups/confluence_backups' + , '/data/ippops5.0/backups/confluence_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/backups'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/attachments'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = cbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + ConfluenceBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = ConfluenceBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + ConfluenceBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + cbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'conf.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, conf_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + cbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_confluence_backup_date_format(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + # return_value = cb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + cb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_confluence_copy(self, tmpdir): + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + cb = ConfluenceBackup(config_file=config) + + today = ConfluenceBackup.get_confluence_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"confluence-xml-backup-{today}.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = cb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_confluence_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + confluence_tar_name = f"{today}_confluence.tar.gz" + expected_confluence_host_tar = os.path.join(tmpdir, "host_backup_path", confluence_tar_name) + expected_confluence_bak1_tar = os.path.join(tmpdir, "backup_1_path", confluence_tar_name) + expected_confluence_bak2_tar = os.path.join(tmpdir, "backup_2_path", confluence_tar_name) + assert not os.path.exists(expected_confluence_host_tar) + assert not os.path.exists(expected_confluence_bak1_tar) + assert not os.path.exists(expected_confluence_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_confluence_host_tar) + assert os.path.exists(expected_confluence_bak1_tar) + assert os.path.exists(expected_confluence_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_confluence_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + + cb = ConfluenceBackup(config_file=config) + + # Expected resulting rsync directories + expected_confluence_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "conf_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_confluence_host_rsync1) + assert not os.path.exists(expected_confluence_bak1_rsync1) + assert not os.path.exists(expected_confluence_bak2_rsync1) + assert not os.path.exists(expected_confluence_host_rsync2) + assert not os.path.exists(expected_confluence_bak1_rsync2) + assert not os.path.exists(expected_confluence_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_confluence_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_confluence_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_confluence_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_confluence_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_confluence_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_confluence_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + an_excluded_file = os.path.join(expected_confluence_host_rsync1, "exclude_me_if_you_like") + assert not os.path.exists(an_excluded_file) + + rsync_result = cb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_confluence_host_rsync1) + assert os.path.exists(expected_confluence_bak1_rsync1) + assert os.path.exists(expected_confluence_bak2_rsync1) + assert os.path.exists(expected_confluence_host_rsync2) + assert os.path.exists(expected_confluence_bak1_rsync2) + assert os.path.exists(expected_confluence_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) + assert not os.path.exists(an_excluded_file) Index: branches/eam_branches/ipp-20191011/backups/deploy_backups.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/deploy_backups.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/deploy_backups.py (revision 41170) @@ -0,0 +1,63 @@ +import argparse +import sys + +import utils.subprocess_utils as sub_utils + +DEFAULT_DEPLOYMENT = \ + { "jira_backup.py" : "ippops4" + , "confluence_backup.py" : "ippops4" + , "svn_backup.py" : "ippops4" + , "mailman_backup.py" : "ippops3" + , "ippops_homedir_backup.py" : "ippops3" + , "website_backup.py" : "ippops3" + } + +ADDITIONAL_FILES = \ + [ './utils' + , 'backup.py' + ] + +PREFIX = "/export/" +SUFFIX = ".0/backups/" + + +def deploy(user=None): + + for file in DEFAULT_DEPLOYMENT: + default_machine = DEFAULT_DEPLOYMENT[file] + + chosen_machine = input(f"Enter machine to copy {file} to [default={default_machine}; ENTER to accept]:") + chosen_machine.strip('\n').strip(' ') + if chosen_machine == '': + chosen_machine = default_machine + + copy_file_and_utils(file, chosen_machine, user) + + +def copy_file_and_utils(file, machine, user): + # copy all the utils across too + user_at = '' if user is None else f'{user}@' + target = f"{user_at}{machine}.ifa.hawaii.edu:{PREFIX}{machine}{SUFFIX}" + args = ['rsync', '-a', '--progress', file, ] + ADDITIONAL_FILES + [target] + print(f"Performing: {' '.join(args)}") + sub_utils.simple_unix_wrapper(args) + + +def main(): + args = parse_args(sys.argv[1:]) + args_dict = vars(args) + deploy(**args_dict) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='Performs deploys backup files to desired locations') + parser.add_argument('--user' + , nargs='?' + , help='specify the user to transfer files as' + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup.py (revision 41170) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') + +IPPOPS_HOMEDIR_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class IPPOpsHomedirBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + ohdb = IPPOpsHomedirBackup(config, schema) + ohdb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/ippops_homedir_backup_test.py (revision 41170) @@ -0,0 +1,133 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import ippops_homedir_backup as ihbck_mod +from ippops_homedir_backup import IPPOpsHomedirBackup +from utils.config_parse_helper import ConfigSchema + +IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/ippops_homedir_test.config') + + +class TestArguments(object): + + def assert_defaults(self, ihb: IPPOpsHomedirBackup): + + assert ihb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/homedir_backups' + , '/data/ippops4.0/backups/homedir_backups' + , '/data/ippops5.0/backups/homedir_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert ihb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/home/panstarrs'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) + self.assert_defaults(ihb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + ihbck_mod.main(config, schema) + assert True is True + + +class TestIppopsHomedirRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'home')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "home") + subdir1 = os.path.join(homedir, "some_user") + subdir2 = os.path.join(homedir, "some_other_user") + file1 = os.path.join(subdir1, "file_in_subdir1") + file2 = os.path.join(subdir2, "file_in_subdir2") + os.makedirs(homedir) + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(file1).touch() + Path(file2).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + ihb = IPPOpsHomedirBackup(config, schema) + + expected_sub1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user') + expected_sub1_file1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user') + expected_sub2_file2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user/file_in_subdir2') + expected_sub1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user') + expected_sub1_file1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user') + expected_sub2_file2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user/file_in_subdir2') + assert not os.path.exists(expected_sub1_backup1) + assert not os.path.exists(expected_sub1_file1_backup1) + assert not os.path.exists(expected_sub2_backup1) + assert not os.path.exists(expected_sub2_file2_backup1) + assert not os.path.exists(expected_sub1_backup2) + assert not os.path.exists(expected_sub1_file1_backup2) + assert not os.path.exists(expected_sub2_backup2) + assert not os.path.exists(expected_sub2_file2_backup2) + ihb._run_rsync() + assert os.path.exists(expected_sub1_backup1) + assert os.path.exists(expected_sub1_file1_backup1) + assert os.path.exists(expected_sub2_backup1) + assert os.path.exists(expected_sub2_file2_backup1) + assert os.path.exists(expected_sub1_backup2) + assert os.path.exists(expected_sub1_file1_backup2) + assert os.path.exists(expected_sub2_backup2) + assert os.path.exists(expected_sub2_file2_backup2) Index: branches/eam_branches/ipp-20191011/backups/jira_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/jira_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/jira_backup.py (revision 41170) @@ -0,0 +1,107 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') + +JIRA_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class JiraBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_jira_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For jira backup there should only be one source: + source = self.copy_dict['sources'][0] + + jira_wildcard_search = f"{source}/{JiraBackup.get_jira_backup_date_format()}*" + jira_xml_filepaths = glob.glob(jira_wildcard_search) + if len(jira_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No jira backups found of form: {jira_wildcard_search}") + + for jxf in jira_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + jb = JiraBackup(config, schema) + jb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/jira_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/jira_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/jira_backup_test.py (revision 41170) @@ -0,0 +1,361 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.backup as bckup +import jira_backup as jbck_mod +from jira_backup import JiraBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after' + , 'sub-dirs' : 'jira_subdir1, jira_subdir2' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_jira_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +JiraBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + jira_backup_path = os.path.join(tmpdir, "jira_backup_path") + jira_attachments_path = os.path.join(tmpdir, "jira_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(jira_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(jira_attachments_path) + + # Create backup files that will be copied + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + # Create some files that will be tar'd + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + + +class TestArguments(object): + + def assert_defaults(self, jb: JiraBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/jira_backups' + , '/data/ippops3.0/backups/jira_backups' + , '/data/ippops5.0/backups/jira_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = jbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + JiraBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = JiraBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + JiraBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + jbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, jira_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_jira_backup_date_format(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_jira_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_jira_copy(self, tmpdir): + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = JiraBackup.get_jira_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_jira_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_jira_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_jira_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + assert jb.should_run_copy is True + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_jira_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = f"{today}_jira.tar.gz" + expected_jira_host_tar = os.path.join(tmpdir, "host_backup_path", jira_tar_name) + expected_jira_bak1_tar = os.path.join(tmpdir, "backup_1_path", jira_tar_name) + expected_jira_bak2_tar = os.path.join(tmpdir, "backup_2_path", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_jira_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + + jb = JiraBackup(config_file=config) + + # Expected resulting rsync directories + expected_jira_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "jira_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_jira_host_rsync1) + assert not os.path.exists(expected_jira_bak1_rsync1) + assert not os.path.exists(expected_jira_bak2_rsync1) + assert not os.path.exists(expected_jira_host_rsync2) + assert not os.path.exists(expected_jira_bak1_rsync2) + assert not os.path.exists(expected_jira_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_jira_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_jira_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_jira_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_jira_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_jira_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_jira_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + + rsync_result = jb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_jira_host_rsync1) + assert os.path.exists(expected_jira_bak1_rsync1) + assert os.path.exists(expected_jira_bak2_rsync1) + assert os.path.exists(expected_jira_host_rsync2) + assert os.path.exists(expected_jira_bak1_rsync2) + assert os.path.exists(expected_jira_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) Index: branches/eam_branches/ipp-20191011/backups/mailman_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/mailman_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/mailman_backup.py (revision 41170) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') + +MAILMAN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class MailmanBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + mb = MailmanBackup(config, schema) + mb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/mailman_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/mailman_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/mailman_backup_test.py (revision 41170) @@ -0,0 +1,119 @@ +import configparser +import os +from pathlib import Path + +import mailman_backup as mailbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from mailman_backup import MailmanBackup +from utils.config_parse_helper import ConfigSchema + + +MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/mailman_test.config') + + +class TestArguments(object): + + def assert_defaults(self, mb: MailmanBackup): + + assert mb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/mailman_backups' + , '/data/ippops4.0/backups/mailman_backups' + , '/data/ippops5.0/backups/mailman_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert mb.rsync_dict == \ + { 'sources' : ['/var/lib/mailman'] + , 'additional_args' : ['-a', '--delete-after', '--copy-links'] + } + + def test_load_default_config_file(self): + mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) + self.assert_defaults(mb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + mailbck_mod.main(config, schema) + assert True is True + + +class TestMailmanRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'mailman')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + mailman_root = os.path.join(tmpdir, "mailman") + subdir = os.path.join(mailman_root, "some_dir") + file = os.path.join(subdir, "file_in_subdir") + os.makedirs(mailman_root) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + mb = MailmanBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir') + expected1file = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir') + expected2file = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + mb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) Index: branches/eam_branches/ipp-20191011/backups/svn_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/svn_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/svn_backup.py (revision 41170) @@ -0,0 +1,45 @@ +import os.path as path + +from utils.config_parse_helper import SpecialSchemaProcessing +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') + +SVN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class SvnBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + sb = SvnBackup(config, schema) + sb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/svn_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/svn_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/svn_backup_test.py (revision 41170) @@ -0,0 +1,193 @@ +import configparser +import os +from pathlib import Path + +import svn_backup as svnbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from svn_backup import SvnBackup +from utils.config_parse_helper import ConfigSchema + +SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') + + +class TestArguments(object): + + def assert_defaults(self, sb: SvnBackup): + + assert sb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/svn_backups' + , '/data/ippops3.0/backups/svn_backups' + , '/data/ippops5.0/backups/svn_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert sb.copy_dict == \ + { 'sources' : ['/etc/apache2/mods-enabled/dav_svn.conf'] + , 'additional_args' : ['-v'] + } + assert sb.rsync_dict == \ + { 'sources' : ['/export/ippops4.0/svnroot'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + sb = SvnBackup(config_file=SVN_TEST_CONFIG) + self.assert_defaults(sb) + + +class TestSvnCopy(object): + """Simple inherit of base class Backup""" + + # Setup config + def test_copy(self, tmpdir): + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'apache2/mods-enabled/dav_svn.conf')}" + , 'additional_args' : "-v" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + apache_folder = os.path.join(tmpdir, "apache2") + subdir = os.path.join(apache_folder, "mods-enabled") + file = os.path.join(subdir, "dav_svn.conf") + os.makedirs(apache_folder) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(COPY_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'dav_svn.conf') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'dav_svn.conf') + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_file_bck2) + sb._run_copy() + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_file_bck2) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + svn_conf_file = "dav_svn.conf" + config['COPY'] = \ + { 'sources' : f'{copy_source}/{svn_conf_file}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, svn_conf_file)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + svnbck_mod.main(config, schema) + assert True is True + + +class TestSvnRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'svnroot')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + svnroot = os.path.join(tmpdir, "svnroot") + subdir = os.path.join(svnroot, "some_dir") + file1 = os.path.join(svnroot, "file_at_root_level") + file2 = os.path.join(subdir, "file_in_subdir") + os.makedirs(svnroot) + os.makedirs(subdir) + Path(file1).touch() + Path(file2).touch() + + # File that will be removed because of --delete-after + os.makedirs(os.path.join(backup_1_path, "svnroot")) + file_that_already_existed = os.path.join(backup_1_path, "svnroot", "file_to_disappear") + Path(file_that_already_existed).touch() + assert os.path.exists(file_that_already_existed) + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir') + expected1file1 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'file_at_root_level') + expected1file2 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir') + expected2file1 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'file_at_root_level') + expected2file2 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file1) + assert not os.path.exists(expected1file2) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file1) + assert not os.path.exists(expected2file2) + assert os.path.exists(file_that_already_existed) + sb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file1) + assert os.path.exists(expected1file2) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file1) + assert os.path.exists(expected2file2) + # file that does not exist in the source, is removed from the target + assert not os.path.exists(file_that_already_existed) Index: branches/eam_branches/ipp-20191011/backups/testing/backup_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/backup_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/backup_test.config (revision 41170) @@ -0,0 +1,34 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = hostname_of_computer +verbose = True + +[COPY] +sources = /some_dir_to_copy_from/, + /some_dir/and_specific_file.jpeg +additional_args = -v + +[TAR] +sources = /this_dir/, + /another_dir/, + /and_a_couple_files/subdir/file1.txt, + /and_a_couple_files/subdir/file2.png, + /and_a_couple_files/subdir/file3.pdf +additional_args = --verbose -z -p +target_filename = cool_tar +prefix_date = True + + +[RSYNC] +sources = /dir/to/sync +additional_args = --progress -r + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +additional_args = +target_filename = itsa_mysqldump +prefix_date = True Index: branches/eam_branches/ipp-20191011/backups/testing/confluence_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/confluence_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/confluence_test.config (revision 41170) @@ -0,0 +1,29 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/confluence_backups, + /data/ippops3.0/backups/confluence_backups, + /data/ippops5.0/backups/confluence_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/backups +additional_args = -v +conf_prefix = confluence-xml-backup- + +[TAR] +sources = /var/atlassian/application-data/confluence/attachments +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = conf-install, conf-home + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: branches/eam_branches/ipp-20191011/backups/testing/ippops_homedir_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/ippops_homedir_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/ippops_homedir_test.config (revision 41170) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/homedir_backups, + /data/ippops4.0/backups/homedir_backups, + /data/ippops5.0/backups/homedir_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/home/panstarrs +additional_args = -a --delete-after Index: branches/eam_branches/ipp-20191011/backups/testing/jira_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/jira_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/jira_test.config (revision 41170) @@ -0,0 +1,28 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/jira_backups, + /data/ippops3.0/backups/jira_backups, + /data/ippops5.0/backups/jira_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = jira-install, jira-home + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: branches/eam_branches/ipp-20191011/backups/testing/mailman_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/mailman_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/mailman_test.config (revision 41170) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/mailman_backups, + /data/ippops4.0/backups/mailman_backups, + /data/ippops5.0/backups/mailman_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /var/lib/mailman +additional_args = -a --delete-after --copy-links Index: branches/eam_branches/ipp-20191011/backups/testing/svn_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/svn_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/svn_test.config (revision 41170) @@ -0,0 +1,14 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/svn_backups, + /data/ippops3.0/backups/svn_backups, + /data/ippops5.0/backups/svn_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /etc/apache2/mods-enabled/dav_svn.conf +additional_args = -v + +[RSYNC] +sources = /export/ippops4.0/svnroot +additional_args = -a --delete-after Index: branches/eam_branches/ipp-20191011/backups/testing/website_test.config =================================================================== --- branches/eam_branches/ipp-20191011/backups/testing/website_test.config (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/testing/website_test.config (revision 41170) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/website_backups, + /data/ippops4.0/backups/website_backups, + /data/ippops5.0/backups/website_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/www +additional_args = -a --delete-after Index: branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper.py (revision 41170) @@ -0,0 +1,180 @@ +from enum import Enum +from configparser import ConfigParser + +from utils.errors import ConfigError, ValidationError + + +class SpecialSchemaProcessing(Enum): + """These enums define how processing should be applied to a string to +separate it out into list. + """ + commma_separated = 1 + space_separated = 2 + + +class ConfigSchemaLine(): + """Defines a config line by what the key is, if it is required or optional, +the type that the value should be processed as, and any special processing required +""" + + def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None): + self.key = key + self.required = required + self.expected_type = expected_type + if expected_type is list: + if type(special_processing) is not SpecialSchemaProcessing: + raise ValidationError("ValidationError: special_processing must be defined if implementing a list in config") + self.special_processing = special_processing + + def __str__(self): + return (f'<{self.__class__.__name__} - ' + f'Key:{self.key}, ' + f'Required:{self.required}, ' + f'Type:{self.expected_type}, ' + f'Processing: {self.special_processing}' + f'>') + + +class ConfigSchemaSection(): + """Defines a schema to be used. Consists of a single section name and +multiple config lines""" + + def __init__(self, section_name, options: [ConfigSchemaLine]): + self.section_name = section_name + self.options = {} + self._process_options(options) + + def _process_options(self, options): + if options is None: + return + if type(options) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} options must be '[ConfigSchemaLine]' or 'None'") + if len(options) == 0: + return + for o in options: + self.add_line(o) + + def add_line(self, line: ConfigSchemaLine): + self.options[line.key] = line + + def key_in_options(self, key: str) -> bool: + return key in self.options.keys() + + def get_schema_line(self, key: str) -> ConfigSchemaLine: + if not self.key_in_options(key): + raise KeyError(f"KeyError: '{key}' is not present in schema section") + return self.options[key] + + def __str__(self): + options_strs = [ o.__str__() for o in self.options] + return (f'{self.__class__.__name__} named "{self.section_name}" with {options_strs}') + + +class ConfigSchema(): + + def __init__(self, sections: [ConfigSchemaSection]): + self.sections = {} + self._process_sections(sections) + + def _process_sections(self, sections): + if sections is None: + return + if type(sections) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} sections must be '[ConfigSchemaSection]' or 'None'") + if len(sections) == 0: + return + for sec in sections: + self.add_section(sec) + + def add_section(self, section: ConfigSchemaSection): + self.sections[section.section_name] = section + + def section_exists(self, section_name: str) -> bool: + return section_name in self.sections.keys() + + def get_schema_section(self, section_name: str) -> ConfigSchemaSection: + if not self.section_exists(section_name): + raise KeyError(f"KeyError: '{section_name}' is not present in schema") + return self.sections[section_name] + + def __str__(self): + section_strs = [ o.__str__() for o in self.sections] + return (f'{self.__class__.__name__} with {section_strs}') + + +def _split_string_to_list(items: str, separator=',') -> [str]: + split_items = items.split(separator) + processed_items = [ i.strip(' ').strip('\n') for i in split_items] + return processed_items + + +def _parse_config_section(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: + """For a given config it will extract all values defined by the schema +and return them as a dictionary of the keys and the assoicated value as the +type defined by the schema +""" + parsed_items = {} + + section = schema.section_name + for csl_key in schema.options: + csl = schema.get_schema_line(csl_key) + + # Parses each line + if config.has_option(section, csl_key): + if csl.expected_type is bool: + parsed_items[csl.key] = config.getboolean(section, csl.key) + elif csl.expected_type is int: + parsed_items[csl.key] = config.getint(section, csl.key) + elif csl.expected_type is float: + parsed_items[csl.key] = config.getfloat(section, csl.key) + elif csl.expected_type is str: + parsed_items[csl.key] = config[section][csl.key] + elif csl.expected_type is list: + if csl.special_processing == SpecialSchemaProcessing.commma_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ',') + parsed_items[csl.key] = splitup + elif csl.special_processing == SpecialSchemaProcessing.space_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ' ') + parsed_items[csl.key] = splitup + else: + + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") + else: + raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") + + else: + if csl.required: + raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config") + + return parsed_items + + +def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: + parsed_items = {} + + for css_name in schema.sections: + css = schema.get_schema_section(css_name) + + # if config.has_section(css_name): + parsed_section = _parse_config_section(config, css) + parsed_items[css_name] = parsed_section + # else: + # raise ConfigError(config, f"ConfigError: Expected section '{css_name}' from schema not present in config") + + return parsed_items + + +class ParsedConfig(): + + def __init__(self, config: ConfigParser, schema: ConfigSchema): + self._section_dict = _parse_config(config, schema) + + def has_section(self, section_name): + return section_name in self._section_dict + + def get_section(self, section_name): + if self.has_section(section_name): + return self._section_dict[section_name] + return None Index: branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/config_parse_helper_test.py (revision 41170) @@ -0,0 +1,295 @@ +import pytest + +import utils.config_parse_helper as helper +from utils.errors import ConfigError, ValidationError + + +class TestSpecialSchemaProcessingEnum(): + + def test_all_enums(self): + cs = helper.SpecialSchemaProcessing.commma_separated + assert cs.name == "commma_separated" + assert cs.value == 1 + + ss = helper.SpecialSchemaProcessing.space_separated + assert ss.name == "space_separated" + assert ss.value == 2 + + assert len(list(helper.SpecialSchemaProcessing)) == 2 + + +class TestConfigSchemaLine(): + + def test_initialising_simple_schema_lines(self): + csl = helper.ConfigSchemaLine('key1', True, bool) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is bool + assert csl.special_processing is None + + def test_initialising_list_in_schema(self): + csl = helper.ConfigSchemaLine('key1', True, list, helper.SpecialSchemaProcessing.commma_separated) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is list + assert csl.special_processing is helper.SpecialSchemaProcessing.commma_separated + + def test_init_list_without_special_processing_fails(self): + not_a_valid_enum = 3 + with pytest.raises(ValidationError): + helper.ConfigSchemaLine('key1', True, list, not_a_valid_enum) + + +class TestConfigSchemaSection(): + + def test_initalising_schema(self): + csl1 = helper.ConfigSchemaLine(key='is_cool', required=False, expected_type=bool) + csl2 = helper.ConfigSchemaLine('fav_word', True, str) + csl3 = helper.ConfigSchemaLine('some_float', False, float) + + schema = helper.ConfigSchemaSection( 'section_name', [csl1, csl2, csl3] ) + assert schema.options == { 'is_cool' : csl1 + , 'fav_word' : csl2 + , 'some_float' : csl3 + } + + def test_empty_schema_section_ok(self): + helper.ConfigSchemaSection("ok_with_none", None) + helper.ConfigSchemaSection("ok_with_empty", []) + + def test_schema_section_fails_if_options_are_not_a_list(self): + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("fails_if_not_a_list", True) + + csl = helper.ConfigSchemaLine('key1', True, bool) + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("even_fails_on_a_single_csl", csl) + + def test_adding_to_schema(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl1 = helper.ConfigSchemaLine('key1', True, bool) + csl2 = helper.ConfigSchemaLine('key2', False, int) + csl3 = helper.ConfigSchemaLine('key3', True, str) + config.add_line(csl1) + assert config.options == { 'key1': csl1 } + + config.add_line(csl2) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + } + + config.add_line(csl3) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + , 'key3': csl3 + } + + def test_adding_a_duplicate_key_to_schema_replaces_previous(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl_ori = helper.ConfigSchemaLine('key', True, bool) + csl_alt = helper.ConfigSchemaLine('key', False, int) + + config.add_line(csl_ori) + assert config.options == { 'key': csl_ori } + + config.add_line(csl_alt) + assert config.options == { 'key': csl_alt } + + +class TestStringSplitting(): + + def test_split_comma_separated_list(self): + before_parse = "/hello/sir/, /glorius/filename,/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected + + def test_removal_of_newlines_too(self): + before_parse = "\n/hello/sir/, /glorius/filename\n,\n/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected +# def _split_string_to_list(items: str, separator=',') -> [str]: + + def test_space_separated_split(self): + before_parse = "--verbose --progress -a -f -z" + expected = [ "--verbose" + , "--progress" + , "-a" + , "-f" + , "-z" + ] + result = helper._split_string_to_list(before_parse, ' ') + assert result == expected + + +class TestParsingConfig(): + + def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser: + config = helper.ConfigParser() + config[section_name] = \ + { 'inty' : '3' + , 'floaty' : '3.141' + , 'mr_bool' : 'True' + , 'stringer' : "bell" + , 'listo_comma' : "i,am,comma,separated" + , 'listo_space' : "yo it's time for space!" + } + return config + + def test_parsing_fails_with_missing_required_key(self): + config = helper.ConfigParser() + config['DEFAULT'] = { 'not_the_required_key' : False } + + csl1 = helper.ConfigSchemaLine('Required_key', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_schema_section_with_only_optional_values_is_ok(self): + csl1 = helper.ConfigSchemaLine('not_required', False, bool) + csl2 = helper.ConfigSchemaLine('not_required_either', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1, csl2] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == {} + + def test_parsing_schema_section_with_ints(self): + csl1 = helper.ConfigSchemaLine('inty', False, int) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'inty' : 3} + + def test_parsing_schema_section_with_floats(self): + csl1 = helper.ConfigSchemaLine('floaty', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + + def test_parsing_schema_section_with_bools(self): + csl1 = helper.ConfigSchemaLine('mr_bool', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'mr_bool' : True } + + def test_parsing_schema_section_with_strs(self): + csl1 = helper.ConfigSchemaLine('stringer', True, str) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'stringer' : 'bell' } + + def test_parsing_schema_section_with_lists_comma_separated(self): + csl1 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_comma' : [ "i" + , "am" + , "comma" + , "separated" + ] + } + + def test_parsing_schema_section_with_lists_space_separated(self): + csl1 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_space' : [ "yo" + , "it's" + , "time" + , "for" + , "space!" + ] + } + + def test_parsing_as_different_types_always_ok_for_str_and_list(self): + config = self.make_simple_config() + + csl_float = helper.ConfigSchemaLine('floaty', True, float) + csl_int = helper.ConfigSchemaLine('floaty', True, int) + csl_bool = helper.ConfigSchemaLine('floaty', True, bool) + csl_str = helper.ConfigSchemaLine('floaty', True, str) + csl_list = helper.ConfigSchemaLine('floaty', True, list, + helper.SpecialSchemaProcessing.commma_separated) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_float]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + assert type(result['floaty']) is float + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_int]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_bool]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_str]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : '3.141' } + assert type(result['floaty']) is str + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_list]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : ['3.141'] } + assert type(result['floaty']) is list + + def test_parsing_schema_line_fails_with_unknown_type(self): + csl1 = helper.ConfigSchemaLine('floaty', True, dict) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_all_special_processing_options_are_handled(self): + config = self.make_simple_config() + config['DEFAULT'] = {'enum_checker' : "comma, and spaces"} + + for e in helper.SpecialSchemaProcessing: + csl = helper.ConfigSchemaLine('enum_checker', False, list, e) + schema = helper.ConfigSchemaSection('DEFAULT', [csl]) + + result = helper._parse_config_section(config, schema) + assert type(result['enum_checker']) is list + + def test_parsing_all(self): + config = self.make_simple_config('FULL_TEST') + + csl1 = helper.ConfigSchemaLine('inty', False, int) + csl2 = helper.ConfigSchemaLine('floaty', False, float) + csl3 = helper.ConfigSchemaLine('mr_bool', True, bool) + csl4 = helper.ConfigSchemaLine('stringer', True, str) + csl5 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + csl6 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) + + result = helper._parse_config_section(config, schema) + assert result == \ + { 'inty' : 3 + , 'floaty' : 3.141 + , 'mr_bool' : True + , 'stringer' : "bell" + , 'listo_comma' : ["i", "am", "comma", "separated"] + , 'listo_space' : ["yo", "it's", "time", "for", "space!"] + } Index: branches/eam_branches/ipp-20191011/backups/utils/errors.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/errors.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/errors.py (revision 41170) @@ -0,0 +1,16 @@ + + +class ConfigError(Exception): + """Error in loading the config file""" + + def __init__(self, config_item, message): + self.config_item = config_item + self.message = message + + +class ValidationError(Exception): + """Validation errors in utils functions. Raised when arguments are not valid""" + + +class SubprocessError(Exception): + """Errors in Popen stderr subprocess in subprocess_utils""" Index: branches/eam_branches/ipp-20191011/backups/utils/generic_utils.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/generic_utils.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/generic_utils.py (revision 41170) @@ -0,0 +1,124 @@ +import glob +import os.path as path +import shutil +from distutils.dir_util import copy_tree + +from .subprocess_utils import simple_unix_wrapper +from .errors import ValidationError + + +def copy_to_dir(source_file: str, destination_path: str, verbose: bool=False): + """ Copies a single file into the destination directory. If a directory is + given as a source then all the contents of the directory will be copied (the + directory itself will not be). +""" + + if source_file is None or destination_path is None: + raise ValidationError("Error: source for copy cannot be None type") + if destination_path is None: + raise ValidationError("Error: copy destination cannot be None type") + if not path.exists(source_file): + raise ValidationError(f"Error: Copy Failure: source does not exist: {source_file}") + if not path.exists(destination_path) or not path.isdir(destination_path): + print("Error: Copy Failure: destination path does not exist: ", destination_path) + raise ValidationError(f"Error: Copy Failure: destination path does not exist: {destination_path}") + + try: + if verbose: + print(f"Copying: {source_file} to {destination_path}") + if path.isdir(source_file): + copy_tree(source_file, destination_path) + else: + shutil.copy2(source_file, destination_path) + except IOError as e: + msg = f"Error: copy failure.\n copying: {source_file}\n to: {destination_path}" + raise IOError(msg) from e + return 0 + + +def _list_checks(sources: [str], targets: [str]): + if sources is None or targets is None: + raise ValidationError("Error: source for copy cannot be None type") + if targets is None: + raise ValidationError("Error: copy destination cannot be None type") + if type(sources) is not list or len(sources) == 0: + raise ValidationError("Error: sources must be given as a list with at least one entry") + if type(targets) is not list or len(targets) == 0: + raise ValidationError("Error: targets must be given as a list with at least one entry") + + +def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False): + _list_checks() + + for i in range(0, len(source_paths)): + + if not path.exists(destination_paths[i]): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", destination_paths[i]) + continue + if not path.isdir(destination_paths[i]): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i]) + continue + + result = copy_to_dir(source_paths[i], destination_paths[i], verbose) + if result != 0: + return result + + +def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): + """ Copies the sources to the destination paths. + +This is a generic function that just wraps shutil.copy2""" + _list_checks(source_paths, destination_paths) + + for dpath in destination_paths: + if not path.exists(dpath): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", dpath) + continue + if not path.isdir(dpath): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", ) + continue + + for spath in source_paths: + result = copy_to_dir(spath, dpath, verbose) + if result != 0: + return result + + return 0 + + +def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + _list_checks(source_dirs, destination_dirs) + + if len(source_dirs) != len(destination_dirs): + raise ValidationError("Error: source and target paths are not equal; aborting file movement") + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + if verbose: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0 Index: branches/eam_branches/ipp-20191011/backups/utils/generic_utils_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/generic_utils_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/generic_utils_test.py (revision 41170) @@ -0,0 +1,224 @@ +import os +import pytest + +from .generic_utils import * +from .errors import ValidationError +from pathlib import Path + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyingToDir(object): + + def test_copy_fails_with_none_args(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir(None, tmpdir) + + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "file.txt") + copy_to_dir(real_file, None) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir("not a real file", tmpdir) + + def test_copy_fails_if_destination_is_not_real(self, tmpdir): + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "real_file.txt") + copy_to_dir(real_file, "not a real directory") + + def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): + + real_file = make_a_real_file(tmpdir, "not_a_directory.txt") + not_a_directory = real_file + assert os.path.exists(not_a_directory) + assert not os.path.isdir(not_a_directory) + + with pytest.raises(ValidationError): + copy_to_dir(real_file, not_a_directory) + + def test_copy_to_dir(self, tmpdir): + + # Setup source file and target dir + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + file_name = "fun_file.txt" + file_to_copy = os.path.join(tmpdir, file_name) + Path(file_to_copy).touch() + assert os.path.exists(file_to_copy) + + # copied file does not exist yet + expected_copy = os.path.join(tmpdir, "destination", file_name) + assert not os.path.exists(expected_copy) + + # run copy function and confirm copy exists (and original) + result = copy_to_dir(file_to_copy, destination_dir) + assert result == 0 + assert os.path.exists(file_to_copy) + assert os.path.exists(expected_copy) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + filepath_a = make_a_real_file(subdir, filename_a) + filepath_b = make_a_real_file(subsubdir, filename_b) + + print(filepath_a) + print(filepath_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + print(expected_filepath_a) + print(expected_filepath_b) + print(os.path.exists(expected_filepath_a)) + if os.path.exists(expected_filepath_a): + print("apple") + else: + print("banana") + print(os.path.exists(expected_filepath_b)) + + os.path.commonpath + + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # Provide only the original tmpdir to the copying function + copy_to_dir(subdir, destination_dir, True) + + # assert that the subdirs and files all exist + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + +class TestMultiCopy(object): + + def test_copy_fails_if_args_are_none(self, tmpdir): + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, tmpdir) + + def test_copy_requires_args_to_be_in_list_format(self, tmpdir): + + not_a_list = "not a list" + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + + tmpfile = make_a_real_file(tmpdir, "a_file.txt") + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + + def test_copy_is_fine_with_single_items(self, tmpdir): + + tmpfile = make_a_real_file(tmpdir, "single_file.txt") + destination = os.path.join(tmpdir, "destination") + os.makedirs(destination) + expected_file = os.path.join(tmpdir, "destination", "single_file.txt") + assert not os.path.exists(expected_file) + + result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + assert result == 0 + assert os.path.exists(expected_file) + + def test_copy_multiple_items_to_multiple_dirs(self, tmpdir): + + file_a = make_a_real_file(tmpdir, "single_file.txt") + file_b = make_a_real_file(tmpdir, "single_file.txt") + file_c = make_a_real_file(tmpdir, "single_file.txt") + source_files = [ file_a + , file_b + , file_c + ] + + destination_1 = os.path.join(tmpdir, "dest_1") + destination_2 = os.path.join(tmpdir, "dest_2") + destination_3 = os.path.join(tmpdir, "dest_3") + dest_paths = [ destination_1 + , destination_2 + , destination_3 + ] + os.makedirs(destination_1) + os.makedirs(destination_2) + os.makedirs(destination_3) + + expected_files = [] + for dp in dest_paths: + for sf in source_files: + expected_files.append( os.path.join(dp, os.path.basename(sf))) + assert not os.path.exists(expected_files[-1]) + assert len(expected_files) == 9 + + result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + assert result == 0 + for ef in expected_files: + assert os.path.exists(ef) + + def test_copying_is_recursive(self, tmpdir): + + # make some subdirs, with sub-subdirs, and files within them + subdir_a = os.path.join(tmpdir, "subdir_a") + subsubdir_aa = os.path.join(subdir_a, "subsubdir_aa") + subsubdir_ab = os.path.join(subdir_a, "subsubdir_ab") + subdir_b = os.path.join(tmpdir, "subdir_b") + subsubdir_ba = os.path.join(subdir_b, "subsubdir_ba") + subsubdir_bb = os.path.join(subdir_b, "subsubdir_bb") + os.makedirs(subsubdir_aa) + os.makedirs(subsubdir_ab) + os.makedirs(subsubdir_ba) + os.makedirs(subsubdir_bb) + make_a_real_file(subdir_a, "file_a.txt") + make_a_real_file(subsubdir_aa, "file_aa.txt") + make_a_real_file(subsubdir_ab, "file_ab.txt") + make_a_real_file(subdir_b, "file_b.txt") + make_a_real_file(subsubdir_ba, "file_ba.txt") + make_a_real_file(subsubdir_bb, "file_bb.txt") + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir_a", "file_a.txt") + expected_filepath_aa = os.path.join(destination_dir, "subdir_a", "subsubdir_aa", "file_aa.txt") + expected_filepath_ab = os.path.join(destination_dir, "subdir_a", "subsubdir_ab", "file_ab.txt") + expected_filepath_b = os.path.join(destination_dir, "subdir_b", "file_b.txt") + expected_filepath_ba = os.path.join(destination_dir, "subdir_b", "subsubdir_ba", "file_ba.txt") + expected_filepath_bb = os.path.join(destination_dir, "subdir_b", "subsubdir_bb", "file_bb.txt") + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_aa) + assert not os.path.exists(expected_filepath_ab) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_ba) + assert not os.path.exists(expected_filepath_bb) + + # Provide only the original tmpdir to the copying function + copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + + # assert that the subdirs and files all exist + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_aa) + assert os.path.exists(expected_filepath_ab) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_ba) + assert os.path.exists(expected_filepath_bb) Index: branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils.py (revision 41170) @@ -0,0 +1,56 @@ +import subprocess + +# Plans are to flesh this out a bit more to make some smarter copying etc +from .errors import ValidationError, SubprocessError + + +def _args_check(args, function_name): + if args is None: + raise ValidationError(f"Error: no arguments provided in {function_name}") + if type(args) != list or len(args) == 0: + raise ValidationError(f"Error: no arguments provided in {function_name}") + return 0 + + +def cp_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {cp_wrapper.__name__}") + command_and_args = ["cp"] + args + return simple_unix_wrapper(command_and_args) + + +def tar_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {tar_wrapper.__name__}") + command_and_args = ["tar"] + args + return simple_unix_wrapper(command_and_args) + + +def chmod_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {chmod_wrapper.__name__}") + command_and_args = ["chmod"] + args + return simple_unix_wrapper(command_and_args) + + +def local_rsync_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}") + command_and_args = ["rsync"] + args + return simple_unix_wrapper(command_and_args) + + +def simple_unix_wrapper(command_and_args: [str]): + """ This should provide the program name and all arguments in a list. + +Only a single command should be called. + """ + _args_check(command_and_args, simple_unix_wrapper.__name__) + + try: + subprocess_exitcode = subprocess.check_call(command_and_args) + except (subprocess.CalledProcessError) as cpe: + raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe + except (ValidationError) as ve: + raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve + return subprocess_exitcode Index: branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/utils/subprocess_utils_test.py (revision 41170) @@ -0,0 +1,240 @@ +import backups.utils.subprocess_utils as utils +import os +import pytest +import stat + +from pathlib import Path +from .errors import SubprocessError, ValidationError + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyWrapper(object): + + def test_copy_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.cp_wrapper(None) + + def test_copy_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", "not a valid copy_location"]) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", tmpdir]) + + def test_simple_copy(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.cp_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_copy_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run copy function and confirm copy exists (and original) + result = utils.cp_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # fails without -r arg: + with pytest.raises(SubprocessError): + utils.cp_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.cp_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + + +class TestTarWrapper(object): + + def test_tar_raise_validation_error_with_none_args(self): + with pytest.raises(ValidationError): + utils.tar_wrapper(None) + + def test_tar_returns_safely_with_too_few_args(self): + with pytest.raises(SubprocessError): + utils.tar_wrapper(["only one arg"]) + + def test_tar_return_error_if_invalid_args_given(self, tmpdir): + tar_filename = os.path.join(tmpdir, "tar_file.tar") + fake_file = os.path.join(tmpdir, "fake_file.txt") + assert not os.path.exists(fake_file) + with pytest.raises(SubprocessError): + utils.tar_wrapper(["-cf", tar_filename, fake_file]) + + def test_simple_tar_of_files(self, tmpdir): + file_a = make_a_real_file(tmpdir, "file_a.txt") + file_b = make_a_real_file(tmpdir, "file_b.txt") + + expected_file = os.path.join(tmpdir, "output.tar") + assert not os.path.exists(expected_file) + + args = ["-cf", expected_file, file_a, file_b] + utils.tar_wrapper(args) + assert os.path.exists(expected_file) + + +class TestChmodWrapper(object): + + def test_chmod_gives_validation_error_when_no_args(self): + with pytest.raises(utils.ValidationError): + utils.chmod_wrapper(None) + + def test_chmod_wrapper_raises_error_with_invalid_args(self): + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["only one arg"]) + + def test_chmod_fails_if_no_permissions_given(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + with pytest.raises(SubprocessError): + utils.chmod_wrapper([real_file]) + + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["dumb_arg", real_file]) + + def test_can_chmod_file(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + # check initial permissions + org_stats = os.stat(real_file) + # Default should be: rw-rw-r-- + assert bool(org_stats.st_mode & stat.S_IRUSR) is True + assert bool(org_stats.st_mode & stat.S_IWUSR) is True + assert bool(org_stats.st_mode & stat.S_IXUSR) is False + assert bool(org_stats.st_mode & stat.S_IRGRP) is True + assert bool(org_stats.st_mode & stat.S_IWGRP) is True + assert bool(org_stats.st_mode & stat.S_IXGRP) is False + assert bool(org_stats.st_mode & stat.S_IROTH) is True + assert bool(org_stats.st_mode & stat.S_IWOTH) is False + assert bool(org_stats.st_mode & stat.S_IXOTH) is False + + # change them + utils.chmod_wrapper(["777", real_file]) + + # check they changed + new_stats = os.stat(real_file) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestLocalRsyncWrapper(object): + """This is essentially a copy of the 'copy' tests above. + """ + + def test_rsync_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.local_rsync_wrapper(None) + + def test_rsync_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"]) + + def test_rsync_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", tmpdir]) + + def test_simple_rsync(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_rsync_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run rsync function and confirm rsync exists (and original) + result = utils.local_rsync_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_rsyncing_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # silently passes without -r, just skips over directories + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.local_rsync_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # requires -r to copy directories (and contents) + utils.local_rsync_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) Index: branches/eam_branches/ipp-20191011/backups/website_backup.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/website_backup.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/website_backup.py (revision 41170) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') + +WEBSITE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class WebsiteBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + wb = WebsiteBackup(config, schema) + wb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: branches/eam_branches/ipp-20191011/backups/website_backup_test.py =================================================================== --- branches/eam_branches/ipp-20191011/backups/website_backup_test.py (revision 41170) +++ branches/eam_branches/ipp-20191011/backups/website_backup_test.py (revision 41170) @@ -0,0 +1,116 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import website_backup as webbck_mod +from website_backup import WebsiteBackup +from utils.config_parse_helper import ConfigSchema + +WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/website_test.config') + + +class TestArguments(object): + + def assert_defaults(self, wb: WebsiteBackup): + + assert wb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/website_backups' + , '/data/ippops4.0/backups/website_backups' + , '/data/ippops5.0/backups/website_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert wb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/www'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) + self.assert_defaults(wb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + webbck_mod.main(config, schema) + assert True is True + + +class TestWebsiteRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'www')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "www") + subdir = os.path.join(homedir, "subdir") + file = os.path.join(subdir, "website_file") + os.makedirs(homedir) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + wb = WebsiteBackup(config, schema) + + expected_subdir_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir') + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir/website_file') + expected_subdir_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir/website_file') + assert not os.path.exists(expected_subdir_bck1) + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_subdir_bck2) + assert not os.path.exists(expected_file_bck2) + wb._run_rsync() + assert os.path.exists(expected_subdir_bck1) + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_subdir_bck2) + assert os.path.exists(expected_file_bck2) Index: branches/eam_branches/ipp-20191011/dbconfig/cam.md =================================================================== --- branches/eam_branches/ipp-20191011/dbconfig/cam.md (revision 41169) +++ branches/eam_branches/ipp-20191011/dbconfig/cam.md (revision 41170) @@ -120,4 +120,10 @@ background_model S16 0 astrom_chips S64 0 + ast_r0 F32 0.0 + ast_d0 F32 0.0 + ast_t0 F32 0.0 + ast_s0 F32 0.0 + ast_rs F32 0.0 + ast_ds F32 0.0 END Index: branches/eam_branches/ipp-20191011/dbconfig/changes.txt =================================================================== --- branches/eam_branches/ipp-20191011/dbconfig/changes.txt (revision 41169) +++ branches/eam_branches/ipp-20191011/dbconfig/changes.txt (revision 41170) @@ -2532,3 +2532,23 @@ -- + ALTER TABLE ippRelease ADD COLUMN accessLevelMin INT DEFAULT 0 AFTER ubercal_file + +-- Update camProcessedExp to store astrometry results from smfs -- + +ALTER TABLE camProcessedExp ADD COLUMN ast_r0 FLOAT after astrom_chips; +ALTER TABLE camProcessedExp ADD COLUMN ast_d0 FLOAT after ast_r0; +ALTER TABLE camProcessedExp ADD COLUMN ast_t0 FLOAT after ast_d0; +ALTER TABLE camProcessedExp ADD COLUMN ast_s0 FLOAT after ast_t0; +ALTER TABLE camProcessedExp ADD COLUMN ast_rs FLOAT after ast_s0; +ALTER TABLE camProcessedExp ADD COLUMN ast_ds FLOAT after ast_rs; + +-- Update addRun to store hostname info -- + +ALTER TABLE addRun ADD COLUMN hostname varchar(64) after minidvodb_name; +ALTER TABLE minidvodbProcessed ADD COLUMN hostname varchar(64) after fault; + +-- +UPDATE dbversion set schema_version = '1.1.81', updated= CURRENT_TIMESTAMP(); +-- + Index: branches/eam_branches/ipp-20191011/doc/release.2015/inputs/lib.bib =================================================================== --- branches/eam_branches/ipp-20191011/doc/release.2015/inputs/lib.bib (revision 41169) +++ branches/eam_branches/ipp-20191011/doc/release.2015/inputs/lib.bib (revision 41170) @@ -6,115 +6,290 @@ % 2012ApJ...756..158S == ubercal -% paper I +% old paper I +%% @ARTICLE{chambers2017, +%% author = {{Chambers}, K.~C. and {Magnier}, E.~A. and {Metcalfe}, N. and et al.}, +%% title = "{IPP}", +%% journal = {ArXiv e-prints}, +%% archivePrefix = "arXiv", +%% eprint ={1612.05560}, +%% primaryClass = "astro-ph.HE", +%% keywords = {Astrophysics}, +%% year = 2017, +%% month = jan, +%% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, +%% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +%% } + +% 2016arXiv161205560C @ARTICLE{chambers2017, - author = {{Chambers}, K.~C. and {Magnier}, E.~A. and {Metcalfe}, N. and et al.}, - title = "{IPP}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint ={1612.05560}, - primaryClass = "astro-ph.HE", - keywords = {Astrophysics}, - year = 2017, - month = jan, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} + author = {{Chambers}, K.~C. and {Magnier}, E.~A. and {Metcalfe}, N. and + {Flewelling}, H.~A. and {Huber}, M.~E. and {Waters}, C.~Z. and + {Denneau}, L. and {Draper}, P.~W. and {Farrow}, D. and + {Finkbeiner}, D.~P. and {Holmberg}, C. and {Koppenhoefer}, J. and + {Price}, P.~A. and {Rest}, A. and {Saglia}, R.~P. and + {Schlafly}, E.~F. and {Smartt}, S.~J. and {Sweeney}, W. and + {Wainscoat}, R.~J. and {Burgett}, W.~S. and {Chastel}, S. and + {Grav}, T. and {Heasley}, J.~N. and {Hodapp}, K.~W. and {Jedicke}, R. and + {Kaiser}, N. and {Kudritzki}, R. -P. and {Luppino}, G.~A. and + {Lupton}, R.~H. and {Monet}, D.~G. and {Morgan}, J.~S. and + {Onaka}, P.~M. and {Shiao}, B. and {Stubbs}, C.~W. and {Tonry}, J.~L. and + {White}, R. and {Ba{\~n}ados}, E. and {Bell}, E.~F. and {Bender}, R. and + {Bernard}, E.~J. and {Boegner}, M. and {Boffi}, F. and + {Botticella}, M.~T. and {Calamida}, A. and {Casertano}, S. and + {Chen}, W. -P. and {Chen}, X. and {Cole}, S. and {Deacon}, N. and + {Frenk}, C. and {Fitzsimmons}, A. and {Gezari}, S. and {Gibbs}, V. and + {Goessl}, C. and {Goggia}, T. and {Gourgue}, R. and {Goldman}, B. and + {Grant}, P. and {Grebel}, E.~K. and {Hambly}, N.~C. and {Hasinger}, G. and + {Heavens}, A.~F. and {Heckman}, T.~M. and {Henderson}, R. and + {Henning}, T. and {Holman}, M. and {Hopp}, U. and {Ip}, W. -H. and + {Isani}, S. and {Jackson}, M. and {Keyes}, C.~D. and + {Koekemoer}, A.~M. and {Kotak}, R. and {Le}, D. and {Liska}, D. and + {Long}, K.~S. and {Lucey}, J.~R. and {Liu}, M. and {Martin}, N.~F. and + {Masci}, G. and {McLean}, B. and {Mindel}, E. and {Misra}, P. and + {Morganson}, E. and {Murphy}, D.~N.~A. and {Obaika}, A. and + {Narayan}, G. and {Nieto-Santisteban}, M.~A. and {Norberg}, P. and + {Peacock}, J.~A. and {Pier}, E.~A. and {Postman}, M. and {Primak}, N. and + {Rae}, C. and {Rai}, A. and {Riess}, A. and {Riffeser}, A. and + {Rix}, H.~W. and {R{\"o}ser}, S. and {Russel}, R. and {Rutz}, L. and + {Schilbach}, E. and {Schultz}, A.~S.~B. and {Scolnic}, D. and + {Strolger}, L. and {Szalay}, A. and {Seitz}, S. and {Small}, E. and + {Smith}, K.~W. and {Soderblom}, D.~R. and {Taylor}, P. and + {Thomson}, R. and {Taylor}, A.~N. and {Thakar}, A.~R. and {Thiel}, J. and + {Thilker}, D. and {Unger}, D. and {Urata}, Y. and {Valenti}, J. and + {Wagner}, J. and {Walder}, T. and {Walter}, F. and {Watters}, S.~P. and + {Werner}, S. and {Wood-Vasey}, W.~M. and {Wyse}, R.}, + title = "{The Pan-STARRS1 Surveys}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics, Astrophysics - Earth and Planetary Astrophysics, Astrophysics - Astrophysics of Galaxies, Astrophysics - Solar and Stellar Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05560}, + pages = {arXiv:1612.05560}, +archivePrefix = {arXiv}, + eprint = {1612.05560}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205560C}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% old paper II +%@ARTICLE{magnier2017.datasystem, +% author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and et al.}, +% title = "{IPP}", +% journal = {ArXiv e-prints}, +%archivePrefix = "arXiv", +% eprint ={1612.05242}, +% primaryClass = "astro-ph.HE", +% keywords = {Astrophysics}, +% year = 2017, +% month = jan, +% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, +% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +%} % paper II +% 2016arXiv161205240M @ARTICLE{magnier2017.datasystem, - author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and et al.}, - title = "{IPP}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint ={1612.05242}, - primaryClass = "astro-ph.HE", - keywords = {Astrophysics}, - year = 2017, - month = jan, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} - + author = {{Magnier}, Eugene A. and {Chambers}, K.~C. and {Flewelling}, H.~A. and + {Hoblitt}, J.~C. and {Huber}, M.~E. and {Price}, P.~A. and + {Sweeney}, W.~E. and {Waters}, C.~Z. and {Denneau}, L. and + {Draper}, P. and {Hodapp}, K.~W. and {Jedicke}, R. and {Kaiser}, N. and + {Kudritzki}, R. -P. and {Metcalfe}, N. and {Stubbs}, C.~W. and + {Wainscoast}, R.~J.}, + title = "{The Pan-STARRS Data Processing System}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05240}, + pages = {arXiv:1612.05240}, +archivePrefix = {arXiv}, + eprint = {1612.05240}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205240M}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% old paper III +% @ARTICLE{waters2017, +% author = {{Waters}, C.~Z. and {Magnier}, E.~A. and {Price}, P.~A. and +% {Chambers}, K.~C. and {Draper}, P. and {Flewelling}, H.~A. and +% {Hodapp}, K.~W. and {Huber}, M.~E. and {Jedicke}, R. and {Kaiser}, N. and +% {Kudritzki}, R.-P. and {Lupton}, R.~H. and {Metcalfe}, N. and +% {Rest}, A. and {Sweeney}, W.~E. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and +% {Wood-Vasey}, W.~M. and {Builders}, P.}, +% title = "{Pan-Starrs Pixel Processing: Detrending, Warping, Stacking}", +% journal = {ArXiv e-prints}, +% archivePrefix = "arXiv", +% eprint = {1612.05245}, +% primaryClass = "astro-ph.IM", +% keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, +% year = 2016, +% month = dec, +% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205245W}, +% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +% } + % paper III +% 2016arXiv161205245W @ARTICLE{waters2017, - author = {{Waters}, C.~Z. and {Magnier}, E.~A. and {Price}, P.~A. and - {Chambers}, K.~C. and {Draper}, P. and {Flewelling}, H.~A. and - {Hodapp}, K.~W. and {Huber}, M.~E. and {Jedicke}, R. and {Kaiser}, N. and - {Kudritzki}, R.-P. and {Lupton}, R.~H. and {Metcalfe}, N. and - {Rest}, A. and {Sweeney}, W.~E. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and - {Wood-Vasey}, W.~M. and {Builders}, P.}, - title = "{Pan-Starrs Pixel Processing: Detrending, Warping, Stacking}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05245}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205245W}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} + author = {{Waters}, C.~Z. and {Magnier}, E.~A. and {Price}, P.~A. and + {Chambers}, K.~C. and {Burgett}, W.~S. and {Draper}, P. and + {Flewelling}, H.~A. and {Hodapp}, K.~W. and {Huber}, M.~E. and + {Jedicke}, R. and {Kaiser}, N. and {Kudritzki}, R. -P. and + {Lupton}, R.~H. and {Metcalfe}, N. and {Rest}, A. and {Sweeney}, W.~E. and + {Tonry}, J.~L. and {Wainscoat}, R.~J. and {Wood-Vasey}, W.~M. and + {Builders}, PS1}, + title = "{Pan-STARRS Pixel Processing: Detrending, Warping, Stacking}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05245}, + pages = {arXiv:1612.05245}, +archivePrefix = {arXiv}, + eprint = {1612.05245}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205245W}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} } +% old paper IV +% @ARTICLE{magnier2017.analysis, +% author = {{Magnier}, E.~A. and {Sweeney}, W.~E. and {Chambers}, K.~C. and +% {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and +% {Waters}, C.~Z. and {Denneau}, L. and {Draper}, P. and {Jedicke}, R. and +% {Hodapp}, K.~W. and {Kudritzki}, R.-P. and {Metcalfe}, N. and +% {Stubbs}, C.~W. and {Wainscoast}, R.~J.}, +% title = "{Pan-STARRS Pixel Analysis : Source Detection $\backslash${\amp} Characterization}", +% journal = {ArXiv e-prints}, +% archivePrefix = "arXiv", +% eprint = {1612.05244}, +% primaryClass = "astro-ph.IM", +% keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, +% year = 2016, +% month = dec, +% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205244M}, +% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +% } + % paper IV +% 2016arXiv161205244M @ARTICLE{magnier2017.analysis, - author = {{Magnier}, E.~A. and {Sweeney}, W.~E. and {Chambers}, K.~C. and - {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and - {Waters}, C.~Z. and {Denneau}, L. and {Draper}, P. and {Jedicke}, R. and - {Hodapp}, K.~W. and {Kudritzki}, R.-P. and {Metcalfe}, N. and - {Stubbs}, C.~W. and {Wainscoast}, R.~J.}, - title = "{Pan-STARRS Pixel Analysis : Source Detection $\backslash${\amp} Characterization}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05244}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205244M}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} + author = {{Magnier}, Eugene A. and {Sweeney}, W.~E. and {Chambers}, K.~C. and + {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and + {Waters}, C.~Z. and {Denneau}, L. and {Draper}, P. and {Jedicke}, R. and + {Hodapp}, K.~W. and {Kaiser}, N. and {Kudritzki}, R. -P. and + {Metcalfe}, N. and {Stubbs}, C.~W. and {Wainscoast}, R.~J.}, + title = "{Pan-STARRS Pixel Analysis : Source Detection and Characterization}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05244}, + pages = {arXiv:1612.05244}, +archivePrefix = {arXiv}, + eprint = {1612.05244}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205244M}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% old paper V +% @ARTICLE{magnier2017.calibration, +% author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and +% {Tonry}, J.~L. and {Goldman}, B. and {R{\"o}ser}, S. and {Schilbach}, E. and +% {Chambers}, K.~C. and {Flewelling}, H.~A. and {Huber}, M.~E. and +% {Price}, P.~A. and {Sweeney}, W.~E. and {Waters}, C.~Z. and +% {Denneau}, L. and {Draper}, P. and {Hodapp}, K.~W. and {Jedicke}, R. and +% {Kudritzki}, R.-P. and {Metcalfe}, N. and {Stubbs}, C.~W. and +% {Wainscoast}, R.~J.}, +% title = "{Pan-STARRS Photometric and Astrometric Calibration}", +% journal = {ArXiv e-prints}, +% archivePrefix = "arXiv", +% eprint = {1612.05242}, +% primaryClass = "astro-ph.IM", +% keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, +% year = 2016, +% month = dec, +% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205242M}, +% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +% } % paper V +% 2016arXiv161205242M @ARTICLE{magnier2017.calibration, - author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and - {Tonry}, J.~L. and {Goldman}, B. and {R{\"o}ser}, S. and {Schilbach}, E. and - {Chambers}, K.~C. and {Flewelling}, H.~A. and {Huber}, M.~E. and - {Price}, P.~A. and {Sweeney}, W.~E. and {Waters}, C.~Z. and - {Denneau}, L. and {Draper}, P. and {Hodapp}, K.~W. and {Jedicke}, R. and - {Kudritzki}, R.-P. and {Metcalfe}, N. and {Stubbs}, C.~W. and - {Wainscoast}, R.~J.}, - title = "{Pan-STARRS Photometric and Astrometric Calibration}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05242}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205242M}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} + author = {{Magnier}, Eugene. A. and {Schlafly}, Edward. F. and + {Finkbeiner}, Douglas P. and {Tonry}, J.~L. and {Goldman}, B. and + {R{\"o}ser}, S. and {Schilbach}, E. and {Chambers}, K.~C. and + {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and + {Sweeney}, W.~E. and {Waters}, C.~Z. and {Denneau}, L. and + {Draper}, P. and {Hodapp}, K.~W. and {Jedicke}, R. and + {Kudritzki}, R. -P. and {Metcalfe}, N. and {Stubbs}, C.~W. and + {Wainscoast}, R.~J.}, + title = "{Pan-STARRS Photometric and Astrometric Calibration}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05242}, + pages = {arXiv:1612.05242}, +archivePrefix = {arXiv}, + eprint = {1612.05242}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205242M}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% old paper VI +% @ARTICLE{flewelling2017, +% author = {{Flewelling}, H.~A. and {Magnier}, E.~A. and {Chambers}, K.~C. and +% {Heasley}, J.~N. and {Holmberg}, C. and {Huber}, M.~E. and {Sweeney}, W. and +% {Waters}, C.~Z. and {Chen}, T. and {Farrow}, D. and {Hasinger}, G. and +% {Henderson}, R. and {Long}, K.~S. and {Metcalfe}, N. and {Nieto-Santisteban}, M.~A. and +% {Norberg}, P. and {Saglia}, R.~P. and {Szalay}, A. and {Rest}, A. and +% {Thakar}, A.~R. and {Tonry}, J.~L. and {Valenti}, J. and {Werner}, S. and +% {White}, R. and {Denneau}, L. and {Draper}, P.~W. and {Jedicke}, R. and +% {Kudritzki}, R.-P. and {Price}, P.~A. and {Chastel}, S. and +% {McClean}, B. and {Postman}, M. and {Shiao}, B.}, +% title = "{The Pan-STARRS1 Database and Data Products}", +% journal = {ArXiv e-prints}, +% archivePrefix = "arXiv", +% eprint = {1612.05243}, +% primaryClass = "astro-ph.IM", +% keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, +% year = 2016, +% month = dec, +% adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205243F}, +% adsnote = {Provided by the SAO/NASA Astrophysics Data System} +% } % paper VI +% 2016arXiv161205243F @ARTICLE{flewelling2017, - author = {{Flewelling}, H.~A. and {Magnier}, E.~A. and {Chambers}, K.~C. and - {Heasley}, J.~N. and {Holmberg}, C. and {Huber}, M.~E. and {Sweeney}, W. and - {Waters}, C.~Z. and {Chen}, T. and {Farrow}, D. and {Hasinger}, G. and - {Henderson}, R. and {Long}, K.~S. and {Metcalfe}, N. and {Nieto-Santisteban}, M.~A. and - {Norberg}, P. and {Saglia}, R.~P. and {Szalay}, A. and {Rest}, A. and - {Thakar}, A.~R. and {Tonry}, J.~L. and {Valenti}, J. and {Werner}, S. and - {White}, R. and {Denneau}, L. and {Draper}, P.~W. and {Jedicke}, R. and - {Kudritzki}, R.-P. and {Price}, P.~A. and {Chastel}, S. and - {McClean}, B. and {Postman}, M. and {Shiao}, B.}, - title = "{The Pan-STARRS1 Database and Data Products}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05243}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205243F}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} + author = {{Flewelling}, H.~A. and {Magnier}, E.~A. and {Chambers}, K.~C. and + {Heasley}, J.~N. and {Holmberg}, C. and {Huber}, M.~E. and + {Sweeney}, W. and {Waters}, C.~Z. and {Calamida}, A. and + {Casertano}, S. and {Chen}, X. and {Farrow}, D. and {Hasinger}, G. and + {Henderson}, R. and {Long}, K.~S. and {Metcalfe}, N. and {Narayan}, G. and + {Nieto-Santisteban}, M.~A. and {Norberg}, P. and {Rest}, A. and + {Saglia}, R.~P. and {Szalay}, A. and {Thakar}, A.~R. and + {Tonry}, J.~L. and {Valenti}, J. and {Werner}, S. and {White}, R. and + {Denneau}, L. and {Draper}, P.~W. and {Hodapp}, K.~W. and + {Jedicke}, R. and {Kaiser}, N. and {Kudritzki}, R.~P. and + {Price}, P.~A. and {Wainscoat}, R.~J. and {Builders}, P.~S. and + {Chastel}, S. and {McLean}, B. and {Postman}, M. and {Shiao}, B.}, + title = "{The Pan-STARRS1 Database and Data Products}", + journal = {arXiv e-prints}, + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2016", + month = "Dec", + eid = {arXiv:1612.05243}, + pages = {arXiv:1612.05243}, +archivePrefix = {arXiv}, + eprint = {1612.05243}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161205243F}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} } @@ -16610,2 +16785,15 @@ } +@INPROCEEDINGS{1988ESOC...30..693B, + author = {{Beckers}, J.~M.}, + title = "{Increasing the Size of the Isoplanatic Patch with Multiconjugate Adaptive Optics}", + booktitle = {European Southern Observatory Conference and Workshop Proceedings}, + year = "1988", + series = {European Southern Observatory Conference and Workshop Proceedings}, + volume = {30}, + month = "Jan", + pages = {693}, + adsurl = {https://ui.adsabs.harvard.edu/abs/1988ESOC...30..693B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + Index: branches/eam_branches/ipp-20191011/doc/release.2015/ps1.analysis/analysis.tex =================================================================== --- branches/eam_branches/ipp-20191011/doc/release.2015/ps1.analysis/analysis.tex (revision 41169) +++ branches/eam_branches/ipp-20191011/doc/release.2015/ps1.analysis/analysis.tex (revision 41170) @@ -104,4 +104,6 @@ % insert additional keywords as appropriate: \keywords{methods: data analysis -- Surveys:\PSONE -- techniques: image processing -- techniques: photometric} + +\note{add Danny Farrow to author list} \section{Introduction} Index: branches/eam_branches/ipp-20191011/doc/release.2015/ps1.calibration/calibration.tex =================================================================== --- branches/eam_branches/ipp-20191011/doc/release.2015/ps1.calibration/calibration.tex (revision 41169) +++ branches/eam_branches/ipp-20191011/doc/release.2015/ps1.calibration/calibration.tex (revision 41170) @@ -105,14 +105,21 @@ and to place all of the observations onto a photometric system with consistent zero points over the entire area surveyed, the \approx -30,000 square degrees north of $\delta = -30$\degrees. The -astrometric calibration compensates for similar systematic effects so -that positions, proper motions, and parallaxes are reliable as well. -The Pan-STARRS Data Release 2 (DR2) astrometry is tied to the Gaia DR1 -release. - +30,000 square degrees north of $\delta = -30$\degrees. \textadd{Using external +comparisons, we demonstrate that the resulting photometic system is +consistent across the sky to between 7 and 12.4 millimags depending on +the filter. For bright stars, the systematic error floor for +individual measurementsis $(\sigma_g, \sigma_r, \sigma_i, \sigma_z, +\sigma_y) = (14, 14, 15, 15, 18)$ millimags.} The astrometric +calibration compensates for similar systematic effects so that +positions, proper motions, and parallaxes are reliable as well. \textadd{The +bright-star systematic error floor for individual astrometric +measurements is 16 milliarcseconds.} \textmod{The Pan-STARRS Data Release 2 +(DR2) astrometry is tied to the Gaia DR1 coordinate frame with a +systematic uncertainty of $\sim 5$ milliarcseconds.} \end{abstract} % insert additional keywords as appropriate: -\keywords{astrometry -- methods: statistical -- proper motions -- Surveys:\PSONE -- techniques: photometric} +\keywords{astrometry -- methods: statistical -- proper motions -- + Surveys:\PSONE -- techniques: photometric} \section{Introduction}\label{sec:intro} @@ -450,5 +457,5 @@ code restricts the exponents with the rule $i + j <= N_{\rm order}$ where the order of the fit, $N_{\rm order}$, may be 1 to 3, under the -restriction that sufficient stars are needed to constrain the order +restriction that sufficient stars are needed to constrain the order. For each chip, a second set of polynomials describes the transformation from the chip coordinate systems to the focal @@ -475,4 +482,22 @@ M & = & C^M_{0,0} + C^M_{1,0} X + C^M_{0,1} Y + \delta M(X, Y) \end{eqnarray} + +\textadd{These high-order transformations are required for the + individual chips to follow small-scale distortions due to the optics + (stable from exposure to exposure) as well as the atmosphere + (changes from over time). The spatial scale on which the + astrometric deviations due to atmosphere are varying is related to + the isoplanetic patch size. We note that, in the typical conditions + at the \PSONE\ site, if the seeing is due to low-lying atmospheric + layers, the isoplanetic patch scale will be a most a few arcminutes + \citep{1988ESOC...30..693B}, and smaller when the seeing comes from + higher altitudes. + +We also note that, in our detailed astrometric analysis within the +database system, we perform an initial correction for several +systematic effects including the color-dependent correction due to +differential chromatic refraction. The corrected chip positions are +the inputs to the equations above (see +Section~\ref{sec:astrometry.systematic}).} \subsection{Cross-Correlation Search} @@ -839,5 +864,5 @@ \cite{2012ApJ...756..158S}. This analysis is performed by the group at Harvard, loading data from the raw detection files into their instance -of the Large Scale Database \citep[LSD,][]{2011AAS...21743319J}, a +of the Large Survey Database \citep[LSD,][]{2011AAS...21743319J}, a system similar to DVO used to manage the detections and determine the calibrations. @@ -845,11 +870,14 @@ Photometric nights are selected and all other exposures are ignored. Each night is allowed to have a single fitted zero point -(corresponding to the sum $zp_{\rm ref} + M_{cal}$ below) and a -single fitted value for the airmass extinction coefficient ($K_{\rm +(corresponding to the sum $zp_{\rm ref} + M_{cal}$ below) and a single +fitted value for the airmass extinction coefficient ($K_{\rm \lambda}$) per filter. The zero points and extinction terms are determined as a least squares minimization process using the repeated measurements of the same stars from different nights to tie nights -together. Flat-field corrections are also determined as part of the -minimization process. In the original (PV1) ubercal analysis, +together. \textadd{This analysis relies on the chemical and + thermodynamic stability of the atmosphere during a photometic night + so that the zero point and extinction slope are stable as a result.} +Flat-field corrections are also determined as part of the minimization +process. In the original (PV1) ubercal analysis, \cite{2012ApJ...756..158S} determined flat-field corrections for $2\times 2$ sub-regions of each chip in the camera and four distinct @@ -872,9 +900,15 @@ aided by the inclusion of multiple Medium Deep field observations every night, helping to tie down overall variations of the system -throughput and acting as internal standard star fields. The resulting -photometric system is shown by \cite{2012ApJ...756..158S} to have reliability -across the survey region at the level of (8.0, 7.0, 9.0, 10.7, 12.4) -millimags in (\grizy). As we discuss below, this conclusion is -reinforced by our external comparison. +throughput and acting as internal standard star fields. \textmod{The + resulting photometric system is shown by \cite{2012ApJ...756..158S} + to have zero-points which are consistent with those determined using + SDSS as an external reference, with standard deviations of (8.0, + 7.0, 9.0, 10.7, 12.4) millimags in (\grizy). Internal comparisons + show the zero-points of indidual exposures to be consistent with the + Ubercal solution with a standard deviation of 5 millimags. The + former is an upper limit on the overall system zero-point stability, + since it includes errors from the SDSS zero points, while the latter + is likely a lower limit. As we discuss below, this zero-point + consistency is confirmed by our additional external comparison.} The overall zero point for each filter is not naturally determined by @@ -885,13 +919,17 @@ on the reference photometric night of MJD 55744 (UT 02 July 2011). \cite{2014ApJ...795...45S} and \cite{2015ApJ...815..117S} have -re-examined the photometry of Calspec standards \citep{1996AJ....111.1743B} as -observed by PS1. \cite{2014ApJ...795...45S} reject 2 of the 7 stars -used by \cite{2012ApJ...750...99T} and add photometry of 5 additional -stars. \cite{2015ApJ...815..117S} further reject measurements of -Calspec standards obtained close to the center of the camera field of -view where the PSF size and shape changes very rapidly. The result of -this analysis modifies the over system zero points by 20 - 35 -millimags compared with the system determined by -\cite{2012ApJ...756..158S}. +re-examined the photometry of Calspec standards +\citep{1996AJ....111.1743B} as observed by PS1. +\cite{2014ApJ...795...45S} reject 2 of the 7 stars used by +\cite{2012ApJ...750...99T} and add photometry of 5 additional stars. +\cite{2015ApJ...815..117S} further reject measurements of Calspec +standards obtained close to the center of the camera field of view +where the PSF size and shape changes very rapidly. The result of this +analysis modifies the over system zero points by 20 - 35 millimags +compared with the system determined by \cite{2012ApJ...756..158S}. \textmod{We +note that this correction to the overall system zero-point is large +compared to the relative zero-point consistency noted by +\cite{2012ApJ...756..158S} because the absolute zero points are not +independently constrained by the Ubercal analysis.} % http://iopscience.iop.org/article/10.1088/0004-637X/815/2/117/pdf @@ -1092,4 +1130,8 @@ \sigma_i^{-2})^2} \end{equation} + +These rejections and the over-weighting of the Ubercal measurements +are admittedly ad hoc. Since the goal at this stage is to tie the +non-Ubercal data to the Ubercal system, we The calculation of the relative photometry zero points is performed Index: branches/eam_branches/ipp-20191011/ippMonitor/INSTALL =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/INSTALL (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/INSTALL (revision 41170) @@ -1,2 +1,3 @@ +#!/bin/sh # below are commands to configure ippMonitor @@ -11,9 +12,25 @@ # testing of those parts of ippMonitor which depend on IPP code without modifyin # the operational installation +# 2019.11.18 CCL: not sure which parts of ippMonitor depends on IPP codes? If there is not many +# , just use trunk version should work fine. psconfig ipp-trunk-20180515 -# run psconfigure to generate Makefile and raw/site.php +# run psconfigure to generate Makefile and raw/site.php +# $CZARPLOTDIR and $METRICSPLOTDIR will be generated a link in the same folder as ippMonitor psconfigure --htdocs /var/www/localhost/htdocs --site ~/ippconfig/ippmonitor.config +# copy these files and modify them (CCL removed CZARPLOTDIR) +cp config.dat.in config.dat + # install the php code: +# czartool_plots and ippMetrics will be linked to different path in different hostnames make + +# for running czarpool.pl, +cp czartool/czarconfig.xml.in czartool/czarconfig.xml +# then set gnuplot path for czartool_plots and ippMetrics in ippMonitor dir +# /var/www/localhost/htdocs/ippMonitor/czartool_plots +# /var/www/localhost/htdocs/ippMonitor/ippMetrics +# enter czartool +./czarpool.pl + Index: branches/eam_branches/ipp-20191011/ippMonitor/Makefile.in =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/Makefile.in (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/Makefile.in (revision 41170) @@ -1,12 +1,14 @@ -default: updateVersion php +default: updateVersion php mklink help: @echo "USAGE: make php" -SRC = src -DEF = def -RAW = raw -SCRIPTS = scripts -DESTBIN = @BINDIR@ -DESTWWW = @HTDOCS@/ippMonitor +SRC = src +DEF = def +RAW = raw +SCRIPTS = scripts +DESTBIN = @BINDIR@ +DESTWWW = @HTDOCS@/ippMonitor +CZARPLOTDIR = /export/@HOST2@.0/ipp/czartool_plots +METRICSPLOTDIR = /export/@HOST2@.0/ipp/ippMetrics GENERATE = $(SCRIPTS)/generate @@ -172,4 +174,5 @@ $(DESTWWW)/fullforceRun.php \ $(DESTWWW)/fullforceSummary.php \ +$(DESTWWW)/fullforce_failure.php \ $(DESTWWW)/failedFullforce.php \ $(DESTWWW)/diffSummary.php \ @@ -280,7 +283,10 @@ $(DESTWWW)/noimage.png -DESTLINK = $(DESTWWW)/index.php - -php: $(RAWSRC) $(DEFSRC) $(PICTURES) $(DESTLINK) $(PROGRAMS) +DESTLINK = $(DESTWWW)/index.php +CZPLOTSLINK = $(DESTWWW)/czartool_plots +METRICSLINK = $(DESTWWW)/ippMetrics + + +php: $(RAWSRC) $(DEFSRC) $(PICTURES) $(DESTLINK) $(PROGRAMS) # dependancy rules for binary code ######################### @@ -288,5 +294,5 @@ # .PRECIOUS: $(BIN)/%.$(ARCH) -$(DESTLINK): $(DESTWWW)/Login.php +$(DESTLINK): $(DESTWWW)/Login.php rm -f $@ ln -s $(DESTWWW)/Login.php $@ @@ -311,6 +317,18 @@ updateVersion: rm -f $(DESTWWW)/version.php - -test: + @if [ ! -d $(CZARPLOTDIR) ]; then mkdir -p $(CZARPLOTDIR); fi + @if [ ! -d $(METRICSPLOTDIR) ]; then mkdir -p $(METRICSPLOTDIR); fi + +mklink: $(CZPLOTSLINK) $(METRICSLINK) + +$(CZPLOTSLINK): $(CZARPLOTDIR) + rm -f $@ + ln -s $(CZARPLOTDIR) $@ + +$(METRICSLINK): $(METRICSPLOTDIR) + rm -f $@ + ln -s $(METRICSPLOTDIR) $@ + +test: #DESTWWW := $(DESTWWW)/test echo $(DESTWWW) @@ -355,4 +373,7 @@ cp $(SCRIPTS)/$* $(DESTBIN)/$* || exit chmod +x $(DESTBIN)/$* || exit + + + # utilities ################################################# Index: branches/eam_branches/ipp-20191011/ippMonitor/configure.tcsh =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/configure.tcsh (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/configure.tcsh (revision 41170) @@ -134,4 +134,5 @@ set hostname = `hostname -f` +set hostname2 = `hostname` echo @@ -182,6 +183,7 @@ if (-e Makefile) mv -f Makefile Makefile~ cat Makefile.in | sed "s|@HTDOCS@|$htdocs|" > tmp.1 -cat tmp.1 | sed "s|@BINDIR@|$bindir|" > Makefile -rm -f tmp.1 +cat tmp.1 | sed "s|@HOST2@|$hostname2|" > tmp.2 +cat tmp.2 | sed "s|@BINDIR@|$bindir|" > Makefile +rm -f tmp.2 # convert raw/site.php.in to raw/site.php @@ -193,7 +195,8 @@ cat tmp.1 | sed "s|@DBI@|$DBI|" > tmp.2 cat tmp.2 | sed "s|@HOST@|$hostname|" > tmp.1 -awk 'BEGIN{t = 0}((t == 0) && ($1 != "@CONFIG.DAT@")){print $0}($1 == "@CONFIG.DAT@"){t = 1}' tmp.1 > raw/site.php +cat tmp.1 | sed "s|@HOST2@|$hostname2|" > tmp.2 +awk 'BEGIN{t = 0}((t == 0) && ($1 != "@CONFIG.DAT@")){print $0}($1 == "@CONFIG.DAT@"){t = 1}' tmp.2 > raw/site.php cat config.dat >> raw/site.php -awk 'BEGIN{t = 0}(t == 1){print $0}($1 == "@CONFIG.DAT@"){t = 1}' tmp.1 >> raw/site.php +awk 'BEGIN{t = 0}(t == 1){print $0}($1 == "@CONFIG.DAT@"){t = 1}' tmp.2 >> raw/site.php rm -f tmp.1 tmp.2 Index: branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarDbSchema.sql =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarDbSchema.sql (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarDbSchema.sql (revision 41170) @@ -404,4 +404,95 @@ -- +-- Table structure for table `science_labels` (from current_labels) +-- + +DROP TABLE IF EXISTS `science_labels`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `science_labels` ( + `server` varchar(128) default NULL, + `label` varchar(128) default NULL, + `telescope` varchar(128) NOT NULL, + `priority` int(11) NOT NULL default '0', + KEY `science_labels` (`server`,`label`, `telescope`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `science_nightlyscience` (from nightlyscience) +-- + +DROP TABLE IF EXISTS `science_nightlyscience`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `science_nightlyscience` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `status` varchar(128) default 'NONE', + `telescope` varchar(128) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `science_reverts` (from reverts) +-- + +DROP TABLE IF EXISTS `science_reverts`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `science_reverts` ( + `stage` varchar(128) default NULL, + `telescope` varchar(128) NOT NULL, + `reverting` tinyint(4) default NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `science_server_dates` (from server_dates) +-- + +DROP TABLE IF EXISTS `science_server_dates`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `science_server_dates` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `server` varchar(64) default 'NONE', + `telescope` varchar(128) NOT NULL, + `date` varchar(64) default 'NONE' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `science_servers` (from servers) +-- + +DROP TABLE IF EXISTS `science_servers`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `science_servers` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `server` varchar(128) default NULL, + `telescope` varchar(128) NOT NULL, + `alive` tinyint(4) default NULL, + `running` tinyint(4) default NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `live_servers` (from servers) +-- + +DROP TABLE IF EXISTS `live_servers`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `live_servers` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `server` varchar(128) default NULL, + `telescope` varchar(128) NOT NULL, + `alive` tinyint(4) default NULL, + `running` tinyint(4) default NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `server_dates` -- Index: branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarpoll.pl =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarpoll.pl (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/czartool/czarpoll.pl (revision 41170) @@ -236,4 +236,25 @@ $pantasks->getServerStatus($server,'gpc2', \$alive, \$running); $czarDb->updateServerStatus($server,'gpc2', $alive, $running); + } +} + +########################################################################### +# +# Updates pantasks server status TODO should really get info for all servers at once +# +########################################################################### +sub updateServerStatusLive { + print "* Checking all pantasks servers\n"; + + my $servers = $pantasks->getServerList(); + + my $server = undef; + my $alive = undef; + my $running = undef; + foreach $server (@{$servers}) { + $pantasks->getServerStatus($server,'gpc1', \$alive, \$running); + $czarDb->updateServerStatusLive($server,'gpc1', $alive, $running); + $pantasks->getServerStatus($server,'gpc2', \$alive, \$running); + $czarDb->updateServerStatusLive($server,'gpc2', $alive, $running); } } @@ -272,9 +293,10 @@ my $newDayTime = $config->getMetricsStartTime(); my $lastDayDailyTasks = "2010-01-01"; - + my $checkneb = 1; # 1 for nebulous, 0 for localhost test # main polling loop while (1) { + my $iterationStartTime = time(); - + #get the time of last czarpoll update $polldate = $czarDb->getPollDate(); @@ -287,9 +309,9 @@ if($timediff > 120.) { print "There is no recent Czarpoll iteration\n"; - print "*Updating database entries*\n"; + print "* Updating database entries*\n"; $dbupdate = 1; } else { print "There is a recent Czarpoll iteration already made ($timediff sec ago)\n"; - print "*Skipping database queries and entries*\n"; + print "* Skipping database queries and entries\n"; $dbupdate = 0; } @@ -326,6 +348,6 @@ #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB. #Therefore, it makes no sense to run the cleanup+optimising twice - #print "* Performing database cleanup\n"; - #$czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval()); + print "* Performing database cleanup\n"; + $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval()); #print "* Optimizing the database\n"; #$czarDb->optimize(); @@ -344,10 +366,18 @@ # check nebulous - print "* Checking Nebulous\n"; - if($dbupdate == 1) {$nebulous->updateClusterSpaceInfo();} - $plotter0->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end); - $plotter0->plotDiskUsageHistogram(); - $plotter0->plotDiskUsageHistogramLarge(); + if ($checkneb == 1) { + print "* Checking Nebulous\n"; + if($dbupdate == 1) {$nebulous->updateClusterSpaceInfo();} + $plotter0->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end); + $plotter0->plotDiskUsageHistogram(); + $plotter0->plotDiskUsageHistogramLarge(); + } + else { + print "* Locahost, so no checking nebulous\n"; + } + + # check pantasks status if($dbupdate == 1) {updateServerStatus();} + if($dbupdate == 1) {updateServerStatusLive();} # check pantasks dates Index: branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/CzarDb.pm (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/CzarDb.pm (revision 41170) @@ -166,4 +166,26 @@ SQL + $query->execute; +} + +########################################################################### +# +# Updates server table +# +########################################################################### +sub updateServerStatusLive { + my ($self, $server,$telescope, $alive, $running) = @_; + + my $query = $self->{_db}->prepare(<execute; + + $query = $self->{_db}->prepare(<execute; } Index: branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/Pantasks.pm =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/Pantasks.pm (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/czartool/czartool/Pantasks.pm (revision 41170) @@ -7,20 +7,25 @@ my @servers = ( -# "addstar", -# "addstarlap", - "ippqub:stdscience_ws", +##### standard pantasks ###### "cleanup", -# "deepstack", -# "detrend", "distribution", "pstamp", -# "update", -# "publishing", "registration", - "replication", "stack", "stdscience", - "summitcopy" + "summitcopy", +##### additional pantasks ###### + "addstar", + "addstarlap", + "deepstack", + "detrend", + "ippqub:stdscience_ws", + "publishing", + "replication", + "update" ); +# current home dir of gpc1 and gpc2 +my $ipphome1 = "/data/ippc64.1/ippitc"; +my $ipphome2 = "/data/ippc18.0/home/ippps2"; ########################################################################### @@ -90,8 +95,8 @@ if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { @@ -134,8 +139,8 @@ if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { @@ -177,8 +182,8 @@ if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { @@ -209,8 +214,8 @@ my $ipphome; if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { @@ -239,8 +244,8 @@ my $ipphome; if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { @@ -279,8 +284,8 @@ if ($telescope eq 'gpc1') { - $ipphome = "/data/ippc64.1/ippitc"; - } - elsif ($telescope eq 'gpc2') { - $ipphome = "/data/ippc18.0/home/ippps2"; + $ipphome = $ipphome1; + } + elsif ($telescope eq 'gpc2') { + $ipphome = $ipphome2; } else { Index: branches/eam_branches/ipp-20191011/ippMonitor/czartool/extra_labels.dat =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/czartool/extra_labels.dat (revision 41170) +++ branches/eam_branches/ipp-20191011/ippMonitor/czartool/extra_labels.dat (revision 41170) @@ -0,0 +1,4 @@ +## TdB20190626: add extra labels for processing in this file. Syntax is as follows: [labelname] [telescope] +## Priorities for the labels must be added to the priorities mySQL table for the relevant telescope, if desired. +PV3.Pole.Reprocess.20180510 gpc1 +ps_ud_QUB gpc1 Index: branches/eam_branches/ipp-20191011/ippMonitor/def/failedFullforce.d =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/def/failedFullforce.d (revision 41170) +++ branches/eam_branches/ipp-20191011/ippMonitor/def/failedFullforce.d (revision 41170) @@ -0,0 +1,26 @@ +TABLE fullForceInput JOIN warpRun USING (warp_id) JOIN fullForceRun USING (ff_id) JOIN fullForceResult USING (ff_id) +TITLE Full Force Failed +FILE failedFullforce.php +MENU ipp.ffs.dat + +ARGS ARG1 fullForceRun.ff_id=$fullForceRun.ff_id +ARGS ARG1 basename=$fullForceResult.path_base + +WHERE fullForceResult.fault != 0 +# field size format name show link to extras +FIELD fullForceRun.ff_id, 5, %d, FF ID +FIELD fullForceRun.label, 7, %s, Label +FIELD fullForceRun.state, 7, %s, State, value, failedFullforce.php, ARG1 +FIELD warpRun.tess_id, 5, %s, Tess ID +FIELD warpRun.label AS warpRunLabel, 5, %s, warp Label +FIELD fullForceRun.data_group, 7, %s, data grp +FIELD fullForceRun.dist_group, 7, %s, dist grp +FIELD fullForceResult.fault, 7, %d, fault +FIELD fullForceResult.quality, 7, %d, quality +FIELD fullForceResult.path_base, 7, %s, path_base, none + +MODE summary +GROUP fullForceRun.ff_id + +TAIL PHP insert_log ('LOG.EXP'); + Index: branches/eam_branches/ipp-20191011/ippMonitor/def/fullforceInputSkyfile.d =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/def/fullforceInputSkyfile.d (revision 41170) +++ branches/eam_branches/ipp-20191011/ippMonitor/def/fullforceInputSkyfile.d (revision 41170) @@ -0,0 +1,17 @@ +TABLE fullForceInput, fullForceRun, warpRun +TITLE Full Force Input Exposures +FILE fullforceInputSkyfile.php +MENU ipp.ffs.dat + +WHERE fullForceRun.ff_id = fullForceInput.ff_id AND fullForceInput.warp_id = warpRun.warp_id + +# field size format name show link to extras +FIELD fullForceRun.ff_id, 5, %s, FF ID +FIELD fullForceRun.label, 7, %s, Label +FIELD fullForceRun.state, 7, %s, State +FIELD warpRun.warp_id, 5, %s, Warp ID +FIELD warpRun.tess_id, 5, %s, Tess ID +FIELD warpRun.label, 5, %s, warp Label +FIELD fullForceRun.data_group, 7, %s, data grp +FIELD fullForceRun.dist_group, 7, %s, dist grp + Index: branches/eam_branches/ipp-20191011/ippMonitor/def/fullforce_failure.d =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/def/fullforce_failure.d (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/def/fullforce_failure.d (revision 41170) @@ -1,27 +1,26 @@ -TABLE staticskyRun JOIN staticskyResult USING (sky_id) JOIN staticskyInput USING (sky_id) JOIN stackRun USING (stack_id) -TITLE Static Sky Failed (Log) -FILE failedStaticsky.php -MENU ipp.stack.dat +TABLE fullForceInput JOIN warpRun USING (warp_id) JOIN fullForceRun USING (ff_id) JOIN fullForceResult USING (ff_id) +TITLE Full Force Failed +FILE fullforce_failure.php +MENU ipp.ffs.dat -WHERE staticskyResult.fault != 0 +ARGS ARG1 fullForceRun.ff_id=$fullForceRun.ff_id +ARGS ARG1 basename=$fullForceResult.path_base -ARGS ARG1 staticskyResult.path_base=$staticskyResult.path_base +WHERE fullForceResult.fault != 0 +# field size format name show link to extras +FIELD fullForceRun.ff_id, 5, %d, FF ID +FIELD fullForceRun.label, 7, %s, Label +FIELD fullForceRun.state, 7, %s, State, value, fullforce_failure.php, ARG1 +FIELD warpRun.tess_id, 5, %s, Tess ID +FIELD warpRun.label AS warpRunLabel, 5, %s, warp Label +FIELD fullForceRun.data_group, 7, %s, data grp +FIELD fullForceRun.dist_group, 7, %s, dist grp +FIELD fullForceResult.fault, 7, %d, fault +FIELD fullForceResult.quality, 7, %d, quality +FIELD fullForceResult.path_base, 7, %s, path_base, none -# field size format name show link to extras -FIELD staticskyRun.sky_id, 5, %d, Sky ID -FIELD staticskyRun.label, 7, %s, Label -FIELD staticskyRun.state, 7, %s, State, value, staticsky_failure.php, ARG1 -FIELD stackRun.skycell_id, 5, %s, Skycell ID -FIELD stackRun.tess_id, 5, %s, Tess ID -FIELD count(stackRun.filter), 5, %d, #Filters -FIELD stackRun.label, 5, %s, Stack Label -FIELD staticskyRun.data_group, 7, %s, data grp -FIELD staticskyRun.dist_group, 7, %s, dist grp -FIELD staticskyResult.sources, 7, %d, n(sources) -FIELD staticskyResult.fault, 7, %d, fault -FIELD staticskyResult.quality, 7, %d, quality -FIELD staticskyResult.path_base, 5, %s, path_base, none - -GROUP staticskyRun.sky_id +MODE summary +GROUP fullForceRun.ff_id TAIL PHP insert_log ('LOG.EXP'); + Index: branches/eam_branches/ipp-20191011/ippMonitor/raw/czartool_labels.php =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/raw/czartool_labels.php (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/raw/czartool_labels.php (revision 41170) @@ -214,105 +214,176 @@ echo ""; -#echo "
"; +///////////////////////////////////////// +// timer begin for status and links +///////////////////////////////////////// +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$start = $time; // status table at top $nsStatus = getNightlyScienceStatus($czardb, $proj); $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; -$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; +$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . + "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; + echo "
"; echo $table; - -createTableTitle("Status and links", 3); +createTableTitle("Status and links", 2); echo "\n"; -echo "Time of last update$lastUpdateTime"; -echo "ippMonitor SVN version"; include 'version.php'; echo ""; -echo "Current nightly science status$nsStatus"; -echo "Postage stamp server statushere"; -echo "Documentationhere"; -echo "Cluster loadhere cpu_report mem_report"; -echo "IPP metricshere"; -echo "Czar log pageshere"; -echo "Exposure summaryhere"; -echo "Confluencehere"; -echo "JIRAhere"; +echo "Time of last update + $lastUpdateTime"; +echo "ippMonitor SVN version + "; include 'version.php'; echo ""; +echo "Current nightly science status + $nsStatus"; +echo "Postage stamp server status + here"; +echo "Documentation + here"; +echo "Cluster load + + here + + cpu_report + + mem_report"; +echo "IPP metrics + here"; +echo "Czar log pages + here"; +echo "Exposure summary + here"; +echo "Confluence + here"; +echo "JIRA + here"; +echo "data store + here"; $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; -$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; - -echo "\n"; - -echo "
"; - - createLabelsTable($pass, - $proj, - $czardb, - $selectedMode, - $labels, - $distLabelsExist, - $pubLabelsExist, - $stages, - "new", - $selectedLabel, - $selectedStage, - $plotType); - - if ($selectedMode == "update") {echo "
"; createPStampDataTable();} - echo "
"; - - # XXX EAM : 2017.09.17 : debugging slowness - createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); echo "
"; - createSummitDataTable($projectdb); echo "
"; // this is slow because of the join between summitExp and rawExp - createChunkDataTable($projectdb); echo "
"; - createDatesTable($czardb, $proj); echo "
"; - - echo $table; - echo ""; - createTableTitle("Database status (update fix log here)", 2); - - echo ""; - createTableColumnHeader("Database"); - createTableColumnHeader("Status (sec behind master)"); - showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1); - showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); - showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY); - - echo ""; - - showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS); - showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); - showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY); - showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY); - -# some mysql replication slaves to add when they are returned to service -# showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP); -# showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP); - echo ""; - - echo $table; - echo ""; - createTableTitle("Critical machine status", 2); - - echo ""; - createTableColumnHeader("Hostname"); - createTableColumnHeader("Status"); - showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY); - showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY); - -# showPingStatus($HOST_APACHE1, $DBNAME_APACHE1); - showPingStatus($HOST_APACHE2, $DBNAME_APACHE2); - showPingStatus($HOST_APACHE3, $DBNAME_APACHE3); - showPingStatus($HOST_APACHE4, $DBNAME_APACHE4); - showPingStatus($HOST_APACHE5, $DBNAME_APACHE5); - showPingStatus($HOST_APACHE6, $DBNAME_APACHE6); - showPingStatus($HOST_APACHE7, $DBNAME_APACHE7); - showPingStatus($HOST_PROXY, $DBNAME_PROXY); - showPingStatus($HOST_SVN, $DBNAME_SVN); - showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE); - - echo ""; - -echo ""; +$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . + "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; + +// timer finished +echo ""; +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$finish = $time; +$total_time = round(($finish - $start), 3); +echo "loading in $total_time seconds."; +echo ""; +echo "
"; + +createLabelsTable($pass, $proj, $czardb, $selectedMode, $labels, + $distLabelsExist, $pubLabelsExist, $stages, "new", $selectedLabel, + $selectedStage, $plotType); +if ($selectedMode == "update") {echo "
"; createPStampDataTable();} + +# XXX EAM : 2017.09.17 : debugging slowness +# CCL : 2019.11.09 : debugging slowness +//createSummitDataTable($projectdb); // this is slow because of the join between summitExp and rawExp, but not the major one +createSummitDataTable2($projectdb); // this is slow because of the join between summitExp and rawExp, but not the major one + +createServersTable($pass, $proj,$selectedMode,$czardb, $servers, $selectedLabel, $selectedStage, $plotType); +// this was the major slowness part due to large table in czardb + +createChunkDataTable($projectdb); // this is another part for slowness during nightly processing + +createDatesTable($czardb, $proj); + +///////////////////////////////////////// +// timer begin for Database status +///////////////////////////////////////// +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$start = $time; + +// status table at top + echo $table; + echo ""; + createTableTitle("Database status (update fix log here)", 2); + + echo ""; + createTableColumnHeader("Database"); + createTableColumnHeader("Status (sec behind master)"); + showDatabaseStatus($HOST_GPC1, $USER_GPC1, $PASSWORD_GPC1, $DBNAME_GPC1); + showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); + showReplicationsStatus($REPL_HOST_GPC1_SECONDARY, $REPL_USER_GPC1_SECONDARY, $REPL_PASSWORD_GPC1_SECONDARY, $REPL_DBNAME_GPC1_SECONDARY); + + echo ""; + showDatabaseStatus($HOST_NEBULOUS, $USER_NEBULOUS, $PASSWORD_NEBULOUS, $DBNAME_NEBULOUS); + showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); + showReplicationsStatus($REPL_HOST_NEBULOUS_SECONDARY, $REPL_USER_NEBULOUS_SECONDARY, $REPL_PASSWORD_NEBULOUS_SECONDARY, $REPL_DBNAME_NEBULOUS_SECONDARY); + showReplicationsStatus($REPL_HOST_DATASTORE_SECONDARY, $REPL_USER_DATASTORE_SECONDARY, $REPL_PASSWORD_DATASTORE_SECONDARY, $REPL_DBNAME_DATASTORE_SECONDARY); + +// some mysql replication slaves to add when they are returned to service +// showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP); +// showReplicationsStatus($REPL_HOST_ISP, $REPL_USER_ISP, $REPL_PASSWORD_ISP, $REPL_DBNAME_ISP); + +// timer finished +echo ""; +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$finish = $time; +$total_time = round(($finish - $start), 3); +echo "loading in $total_time seconds."; +echo ""; +echo "
"; + +///////////////////////////////////////// +// timer begin for critical machines +///////////////////////////////////////// +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$start = $time; + +// status table at top + $PingStatus = "DOWN"; + echo $table; + echo ""; + createTableTitle("Critical machine status", 3); + + echo ""; + createTableColumnHeader("Hostname"); + createTableColumnHeader("Status"); + createTableColumnHeader("Loading time"); + showPingStatus($HOST_GATEWAY, $DBNAME_GATEWAY); + showPingStatus($HOST_GATEWAY_SECONDARY, $DBNAME_GATEWAY_SECONDARY); + showPingStatus($HOST_APACHE1, $DBNAME_APACHE1); + showPingStatus($HOST_APACHE2, $DBNAME_APACHE2); + showPingStatus($HOST_APACHE3, $DBNAME_APACHE3); + showPingStatus($HOST_APACHE4, $DBNAME_APACHE4); + showPingStatus($HOST_APACHE5, $DBNAME_APACHE5); + showPingStatus($HOST_APACHE6, $DBNAME_APACHE6); + showPingStatus($HOST_APACHE7, $DBNAME_APACHE7); + showPingStatus($HOST_APACHE8, $DBNAME_APACHE8); +// echo "ippc70 (apache node) $PingStatus 0 seconds "; // comment out if ippc70 is back + showPingStatus($HOST_PROXY, $DBNAME_PROXY); + showPingStatus($HOST_SVN, $DBNAME_SVN); + showPingStatus($HOST_CONFLUENCE, $DBNAME_CONFLUENCE); + +// timer finished +echo ""; +$time = microtime(); +$time = explode(' ', $time); +$time = $time[1] + $time[0]; +$finish = $time; +$total_time = round(($finish - $start), 3); +echo "loading in $total_time seconds."; +echo ""; +echo "
"; + + menu_end(); + +############################################################################# +############################# Functions Below ############################# +############################################################################# ########################################################################### @@ -361,90 +432,4 @@ } -########################################################################### -# -# Creates a table of summitExp against rawExp -# -########################################################################### -function createSummitDataTable($projectdb) { - - $date = date("Y-m-d"); - $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'"; - - $qry = $projectdb->query($sql); - if (dberror($qry)) { - echo "error reading newExp table
\n"; - echo "
table query : $sql
\n"; - menu_end(); - } - - // set up the table - global $table; - echo $table; - echo ""; - createTableTitle("Status of last night's data", 4); - echo ""; - createTableColumnHeader("Exposure type"); - createTableColumnHeader("At summit"); - createTableColumnHeader("Registered at MHPCC"); - createTableColumnHeader("At summit (Dropped)"); - - $msg = "No science images taken since $date"; - - // list the results - while ($qry->fetchInto($expType)) { - - $sql = "SELECT COUNT(*) FROM summitExp left join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND exp_type = '$expType[0]' and (state != 'drop' or state is null) and fault != 1042"; -# $sql = "SELECT COUNT(*) FROM summitExp WHERE dateobs > '$date' AND exp_type = '$expType[0]'"; - $qry2 = $projectdb->query($sql); - $qry2->fetchInto($summit); - - - $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name WHERE summitExp.dateobs > '$date' AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'"; - $qry2 = $projectdb->query($sql); - $qry2->fetchInto($mhpcc); - - $sql = "SELECT COUNT(*) FROM summitExp join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND exp_type = '$expType[0]' and state = 'drop'"; - $qry2 = $projectdb->query($sql); - $qry2->fetchInto($dropped); - - - #Query with 30 minutes delay to check during the night if any exposures are missing. - $datehour = gmdate("Y-m-d H:i:s"); - $sql = "SELECT COUNT(*) FROM summitExp left join pzDownloadExp using (summit_id) WHERE dateobs > '$date' AND dateobs <= date_sub('$datehour', interval 30 minute) AND exp_type = '$expType[0]' and (state != 'drop' or state is null) and fault != 1042"; - $qry2 = $projectdb->query($sql); - $qry2->fetchInto($summitdelay); - - $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name WHERE summitExp.dateobs > '$date' AND summitExp.dateobs <= date_sub('$datehour', interval 30 minute) AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'"; - $qry2 = $projectdb->query($sql); - $qry2->fetchInto($mhpccdelay); - - echo ""; - - $DLdiff = $summit[0] - $mhpcc[0]; - if ($DLdiff == 0) $error = 0; - else $error = 1; - - $delayDLdiff = $summitdelay[0] - $mhpccdelay[0]; - if ($delayDLdiff == 0) $errordelay = 0; - else $errordelay = 1; - - echo "$expType[0]"; - echo "$summit[0]"; - #createFormattedTableCell("", "", $mhpcc[0], $error, 0); - if ($delayDLdiff==0) { - createFormattedTableCell("", "", $mhpcc[0], $error, 0); - } else { - $DLStatus = "$mhpcc[0] ($delayDLdiff)"; - echo "$DLStatus"; - } - - echo "$dropped[0]"; - - echo ""; - } - - echo "\n"; - -} ########################################################################### @@ -455,4 +440,11 @@ function createChunkDataTable($projectdb) { + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin #First find the reference exposure ID to speed up the subsequent queries $date = gmdate("Y-m-d"); @@ -464,34 +456,33 @@ } $qry->fetchInto($refExpID); - - #Find the number of chunks - $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM"; - $query .= " ("; - $query .= " SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit"; - $query .= " FROM rawExp"; - $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0]"; - $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'"; - $query .= " AND comment LIKE '%visit 1%'"; - $query .= " GROUP BY chunk"; - $query .= " ) AS groupchunk"; - $query .= " LEFT JOIN"; - $query .= " ("; - $query .= " SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM "; - $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits"; - $query .= " FROM rawExp"; - $query .= " WHERE dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; - $query .= " AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'"; - $query .= " GROUP BY object, filter, chunk"; - $query .= " ) AS groupobj"; - $query .= " WHERE groupobj.Nvisits = 4 "; - $query .= " GROUP BY groupobj.chunk"; - $query .= " ) AS group4"; - $query .= " on groupchunk.chunk=group4.chunk ORDER BY dateobs"; - - #$sql = "SELECT DISTINCT substr(comment, 1, position(' ' in comment)) FROM rawExp WHERE dateobs LIKE '$date%' AND exp_id > $refExpID AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit 1%'"; + if ($refExpID[0] = 'NULL') { + $refExpID[0] = 0; + } + + #do a query of raw/chip/cam/warp counts for this night, split by chunk. + #we need to do a subquery to find the total number of skycells in warpSkyfile, as well as the number of those with a bad quality + $query = "SELECT suba.chunk,suba.Nvis1,suba.Nvis2,suba.Nvis3,suba.Nvis4,suba.Ncamgood,suba.Ncambad,subb.Nwarpgood,subb.Nwarpbad FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(if(rawExp.comment LIKE '%visit 1%',1,NULL)) AS Nvis1,"; + $query .= " count(if(rawExp.comment LIKE '%visit 2%',1,NULL)) AS Nvis2,count(if(rawExp.comment LIKE '%visit 3%',1,NULL)) AS Nvis3,"; + $query .= " count(if(rawExp.comment LIKE '%visit 4%',1,NULL)) AS Nvis4, count(if(camProcessedExp.sigma_ra <= 5 AND camProcessedExp.sigma_dec <= 5 AND camProcessedExp.quality = 0,1,NULL)) AS Ncamgood,"; + $query .= " count(if(camProcessedExp.sigma_ra > 5 OR camProcessedExp.sigma_dec > 5 OR camProcessedExp.quality > 0,1,NULL)) AS Ncambad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) JOIN camProcessedExp USING (cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID[0] AND exp_type = 'OBJECT' AND rawExp.comment like '%visit%'"; + $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY chunk ORDER BY dateobs) as suba "; + $query .= "LEFT JOIN"; + $query .= " (SELECT subc.chunk, count(if(subc.Nwarpskycellbad < subc.Nwarpskycell,1,NULL)) AS Nwarpgood, count(if(subc.Nwarpskycellbad = subc.Nwarpskycell,1,NULL)) AS Nwarpbad"; + $query .= " FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(warpSkyfile.quality) AS Nwarpskycell,count(if(warpSkyfile.quality > 0,1,NULL)) AS Nwarpskycellbad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) JOIN warpSkyfile USING (warp_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID[0] AND exp_type = 'OBJECT' AND rawExp.comment like '%visit%'"; + $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY exp_name) as subc"; + $query .= " GROUP BY chunk) AS subb "; + $query .= "ON suba.chunk=subb.chunk"; $qry = $projectdb->query($query); if (dberror($qry)) { - echo "error reading rawExp table
\n"; + echo "error querying SQL table
\n"; echo "
table query : $query
\n"; menu_end(); @@ -502,10 +493,15 @@ echo $table; echo ""; - createTableTitle("Status of last night's chunks", 5); + createTableTitle("Status of last night's chunks", 10); echo ""; createTableColumnHeader("Chunk"); - createTableColumnHeader("Nobs_1stvisit"); - createTableColumnHeader("Nobs_4visits(bad cam)"); - createTableColumnHeader("Ndiffs_published"); + createTableColumnHeader("Nobs"); + createTableColumnHeader("Ncam"); + createTableColumnHeader("Nwrp"); + createTableColumnHeader("wrp_pub"); + createTableColumnHeader("Nwwd"); + createTableColumnHeader("wwd_pub"); + createTableColumnHeader("Nwsd"); + createTableColumnHeader("wsd_pub"); createTableColumnHeader("Completion"); @@ -514,80 +510,133 @@ // list the results while ($qry->fetchInto($row)) { - $Nfirstvisits = 0; - $Nfourvisits = 0; - $badcam = 0; + $Nvis1 = 0; + $Nvis2 = 0; + $Nvis3 = 0; + $Nvis4 = 0; + $Ncamgood = 0; + $Ncambad = 0; + $Nwarpgood = 0; + $Nwarpbad = 0; + $NexpWWdiff = 0; + $Npubwarps = 0; $chunkname = $row[0]; - if($row[1] > 0) $Nfirstvisits = $row[1]; - if($row[2] > 0) $Nfourvisits = $row[2]; - - $query2 = " SELECT warp_id FROM "; - $query2 .= " warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; - $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; - $query2 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'"; + if($row[1] > 0) $Nvis1 = $row[1]; + if($row[2] > 0) $Nvis2 = $row[2]; + if($row[3] > 0) $Nvis3 = $row[3]; + if($row[4] > 0) $Nvis4 = $row[4]; + if($row[5] > 0) $Ncamgood = $row[5]; + if($row[6] > 0) $Ncambad = $row[6]; + if($row[7] > 0) $Nwarpgood = $row[7]; + if($row[8] > 0) $Nwarpbad = $row[8]; + + #set flags if present + if($Ncambad >= 1) $badcamflag = 1; + if($Nwarpbad >= 1) $badwarpflag = 1; + + $query1 = "SELECT suba.chunk, SUM(IF(suba.Nvisits = 2,1,NULL)) as totvis2, SUM(IF(suba.Nvisits = 3 OR suba.Nvisits = 4,2,NULL)) as totvis34 FROM"; + $query1 .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits FROM"; + $query1 .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) WHERE dateobs like '$date%'"; + $query1 .= " AND exp_id > $refExpID[0] AND exp_type = 'OBJECT' AND rawExp.comment like '%visit%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query1 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' group by object) as suba group by chunk;"; + $qry1 = $projectdb->query($query1); + while ($qry1->fetchInto($row)) {$NexpWWdiff = $row[1] + $row[2];} + + #find the unique diff_ids for each chunk, to be matched to the diffRun catalog + $query2 = " SELECT diff_id,warp1,stack1,warp2,stack2 FROM "; + $query2 .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID[0] AND exp_type = 'OBJECT' AND rawExp.comment like '%visit%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query2 .= " AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL' AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' GROUP By diff_id"; $qry2 = $projectdb->query($query2); - $Ndiffs = 0; + $NWWdiffgood = 0; + $NWWdiffbad = 0; + $NWWdiffpub = 0; + + $NWSdiffgood = 0; + $NWSdiffbad = 0; + $NWSdiffpub = 0; + $wrpcnt = array(); + while ($qry2->fetchInto($row2)) { - $query3 = " SELECT diff_id FROM "; - $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)"; - $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; - $qry3 = $projectdb->query($query3); - while ($qry3->fetchInto($row3)) { - if ($row3[0] !=0) $Ndiffs ++; - } - - $query3 = " SELECT diff_id FROM "; - $query3 .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)"; - $query3 .= " WHERE stack2 IS NULL AND publishRun.state LIKE 'full' AND warp_id='$row2[0]' GROUP BY diff_id"; - $qry3 = $projectdb->query($query3); - while ($qry3->fetchInto($row3)) { - if ($row3[0] !=0) $Ndiffs ++; - } + #distinguish between WW and WS diffs based on stack2 + if ($row2[4] == 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + $qry3 = $projectdb->query($query3); + + while ($qry3->fetchInto($row3)) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) $NWWdiffgood ++; + if ($row3[1]==$row3[0]) $NWWdiffbad ++; + if ($row3[2]==$row3[0]) $NWWdiffpub ++; + + #if this wwdiff is published, record its corresponding warp_ids to see how many unique ones there are + if ($row3[2]==$row3[0]) $wrpcnt{$row2[1]}++; # record the presence of warp1 + if ($row3[2]==$row3[0]) $wrpcnt{$row2[3]}++; # record the presence of warp2 + } + } + + if ($row2[4] > 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + $qry3 = $projectdb->query($query3); + + while ($qry3->fetchInto($row3)) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) $NWSdiffgood ++; + if ($row3[1]==$row3[0]) $NWSdiffbad ++; + if ($row3[2]==$row3[0]) $NWSdiffpub ++; + } + } } - - if ($Ndiffs != ($Nfirstvisits*4)) $complete = "chunk not done"; - if ($Ndiffs == ($Nfirstvisits*4)) $complete = "chunk done"; - - if ($Nfirstvisits != $Nfourvisits) { - $Ndiffs .= "/? "; + #pull out the unique warp_ids + $Npubwarps = count($wrpcnt); + + #check for bad diffs + if(($NWWdiffbad >= 1) or ($NWSdiffbad >= 1)) $baddiffflag = 1; + + if ($Nvis1 != $Nvis4) { + if ($NWWdiffpub != $NexpWWdiff) $complete = "chunk incomplete, not done"; + if ($NWWdiffpub == $NexpWWdiff) $complete = "chunk incomplete, done"; } - if ($Nfirstvisits == $Nfourvisits) { - $Ndiffs .= "/"; - $Ndiffs .= $Nfirstvisits*4; + if ($Nvis1 == $Nvis4) { + if ($NWWdiffpub != $NexpWWdiff) $complete = "chunk complete, not done"; + if ($NWWdiffpub == $NexpWWdiff) $complete = "chunk complete, done"; + } - #also query for bad quality camRun exposures - $query4 = "SELECT count(*) "; - $query4 .= " FROM rawExp"; - $query4 .= " JOIN chipRun USING (exp_id)"; - $query4 .= " JOIN camRun USING (chip_id)"; - $query4 .= " JOIN camProcessedExp USING (cam_id)"; - $query4 .= " WHERE dateobs LIKE '$date%'"; - $query4 .= " AND exp_id > $refExpID[0]"; - $query4 .= " AND exp_type = 'OBJECT'"; - $query4 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'"; - $query4 .= " AND (camProcessedExp.quality > 0)"; - $qry4 = $projectdb->query($query4); - $qry4->fetchInto($badcam); - - - echo ""; echo "$chunkname"; - echo "$Nfirstvisits"; - if ($badcam[0]==0) { - echo "$Nfourvisits"; - } else { - $DLStatus = "$Nfourvisits ($badcam[0])"; - echo "$DLStatus"; - } - echo "$Ndiffs"; + echo "$Nvis1/$Nvis2/$Nvis3/$Nvis4"; + echo "$Ncamgood/$Ncambad"; + echo "$Nwarpgood/$Nwarpbad"; + echo "$Npubwarps"; + echo "$NWWdiffgood/$NWWdiffbad"; + echo "$NWWdiffpub"; + echo "$NWSdiffgood/$NWSdiffbad"; + echo "$NWSdiffpub"; echo "$complete"; echo ""; } - echo "\n"; + + + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
"; } @@ -631,4 +680,11 @@ $selectedLabel, $selectedStage, $plotType) { + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin $isUpdate = ($selectedMode == "update"); global $headerColor; @@ -754,5 +810,16 @@ echo "\n"; - echo "\n"; + + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
"; } @@ -902,4 +969,11 @@ function createDatesTable($db, $proj) { + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin global $table; echo $table; @@ -914,5 +988,16 @@ createServerDateRow($db, $proj, "registration"); - echo "\n"; + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
"; + } @@ -956,4 +1041,11 @@ function createServersTable($pass, $proj, $selectedMode, $db, $servers, $selectedLabel, $selectedStage, $plotType) { + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin // set up table columns global $table; @@ -1042,5 +1134,15 @@ } - echo "\n"; + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
\n"; } @@ -1084,5 +1186,6 @@ function getServerStatus($db,$proj, $server, &$alive, &$running) { - $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1"; +# $sql = "SELECT alive, running FROM science_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1"; + $sql = "SELECT alive, running FROM live_servers WHERE server LIKE '$server' AND telescope LIKE '$proj' ORDER BY timestamp DESC LIMIT 1"; if($debug){echo "$sql
";} @@ -1203,4 +1306,11 @@ ########################################################################### function showPingStatus($Host, $DatabaseName) { + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin exec("ping -c 1 $Host", $res, $rval); if ($rval == 0){ @@ -1209,7 +1319,186 @@ $PingStatus = "DOWN"; } - echo "$DatabaseName$PingStatus"; -} - + echo "$DatabaseName"; + + echo "$PingStatus"; + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "$total_time seconds"; + $start= $finish; + echo ""; +} + + +########################################################################### +# +# Creates a table of summitExp against rawExp +# +########################################################################### +function createSummitDataTable($projectdb) { + + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin + $date = date("Y-m-d"); + $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'"; + + $qry = $projectdb->query($sql); + if (dberror($qry)) { + echo "error reading newExp table
\n"; + echo "
table query : $sql
\n"; + menu_end(); + } + + // set up the table + global $table; + echo $table; + echo ""; + createTableTitle("Data status > $date", 4); + echo ""; + createTableColumnHeader("Exposure type"); + createTableColumnHeader("At summit"); + createTableColumnHeader("Registered at MHPCC"); + createTableColumnHeader("Summit (Drop/Null)"); + + $msg = "No science images taken since $date"; + + // list the results + while ($qry->fetchInto($expType)) { + + $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id) + WHERE dateobs > '$date' AND exp_type = '$expType[0]'"; + // AND state != 'drop' AND state is not null AND fault != 1042"; + $qry2 = $projectdb->query($sql); + $qry2->fetchInto($summit); + + $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name + WHERE summitExp.dateobs > '$date' AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'"; + $qry2 = $projectdb->query($sql); + $qry2->fetchInto($mhpcc); + + $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id) + WHERE dateobs > '$date' AND exp_type = '$expType[0]' AND (state = 'drop' OR state is null)"; + $qry2 = $projectdb->query($sql); + $qry2->fetchInto($dropped); + + + #Query with 30 minutes delay to check during the night if any exposures are missing. + $datehour = gmdate("Y-m-d H:i:s"); + $sql = "SELECT COUNT(*) FROM summitExp LEFT JOIN pzDownloadExp USING (summit_id) + WHERE dateobs > '$date' AND dateobs <= date_sub('$datehour', interval 30 minute) + AND exp_type = '$expType[0]' AND (state != 'drop' OR state is null) and fault != 1042"; + + $qry2 = $projectdb->query($sql); + $qry2->fetchInto($summitdelay); + + $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name + WHERE summitExp.dateobs > '$date' AND summitExp.dateobs <= date_sub('$datehour', interval 30 minute) + AND rawExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'"; + $qry2 = $projectdb->query($sql); + $qry2->fetchInto($mhpccdelay); + + echo ""; + + $DLdiff = $summit[0] - $mhpcc[0]; + if ($DLdiff == 0) $error = 0; + else $error = 1; + + $delayDLdiff = $summitdelay[0] - $mhpccdelay[0]; + if ($delayDLdiff == 0) $errordelay = 0; + else $errordelay = 1; + + echo "$expType[0]"; + echo "$summit[0]"; + if ($delayDLdiff==0) { + createFormattedTableCell("", "", $mhpcc[0], $error, 0); + } else { + $DLStatus = "$mhpcc[0] ($delayDLdiff)"; + echo "$DLStatus"; + } + + echo "$dropped[0]"; + + echo ""; + } + + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
"; +} + + +function createSummitDataTable2($projectdb) { + + // timer start + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $start = $time; + + // function begin + $date = date("Y-m-d"); + // set up the table + global $table; + global $columnHeaderColor; + echo $table; + echo ""; + createTableTitle("Data status > $date", 4); + echo ""; + createTableColumnHeader("Exp_type at summit"); + createTableColumnHeader("Status at summit"); + createTableColumnHeader("Status at mhpcc"); + createTableColumnHeader("Number"); + $sql = "SELECT summitExp.exp_type,pzDownloadExp.state,rawExp.state,COUNT(*) FROM summitExp + LEFT JOIN pzDownloadExp USING (summit_id) LEFT JOIN rawExp on summitExp.exp_name = rawExp.exp_name + WHERE summitExp.dateobs > '$date' group by summitExp.exp_type,pzDownloadExp.state,rawExp.state"; + $qry = $projectdb->query($sql); + while ($qry->fetchInto($tmp)) { + if ($tmp[1] == NULL) { + echo "$tmp[0] null null $tmp[3]"; + } + else { + if ($tmp[1] == 'run') { + echo "$tmp[0] loading null $tmp[3]"; + } + else { + if ($tmp[2] == NULL) { + echo "$tmp[0] $tmp[1] loading $tmp[3]"; + } + else { + echo "$tmp[0] $tmp[1] downloaded $tmp[3]"; + } + } + } + } + + // timer finished + echo ""; + $time = microtime(); + $time = explode(' ', $time); + $time = $time[1] + $time[0]; + $finish = $time; + $total_time = round(($finish - $start), 3); + echo "loading in $total_time seconds."; + $start= $finish; + echo ""; + echo "
"; +} ?> Index: branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.dat =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.dat (revision 41170) +++ branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.dat (revision 41170) @@ -0,0 +1,11 @@ +input | ipp.menu.dat + +menutop | menutop | plain |   | +menulink | menuselect | link | Full Force Summary | fullforceSummary.php +menulink | menuselect | link | Full Force Run | fullforceRun.php +menulink | menuselect | link | Full Force Input Skyfile | fullforceInputSkyfile.php +menulink | menuselect | link | Full Force Processed Skyfile | fullforceProcessedSkyfile.php +menulink | menuselect | link | Full Force Failed Skyfile | fullforceFailedSkyfile.php +menulink | menuselect | link | Full Force Failed | failedFullforce.php + + Index: branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.php =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.php (revision 41170) +++ branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.ffs.php (revision 41170) @@ -0,0 +1,17 @@ +\n"; + +menu_end(); + +?> Index: branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.php =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.php (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/raw/ipp.php (revision 41170) @@ -351,5 +351,6 @@ // loop over the lines in the file and generate lines in the menu while ($line = fgetcsv ($file, 1024, "|")) { - if (ereg ('^[:blank:]*#', $line[0])) continue; + if (preg_match ("/#/", $line[0])) continue; + // if (ereg ('^[:blank:]*#', $line[0])) continue; if (count($line) < 2) continue; Index: branches/eam_branches/ipp-20191011/ippMonitor/raw/site.php.in =================================================================== --- branches/eam_branches/ipp-20191011/ippMonitor/raw/site.php.in (revision 41169) +++ branches/eam_branches/ipp-20191011/ippMonitor/raw/site.php.in (revision 41170) @@ -13,4 +13,8 @@ $DBI = "@DBI@"; +// location of plots produced by czarpoll.pl +$CZARPLOTDIR = "./czartool_plots"; +$METRICSPLOTDIR = "./ippMetrics"; + // insert config.dat here. this contains a number of system-specific things like // the names of database machines and users Index: branches/eam_branches/ipp-20191011/ippScripts/scripts/night_report.pl =================================================================== --- branches/eam_branches/ipp-20191011/ippScripts/scripts/night_report.pl (revision 41169) +++ branches/eam_branches/ipp-20191011/ippScripts/scripts/night_report.pl (revision 41170) @@ -17,4 +17,5 @@ 'very-verbose' => \$verbose2, # Print to stdout 'chunks' => \$chunks, # give information on the current status of the chunks + 'mopsreport' => \$mopsreport, # return a report in the new style requested by MOPS ) or pod2usage( 2 ); @@ -40,7 +41,4 @@ $dataGroupDate=~ s/\.//g; -print "\n\n\n"; -print "--- Report for $dbname : $date ---\n\n"; - # set up the database connection: my $dsn = "DBI:mysql:host=$dbhost;database=$dbname"; @@ -69,4 +67,261 @@ ################## +# testing how to get more information parsed by chunk +if ($mopsreport or $report) { + print "\n\n\n"; + print "--- MOPS report for $dbname : $date ---\n\n"; + + my $badcamflag = 0; + my $badwarpflag = 0; + my $baddiffflag = 0; + + #do a query of raw/chip/cam/warp counts for this night, split by chunk. + #We want to stick to only MOPS products, so we do labels %SS%, %EU% and %Bright.% (to avoid BrightTwi and Bright3PI which are not in quads) + #we need to do a subquery to find the total number of skycells in warpSkyfile, as well as the number of those with a bad quality + my $query = "SELECT suba.chunk,suba.Nvis1,suba.Nvis2,suba.Nvis3,suba.Nvis4,suba.Ncamgood,suba.Ncambad,subb.Nwarpgood,subb.Nwarpbad FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(if(rawExp.comment LIKE '%visit 1%',1,NULL)) AS Nvis1,"; + $query .= " count(if(rawExp.comment LIKE '%visit 2%',1,NULL)) AS Nvis2,count(if(rawExp.comment LIKE '%visit 3%',1,NULL)) AS Nvis3,"; + $query .= " count(if(rawExp.comment LIKE '%visit 4%',1,NULL)) AS Nvis4, count(if(camProcessedExp.sigma_ra <= 5 AND camProcessedExp.sigma_dec <= 5 AND camProcessedExp.quality = 0,1,NULL)) AS Ncamgood,"; + $query .= " count(if(camProcessedExp.sigma_ra > 5 OR camProcessedExp.sigma_dec > 5 OR camProcessedExp.quality > 0,1,NULL)) AS Ncambad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) JOIN camProcessedExp USING (cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY chunk ORDER BY dateobs) as suba "; + $query .= "LEFT JOIN"; + $query .= " (SELECT subc.chunk, count(if(subc.Nwarpskycellbad < subc.Nwarpskycell,1,NULL)) AS Nwarpgood, count(if(subc.Nwarpskycellbad = subc.Nwarpskycell,1,NULL)) AS Nwarpbad"; + $query .= " FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(warpSkyfile.quality) AS Nwarpskycell,count(if(warpSkyfile.quality > 0,1,NULL)) AS Nwarpskycellbad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) JOIN warpSkyfile USING (warp_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY exp_name) as subc"; + $query .= " GROUP BY chunk) AS subb "; + $query .= "ON suba.chunk=subb.chunk"; + + my $result = &mysql_select ($query); + + printf "%18s | %11s | %4s | %4s | %7s | %4s | %7s | %4s | %7s |\n", "chunk", "Nobs", "Ncam","Nwrp","Nwrp_pub", "Nwwd", "Nwwd_pub", "Nwsd", "Nwsd_pub"; + while (@row = $result->fetchrow_array()) { + my $Nvis1 = 0; + my $Nvis2 = 0; + my $Nvis3 = 0; + my $Nvis4 = 0; + my $Ncamgood = 0; + my $Ncambad = 0; + my $Nwarpgood = 0; + my $Nwarpbad = 0; + my $NexpWWdiff = 0; + my $Npubwarps = 0; + my $uniwarps = undef; + + my $chunkname = $row[0]; + if($row[1] gt 0) {$Nvis1 = $row[1]}; + if($row[2] gt 0) {$Nvis2 = $row[2]}; + if($row[3] gt 0) {$Nvis3 = $row[3]}; + if($row[4] gt 0) {$Nvis4 = $row[4]}; + if($row[5] gt 0) {$Ncamgood = $row[5]}; + if($row[6] gt 0) {$Ncambad = $row[6]}; + if($row[7] gt 0) {$Nwarpgood = $row[7]}; + if($row[8] gt 0) {$Nwarpbad = $row[8]}; + + #set flags if present + if($Ncambad >= 1) {$badcamflag = 1}; + if($Nwarpbad >= 1) {$badwarpflag = 1}; + + #Find the total published number of unique warps for this chunk + #my $query0 = " SELECT count(DISTINCT warpRun.warp_id) FROM "; + #$query0 .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) JOIN publishRun ON (diffInputSkyfile.diff_id = stage_id)"; + #$query0 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%')"; + #$query0 .= " AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL' AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' group by warpRun.warp_id"; + #my $result0 = &mysql_select ($query0); + #while (@row = $result0->fetchrow_array()) {$Npubwarps = $row[0];} + + + #Find the expected number of diffs for this chunk + my $query1 = "SELECT suba.chunk, SUM(IF(suba.Nvisits = 2,1,NULL)) as totvis2, SUM(IF(suba.Nvisits = 3 OR suba.Nvisits = 4,2,NULL)) as totvis34 FROM"; + $query1 .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits FROM"; + $query1 .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) WHERE dateobs like '$date%'"; + $query1 .= " AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query1 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' group by object) as suba group by chunk;"; + my $result1 = &mysql_select ($query1); + while (@row = $result1->fetchrow_array()) {$NexpWWdiff = $row[1] + $row[2];} + + #find the unique diff_ids for each chunk, to be matched to the diffRun catalog + my $query2 = " SELECT diff_id,warp1,stack1,warp2,stack2 FROM "; + $query2 .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%')"; + $query2 .= " AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL' AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' GROUP By diff_id"; + my $result2 = &mysql_select ($query2); + + my $NWWdiffgood = 0; + my $NWWdiffbad = 0; + my $NWWdiffpub = 0; + + my $NWSdiffgood = 0; + my $NWSdiffbad = 0; + my $NWSdiffpub = 0; + my %wrpcnt = (); + + while (@row2 = $result2->fetchrow_array()) { + #distinguish between WW and WS diffs based on stack2 + if ($row2[4] == 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) {$NWWdiffgood ++;} + if ($row3[1]==$row3[0]) {$NWWdiffbad ++;} + if ($row3[2]==$row3[0]) {$NWWdiffpub ++;} + + #if this wwdiff is published, record its corresponding warp_ids to see how many unique ones there are + if ($row3[2]==$row3[0]) {$wrpcnt{$row2[1]}++;} # record the presence of warp1 + if ($row3[2]==$row3[0]) {$wrpcnt{$row2[3]}++;} # record the presence of warp2 + } + } + + if ($row2[4] > 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) {$NWSdiffgood ++;} + if ($row3[1]==$row3[0]) {$NWSdiffbad ++;} + if ($row3[2]==$row3[0]) {$NWSdiffpub ++;} + } + } + } + #pull out the unique warp_ids + $Npubwarps = keys %wrpcnt; + + #check for bad diffs + if(($NWWdiffbad >= 1) or ($NWSdiffbad >= 1)) {$baddiffflag = 1;} + + if ($Nvis1 ne $Nvis4) { + if ($NWWdiffpub ne $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk incomplete, not done";} + if ($NWWdiffpub eq $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk incomplete, done";} + } + if ($Nvis1 eq $Nvis4) { + if ($NWWdiffpub ne $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk complete, not done";} + if ($NWWdiffpub eq $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk complete, done";} + } + } + + print "\n"; + + #----------------------------------------- + #list bad quality cam exposures + if ($badcamflag != 0) { + my $query = "SELECT exp_name,cam_id, camProcessedExp.quality, camProcessedExp.sigma_ra, camProcessedExp.sigma_dec, comment"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " JOIN camRun USING (chip_id)"; + $query .= " JOIN camProcessedExp USING (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " AND (camProcessedExp.quality > 0 OR camProcessedExp.sigma_ra > 5 OR camProcessedExp.sigma_dec > 5)"; + $query .= " ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Bad quality camRun exposures:\n"; + printf "%11s | %7s | %7s | %9s | %9s | %15s\n", "exp_name","cam_id", "quality", "sigma_ra", "sigma_dec", "comment"; + $doHeader = 0; + } + printf "%11s | %7s | %7d | %9s | %9s | %15s\n", $row[0], $row[1], $row[2], $row[3], $row[4], $row[5]; + } + if (!$doHeader) { print "\n"; } + } + + #----------------------------------------- + #list bad quality warp exposures + if ($badwarpflag != 0) { + $query = "SELECT exp_name,warp_id,count(warpSkyfile.quality) AS Nwarpskycell,count(if(warpSkyfile.quality > 0,1,NULL)) AS Nwarpskycellbad, comment"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) JOIN warpSkyfile USING (warp_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY exp_name ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($row[2] == $row[3]) { + if ($doHeader) { + print "Bad quality warpRun exposures:\n"; + printf "%11s | %7s | %15s\n", "exp_name", "warp_id", "comment"; + $doHeader = 0; + } + printf "%11s | %7d | %15s \n", $row[0], $row[1], $row[4]; + } + } + if (!$doHeader) { print "\n"; } + } + + #----------------------------------------- + #list bad quality diff exposures + if ($baddiffflag != 0) { + my $query = " SELECT diff_id,warp1,stack1,warp2,stack2,exp_name,comment FROM "; + $query .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP By diff_id"; + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Bad quality diffRun exposures:\n"; + printf "%11s | %7s | %7s | %7s | %7s | %7s | %15s\n", "exp_name", "diff_id", "warp1", "stack1", "warp2", "stack2", "comment"; + $doHeader = 0; + } + + #distinguish between WW and WS diffs based on stack2 + if ($row[4] == 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]==$row3[0]) {printf "%11s | %7d | %7d | %7d | %7d | %7d | %15s \n", $row[5], $row[0], $row[1], $row[2], $row[3], $row[4], $row[6];} + } + } + + if ($row[4] > 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]==$row3[0]) {printf "%11s | %7d | %7d | %7d | %7d | %7d | %15s \n", $row[5], $row[0], $row[1], $row[2], $row[3], $row[4], $row[6];} + } + } + } + if (!$doHeader) { print "\n"; } + } +} + + +print "\n\n\n"; +print "--- IPP Report for $dbname : $date ---\n\n"; + + + ### testing the diff grouping -- this does not work for the Euclid data taken 2018.07.29 because the OBJECT ### field is missing @@ -574,5 +829,5 @@ print "\n"; print "Listing objects with 1< Nvisits < 4:\n"; - printf "| %8s | %8s | %17s | %8s |\n", "object", "filter", "chunk", "Nvisit"; + printf "| %12s | %8s | %19s | %8s |\n", "object", "filter", "chunk", "Nvisit"; my $query = "SELECT * FROM"; @@ -594,9 +849,9 @@ $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; - $query .= " ) AS tableVisits WHERE Nvisit > 1 AND Nvisit < 4 ORDER BY Nvisit "; + $query .= " ) AS tableVisits WHERE Nvisit > 1 AND Nvisit < 4 ORDER BY chunk,Nvisit "; my $result = &mysql_select ($query); while (@row = $result->fetchrow_array()) { - printf "| %8s | %8s | %17s | %8s |\n", $row[0], $row[1], $row[2], $row[3]; + printf "| %12s | %8s | %19s | %8s |\n", $row[0], $row[1], $row[2], $row[3]; } print "\n"; @@ -864,5 +1119,5 @@ # count the number of chunks, and their status: -if ($verbose or $chunks) { +if ($verbose or $report) { print "Current status of SS and Euclid chunks:\n"; @@ -937,4 +1192,5 @@ + exit 0; Index: branches/eam_branches/ipp-20191011/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/share/pxadmin_create_tables.sql (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/share/pxadmin_create_tables.sql (revision 41170) @@ -524,4 +524,10 @@ background_model SMALLINT, astrom_chips BIGINT, + ast_r0 FLOAT, + ast_d0 FLOAT, + ast_t0 FLOAT, + ast_s0 FLOAT, + ast_rs FLOAT, + ast_ds FLOAT, PRIMARY KEY(cam_id), KEY(fault), @@ -539,4 +545,5 @@ stage VARCHAR(64), stage_id BIGINT, + stage_extra1 INT, state VARCHAR(64), workdir VARCHAR(255), @@ -1447,5 +1454,5 @@ proj_id BIGINT, registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - timestamp TIMESTAMP, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, fault SMALLINT, PRIMARY KEY(req_id), @@ -2202,4 +2209,5 @@ dvodb VARCHAR(255), ubercal_file VARCHAR(255), + accessLevelMin INT DEFAULT 0, PRIMARY KEY(rel_id), UNIQUE KEY(surveyID, release_name), Index: branches/eam_branches/ipp-20191011/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/addtool.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/addtool.c (revision 41170) @@ -1668,4 +1668,5 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + if (!psDBTransaction(config->dbh)) { Index: branches/eam_branches/ipp-20191011/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/camtool.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/camtool.c (revision 41170) @@ -524,4 +524,11 @@ PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false); + PXOPT_LOOKUP_F32(ast_r0, config->args, "-ast_r0", false, false); + PXOPT_LOOKUP_F32(ast_d0, config->args, "-ast_d0", false, false); + PXOPT_LOOKUP_F32(ast_t0, config->args, "-ast_t0", false, false); + PXOPT_LOOKUP_F32(ast_s0, config->args, "-ast_s0", false, false); + PXOPT_LOOKUP_F32(ast_rs, config->args, "-ast_rs", false, false); + PXOPT_LOOKUP_F32(ast_ds, config->args, "-ast_ds", false, false); + // we store actual detection efficiency by adding in zpt_obs PXOPT_LOOKUP_F32(deteff_inst, config->args, "-deteff_inst", false, false); @@ -695,5 +702,12 @@ quality, background_model, - astrom_chips + astrom_chips, + + ast_r0, + ast_d0, + ast_t0, + ast_s0, + ast_rs, + ast_ds ); Index: branches/eam_branches/ipp-20191011/ippTools/src/camtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/camtoolConfig.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/camtoolConfig.c (revision 41170) @@ -207,4 +207,11 @@ psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-astrom_chips", 0, "chips with successful astrom", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_r0", 0, "define boresite offset in RA", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_d0", 0, "define boresite offset in DEC", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_t0", 0, "define boresite angle", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_s0", 0, "define bosite scale", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_rs", 0, "define bosite scatter in RA", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ast_ds", 0, "define bosite scatter in DEC", NAN); + // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); Index: branches/eam_branches/ipp-20191011/ippTools/src/pxcam.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/pxcam.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/pxcam.c (revision 41170) @@ -186,14 +186,14 @@ // Here follows the software version and mask statistics which shouldn't be propagated "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", - "NULL", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", note ? note : "NULL", (long long)chip_id Index: branches/eam_branches/ipp-20191011/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/stacktool.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/stacktool.c (revision 41170) @@ -1211,5 +1211,5 @@ PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); Index: branches/eam_branches/ipp-20191011/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/ipp-20191011/ippTools/src/stacktoolConfig.c (revision 41169) +++ branches/eam_branches/ipp-20191011/ippTools/src/stacktoolConfig.c (revision 41170) @@ -207,5 +207,5 @@ psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); - psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); #ifdef notdef // These don't work so omit (for now) We probably should create a different mode for this type of search. Index: branches/eam_branches/ipp-20191011/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/ipp-20191011/ippconfig/recipes/ppImage.config (revision 41169) +++ branches/eam_branches/ipp-20191011/ippconfig/recipes/ppImage.config (revision 41170) @@ -1406,26 +1406,26 @@ # Photometry and astrometry only (for pre-reduced data) PPIMAGE_PA METADATA - BASE.FITS BOOL FALSE # Save base detrended image? - BASE.MASK.FITS BOOL FALSE # Save base detrended image? - BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? - CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? - CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? - CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? - OVERSCAN BOOL FALSE # Overscan subtraction - NONLIN BOOL FALSE # Non-linearity correction; not implemented - BIAS BOOL FALSE # Bias subtraction - DARK BOOL FALSE # Dark subtraction - SHUTTER BOOL FALSE # Shutter correction - FLAT BOOL FALSE # Flat-field normalisation - MASK BOOL FALSE # Mask bad pixels - MASK.BUILD BOOL FALSE # Build internal mask? - FRINGE BOOL FALSE # Fringe subtraction - BIN1.FITS BOOL TRUE # Save 1st binned chip image? - BIN2.FITS BOOL TRUE # Save 2nd binned chip image? - BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? - BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? - PHOTOM BOOL TRUE # Source identification and photometry - ASTROM.CHIP BOOL FALSE # Astrometry per chip? - ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? END @@ -2593,2 +2593,24 @@ END +# for localhost test without bias, dark, flat, mask - ccl 2019.11.06 +CHIP_LOCALHOSTTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask image + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? +END Index: branches/eam_branches/ipp-20191011/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- branches/eam_branches/ipp-20191011/ippconfig/recipes/ppStatsFromMetadata.config (revision 41169) +++ branches/eam_branches/ipp-20191011/ippconfig/recipes/ppStatsFromMetadata.config (revision 41170) @@ -276,4 +276,11 @@ ENTRY VAL ASTROM_CHIPS U64 BITWISEOR -astrom_chips + ENTRY VAL AST_R0 F32 CONSTANT -ast_r0 + ENTRY VAL AST_D0 F32 CONSTANT -ast_d0 + ENTRY VAL AST_T0 F32 CONSTANT -ast_t0 + ENTRY VAL AST_S0 F32 CONSTANT -ast_s0 + ENTRY VAL AST_RS F32 CONSTANT -ast_rs + ENTRY VAL AST_DS F32 CONSTANT -ast_ds + ENTRY VAL ZPT_OBS F32 SAMPLE_MEAN -zpt_obs ENTRY VAL ZPT_ERR F32 RMS -zpt_err Index: branches/eam_branches/ipp-20191011/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/ipp-20191011/ippconfig/recipes/reductionClasses.mdc (revision 41169) +++ branches/eam_branches/ipp-20191011/ippconfig/recipes/reductionClasses.mdc (revision 41170) @@ -1873,2 +1873,13 @@ END +# for localhost test without bias, dark, flat, mask - ccl 2019.11.06 +LOCALHOSTTEST METADATA + CHIP_PPIMAGE STR LOCALHOSTTEST + CHIP_PSPHOT STR CHIP + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + PSASTRO STR DEFAULT_RECIPE + # this is for running addstar + ADDSTAR STR ADDSTAR +END + Index: branches/eam_branches/ipp-20191011/psastro/src/psastroMetadataStats.c =================================================================== --- branches/eam_branches/ipp-20191011/psastro/src/psastroMetadataStats.c (revision 41169) +++ branches/eam_branches/ipp-20191011/psastro/src/psastroMetadataStats.c (revision 41170) @@ -38,4 +38,10 @@ psMetadataItemSupplement(&status, stats, header, "CERROR"); psMetadataItemSupplement(&status, stats, header, "NASTRO"); + psMetadataItemSupplement(&status, stats, header, "AST_R0"); + psMetadataItemSupplement(&status, stats, header, "AST_D0"); + psMetadataItemSupplement(&status, stats, header, "AST_T0"); + psMetadataItemSupplement(&status, stats, header, "AST_S0"); + psMetadataItemSupplement(&status, stats, header, "AST_RS"); + psMetadataItemSupplement(&status, stats, header, "AST_DS"); }