Index: /branches/eam_branches/relphot.20210521/Makefile =================================================================== --- /branches/eam_branches/relphot.20210521/Makefile (revision 41602) +++ /branches/eam_branches/relphot.20210521/Makefile (revision 41602) @@ -0,0 +1,156 @@ +default: relphot relphot_client +help: + @echo "make options: relphot (default)" + +include ../../Makefile.System +HOME = $(ROOT)/src/relphot +BIN = $(HOME)/bin +LIB = $(HOME)/lib +SRC = $(HOME)/src +MAN = $(HOME)/doc +INC = $(HOME)/include +include ../../Makefile.Common + +# programs may add their own internal requirements here +FULL_CFLAGS = $(BASE_CFLAGS) +FULL_CPPFLAGS = $(BASE_CPPFLAGS) +FULL_LDFLAGS = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS) + +relphot: $(BIN)/relphot.$(ARCH) +relphot_client: $(BIN)/relphot_client.$(ARCH) + +test_liststats: $(BIN)/test_liststats.$(ARCH) + +test_fitirls: $(BIN)/test_fitirls.$(ARCH) + +install: $(DESTBIN)/relphot $(DESTBIN)/relphot_client + +RELPHOT = \ +$(SRC)/StatDataSetOps.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/GridOps.$(ARCH).o \ +$(SRC)/ImageOps.$(ARCH).o \ +$(SRC)/ImageSubset.$(ARCH).o \ +$(SRC)/BrightCatalog.$(ARCH).o \ +$(SRC)/MosaicOps.$(ARCH).o \ +$(SRC)/TGroupOps.$(ARCH).o \ +$(SRC)/ReadTGroupFile.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/StarOps.$(ARCH).o \ +$(SRC)/WarpImageMaps.$(ARCH).o \ +$(SRC)/StackImageMaps.$(ARCH).o \ +$(SRC)/RepairWarpMeasures.$(ARCH).o \ +$(SRC)/myIndex.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/help.$(ARCH).o \ +$(SRC)/extra.$(ARCH).o \ +$(SRC)/bcatalog.$(ARCH).o \ +$(SRC)/global_stats.$(ARCH).o \ +$(SRC)/initialize.$(ARCH).o \ +$(SRC)/fit1d_irls.$(ARCH).o \ +$(SRC)/liststats.$(ARCH).o \ +$(SRC)/load_catalogs.$(ARCH).o \ +$(SRC)/load_images.$(ARCH).o \ +$(SRC)/save_images.$(ARCH).o \ +$(SRC)/MagResidSave.$(ARCH).o \ +$(SRC)/synthetic_mags.$(ARCH).o \ +$(SRC)/plot_scatter.$(ARCH).o \ +$(SRC)/plotstuff.$(ARCH).o \ +$(SRC)/reload_catalogs.$(ARCH).o \ +$(SRC)/relphot.$(ARCH).o \ +$(SRC)/relphot_images.$(ARCH).o \ +$(SRC)/relphot_objects.$(ARCH).o \ +$(SRC)/relphot_parallel_regions.$(ARCH).o \ +$(SRC)/relphot_parallel_images.$(ARCH).o \ +$(SRC)/relphot_synthphot.$(ARCH).o \ +$(SRC)/relphot_synthphot_catalog.$(ARCH).o \ +$(SRC)/synthetic_zpts.$(ARCH).o \ +$(SRC)/select_images.$(ARCH).o \ +$(SRC)/assign_images.$(ARCH).o \ +$(SRC)/launch_region_hosts.$(ARCH).o \ +$(SRC)/ImageTable.$(ARCH).o \ +$(SRC)/ImageMagIO.$(ARCH).o \ +$(SRC)/syncfile.$(ARCH).o \ +$(SRC)/share_image_mags.$(ARCH).o \ +$(SRC)/share_mean_mags.$(ARCH).o \ +$(SRC)/MeanMagIO.$(ARCH).o \ +$(SRC)/indexCatalog.$(ARCH).o \ +$(SRC)/client_logger.$(ARCH).o \ +$(SRC)/ZeroPointModes.$(ARCH).o \ +$(SRC)/setExclusions.$(ARCH).o \ +$(SRC)/setMrelFinal.$(ARCH).o \ +$(SRC)/setMrelCatalog.$(ARCH).o \ +$(SRC)/BoundaryTreeOps.$(ARCH).o \ +$(SRC)/write_coords.$(ARCH).o + +$(RELPHOT): $(INC)/relphot.h +$(BIN)/relphot.$(ARCH): $(RELPHOT) + +RELPHOT_CLIENT = \ +$(SRC)/StatDataSetOps.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/GridOps.$(ARCH).o \ +$(SRC)/ImageOps.$(ARCH).o \ +$(SRC)/ImageSubset.$(ARCH).o \ +$(SRC)/BrightCatalog.$(ARCH).o \ +$(SRC)/MosaicOps.$(ARCH).o \ +$(SRC)/TGroupOps.$(ARCH).o \ +$(SRC)/ReadTGroupFile.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/StarOps.$(ARCH).o \ +$(SRC)/WarpImageMaps.$(ARCH).o \ +$(SRC)/StackImageMaps.$(ARCH).o \ +$(SRC)/RepairWarpMeasures.$(ARCH).o \ +$(SRC)/myIndex.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o \ +$(SRC)/bcatalog.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/help.$(ARCH).o \ +$(SRC)/extra.$(ARCH).o \ +$(SRC)/synthetic_mags.$(ARCH).o \ +$(SRC)/plotstuff.$(ARCH).o \ +$(SRC)/fit1d_irls.$(ARCH).o \ +$(SRC)/liststats.$(ARCH).o \ +$(SRC)/initialize.$(ARCH).o \ +$(SRC)/syncfile.$(ARCH).o \ +$(SRC)/select_images.$(ARCH).o \ +$(SRC)/load_images.$(ARCH).o \ +$(SRC)/save_images.$(ARCH).o \ +$(SRC)/MagResidSave.$(ARCH).o \ +$(SRC)/load_catalogs.$(ARCH).o \ +$(SRC)/reload_catalogs.$(ARCH).o \ +$(SRC)/relphot_objects.$(ARCH).o \ +$(SRC)/relphot_client.$(ARCH).o \ +$(SRC)/relphot_synthphot.$(ARCH).o \ +$(SRC)/relphot_synthphot_catalog.$(ARCH).o \ +$(SRC)/synthetic_zpts.$(ARCH).o \ +$(SRC)/client_logger.$(ARCH).o \ +$(SRC)/ZeroPointModes.$(ARCH).o \ +$(SRC)/setExclusions.$(ARCH).o \ +$(SRC)/setMrelFinal.$(ARCH).o \ +$(SRC)/setMrelCatalog.$(ARCH).o \ +$(SRC)/BoundaryTreeOps.$(ARCH).o \ +$(SRC)/write_coords.$(ARCH).o + +$(RELPHOT_CLIENT): $(INC)/relphot.h +$(BIN)/relphot_client.$(ARCH): $(RELPHOT_CLIENT) + +TEST_LISTSTATS = \ +$(SRC)/StatDataSetOps.$(ARCH).o \ +$(SRC)/liststats.$(ARCH).o \ +$(SRC)/test_liststats.$(ARCH).o + +$(TEST_LISTSTATS): $(INC)/relphot.h +$(BIN)/test_liststats.$(ARCH): $(TEST_LISTSTATS) + +TEST_FITIRLS = \ +$(SRC)/fit1d_irls.$(ARCH).o \ +$(SRC)/extra.$(ARCH).o \ +$(SRC)/test_fitirls.$(ARCH).o + +$(TEST_FITIRLS): $(INC)/relphot.h +$(BIN)/test_fitirls.$(ARCH): $(TEST_FITIRLS) + Index: /branches/eam_branches/relphot.20210521/doc/ChangeLog.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/ChangeLog.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/ChangeLog.txt (revision 41602) @@ -0,0 +1,22 @@ + +- relphot-1-4: + * converted to gfits APIs (forces libfits 1.6) + * converted to new DVO APIs (forces libdvo 1.3) + * allow seletion by RA/DEC limits + * removed old, unused code + * added time range to time selections + +- relphot-1-3: + * dropped IMAGE_CATALOG from config + * convert to use libkapa + * dropped _PS from average.R,D,M,dM + * convert to dvo_image_lock,unlock + +- relphot-1-2: released 2005.10.20 + * substantial changes to sync with addstar v1.3 + (added mode/format support, new dvo load functions, etc). + * various cleanups (functions moved to libohana) + * changes to use new libohana (v1.5) / libfits (v1.4) + +- relphot-1-1 + * minor cleanup, loneos.h -> dvo.h Index: /branches/eam_branches/relphot.20210521/doc/allsky.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/allsky.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/allsky.txt (revision 41602) @@ -0,0 +1,82 @@ + +2011.05.10 + + Looking into them memory usage of relphot: + AverageTiny = 32 bytes (including 8 byte padding) + MeasureTiny = 56 bytes (including 8 byte padding) + Image Indexes: + bin : Nmeasure * off_t + clist : Nmeasure * off_t + mlist : Nmeasure * off_t + Mosaic Indexes: + bin : Nmeasure * off_t + clist : Nmeasure * off_t + mlist : Nmeasure * off_t + Total: + 32 bytes * Naverage (values) + 56 bytes * Nmeasure (values) + 64 bytes * Nmeasure (indexes) + + In my test run, I have 3.3M average + 37.6M measure: + 2.1GB values + 2.2GB indexes + + This matches the memory usage graph. But, after the indexes have + been loaded, the memory footprint grows (slowly) by another 1GB as + the processing runs along. + + +using 23153493 of 640663697 stars (192585953 of 289672671 measurements) +using 3306896 of 48399614 stars ( 37582679 of 193863612 measurements) + + + +2011.04.12 + + I have modified the code to operate on multiple filter at a time, + and to have a limited density, with the goal of running the analysis + on a huge area. However, we are somewhat limited by the fact that the + 'measure' and 'average' structures are huge and have far more than + is needed for just the relphot (or even relastro) analysis. + + I am exploring the idea of using a stripped-down structure in the + relphot analysis -- it would be assigned in bcatalog. + + Here are the elements we actually need in relphot proper: + +average: + FIELD R, RA, double, RA, decimal degrees + FIELD D, DEC, double, DEC, decimal degrees + FIELD Nmeasure, NMEASURE, unsigned short, number of psf measurements + FIELD measureOffset, OFF_MEASURE, int, offset to first psf measurement + FIELD flags, FLAGS, uint32_t, average object flags (star; ghost; etc) + = 2*8 + 2 + 4 + 4 = 26 bytes + +measure: + FIELD dR, D_RA, float, RA offset, arcsec + FIELD dD, D_DEC, float, DEC offset, arcsec + FIELD M, MAG, float, catalog mag, mag + FIELD Mcal, M_CAL, float, image cal mag, mag + + FIELD dM, MAG_ERR, float, mag error, mag + FIELD airmass, AIRMASS, float, (airmass - 1), airmass + FIELD Xccd, X_CCD, float, X coord on chip, pixels + FIELD Yccd, Y_CCD, float, Y coord on chip, pixels + + FIELD t, TIME, int, time in seconds (UNIX) + FIELD averef, AVE_REF, unsigned int, reference to average entry + FIELD imageID, IMAGE_ID, unsigned int, reference to DVO image ID + FIELD dbFlags, DB_FLAGS, unsigned int, flags supplied by analysis in database + FIELD photcode, PHOTCODE, unsigned short, photcode + = 8*4 + 4*4 + 2 = 50 bytes + + (note that psfQual and photFlags are only used in bcatalog to select measurements) + +eg, 1 year 3 pi has 1 average and 20 measures per object (and 8 secfilt entries) + +* current: 112 + 8*24 + 20*148 = 3264 bytes / star +* propose: 26 + 8*24 + 20*50 = 1218 bytes / star + +for the full 3pi area, in 30GB, we could use 820 stars per square +degree (5740 per exposure), as opposed to the current limit of 300 +(2100 per exposure). Index: /branches/eam_branches/relphot.20210521/doc/config.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/config.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/config.txt (revision 41602) @@ -0,0 +1,28 @@ + +for choosing the bright (tied-down) stars, these are used: + +MAG_LIM &MAG_LIM -- faintest stars used +SIGMA_LIM &SIGMA_LIM -- lowest S/N used (sigma = 1/(S/N)) +STAR_SCATTER &STAR_SCATTER -- max scatter for a star to be include in the fits +IMAGE_SCATTER &IMAGE_SCATTER -- max scatter for an image to be kept in the fit +IMAGE_OFFSET &IMAGE_OFFSET -- max zero point offset of an image to be kept in the fit +STAR_CHISQ &STAR_CHISQ -- max chisq for a star to be include in the fits +STAR_TOOFEW &STAR_TOOFEW -- star must have more than this good measurements to be used +IMAGE_TOOFEW &IMAGE_TOOFEW -- image must have more than this good measurements to be used +IMAGE_GOOD_FRACTION &IMAGE_GOOD_FRACTION -- image must have more than this (good measurements/all meas) to be used + + +GSCFILE GSCFILE -- source for database sky table if missing +CATDIR CATDIR -- catdir +CATMODE CATMODE -- output format for DVO tables (should not be used by relphot?) +CATFORMAT CATFORMAT -- output format for DVO tables (should not be used by relphot?) +PHOTCODE_FILE MasterPhotcodeFile -- source for photcode table if missing +SKY_DEPTH SKY_DEPTH)) { SKY_DEPTH = 2; } -- output sky partition depth (default is 2, should not be used by relphot) +ZERO_PT &ZERO_POINT -- reference internal zero point, should not be needed (defined by DB) + +CAMERA_CONFIG CameraConfig -- source of camera description, used for generating correction grid +MOSAICNAME MOSAICNAME -- name to define relationship between chip and camera +GRID_TOOFEW &GRID_TOOFEW -- correction grid cell must have more than this stars (or is nan) +RELPHOT_GRID_X &RELPHOT_GRID_X -- number of grid cells per chip (x) +RELPHOT_GRID_Y &RELPHOT_GRID_Y -- number of grid cells per chip (y) +RELPHOT_GRID_BINNING &RELPHOT_GRID_BINNING -- ??? Index: /branches/eam_branches/relphot.20210521/doc/flatcorr.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/flatcorr.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/flatcorr.txt (revision 41602) @@ -0,0 +1,90 @@ + +2012.01.18 + +I have updated libdvo to define a flatcorr table, and have the APIs to +read and write the table. I have also updated setphot to load the +ubercal measurements and to define the flatcorr table and links (from +image -> flatcorr). Now I need to understand where to include this in +relphot. + +One option would be to include it as part of the function 'getMcal'. +This function goes from (cat,meas) -> (image) to find the image zero +point (Mcal). It could also check for a photom_map_id value for that +image, and include that in the calculation (if it exists). + + +Another option would be to add a new function (getMmap or something) +and to include this essentially everywhere getMcal is called. + +2012.01.13 + +Thinking it over, I've decided that the easiest way to move forward on +the implementation is to define a second grid-like correction to +represent the flat-field, and defer unifying the two (conceptually +similar) things to the future. + +So, we start with infrastructure: + +* APIs to read and write the flat-field corrections + + * how to represent the flat-field correction? + * each pixel as a row in a table? + * easy to do the file I/O (one table for all corrections) + * need to unwrap the values into correction[photcode][x][y] + * + * each photcode as a FITS image extention? + * + * each camera as a full image (like Grid)? + + +2012.01.12 + +The Ubercal analysis gives me zero points along with flat-field +corrections defined for certain time ranges as a function of +fractional cell location. I need to include equivalent information +into the DVO tables, both to apply the ubercal values and to do +equivalent fits. This needs to be done in both relphot and setphot. +(Other programs refer to the value of Mcal for each detection; only +relphot and setphot modify or apply the image Mcal to the detection +value). + +In setphot, the needed modification is in update_catalog_setphot, +where the value of Mcal is set from the image. This function needs to +also look up the corresponding flat-correction and include it in that +calculation. It is also necessary to load the externally-defined +description of the flat-correction and to determine which images go +which which corrections. + +In relphot, we would like to be able to fit for the value of interest, +as well as to apply the values loaded from the database. + +In relphot: + +The survey-wide flat-field correction information is similar to the +Grid that is currently used to calculate the flat-field correction +terms, though there are some subtleties. + +Currently, the grid analysis is a bit complicated by the way we relate +the chip coordinates to the camera coordinates. The main problem is +that we do not do a good job of tracking which chip a detection comes +from. + +GridOps loads a configuration table which describes the layout of the +camera. This means the relative orientations of each of the chips. +Each chip is define by its ccdnum value and by its ccdname. + +The code assigns the Grid bin by using the camera layout information +to relate the chip X,Y position to the camera X,Y coordinate. The +grid cells are then defined as a function of camera coordinate. + +This is not a terrible solution, but it conflates two things: the chip +and position in the chip with the layout of the focal plane. To solve +or apply a flat-field term, we don't actually need to know anything +about the location of the chip in the camera. + +In other words, Grid[Xbin][Ybin] is currently a single array assuming +a monotonic relationship between Xbin and Xchip, etc. We could have +Grid[chip][Xbin][Ybin], where Xbin and Ybin come directl from +Xchip,Ychip and have a different function which converts the resulting +image to a camera-coordinate system image. + Index: /branches/eam_branches/relphot.20210521/doc/mosaic.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/mosaic.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/mosaic.txt (revision 41602) @@ -0,0 +1,54 @@ + + +2014.07.09 + +* trying to make sense of mosaic.Mcal vs image.Mcal in the context of UBERCAL: + + * relphot_images: + * load_images (convert raw FITS table to Image structure, select subset matching selection) + * initMosaics (associate mosaics to images (gpc1 photcodes only)) + * initMosaicGrid (define spatial range of mosaics [only used by grid analysis]; set Mcal to and image.Mcal to 0.0) + --- calculate image or mosaic Mcal values + * setMcalFinal (set image.Mcal = mosaicMcal) + + * relphot_parallel_regions: + * assign_images + * makeMosaics (equivalent to initMosaics, but works on full image table, not subset) + * setMosaicCenters (set Mcal to and image.Mcal to 0.0) <--- this is wrong! + (sends images ONLY, not mosaics, to remote machines) + (slurps back new image values, applies to db) + + * relphot_parallel_images: + * makeMosaics (associate mosaics to images, gpc1 photcodes only) + * initMosaicGrid (needed in this one) +--- + +For 'mosaic' zero points, I define the mosaics by grouping the images +by observation time. I do this by generating a list of the image +start times. I sort this list and then find the unique entries. Each +time defines a new mosaic entry. I then assign images to the mosaics +by matching their start times. + +The above currently happens on the image subset. I need to identify +the images which are NOT in the subset but for which a mosaic has been +constructed -- these are partially accepted images. I then need to be +able to do something with those images and mosaics: either reject all +partial mosaics or accept all images from partial mosaics. The +advantage of the latter is that the choice of accepted images depends +on the catalog / regions loaded -- I need to accept images only if all +overlapping region files are loaded, right? + +So, a few things are needed: + +1) identify the images which are NOT selected by have matching start +times. +2) exclude them from the subset somehow. + +Some questions: + + * the image subset is currently a *copy* of the Image array, not a + view. does it help to change this? (beyond the memory savings?) + + * I'm still a bit unclear on what 'load_images' vs 'reload_images' + does. + Index: /branches/eam_branches/relphot.20210521/doc/notes.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/notes.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/notes.txt (revision 41602) @@ -0,0 +1,160 @@ + +2008.09.20 + + I am working on the flat-field correction analysis. This is + currently triggered by the -grid flag. This analysis needs to + require a single camera. + + * need to define the relationship between image and ccdnum + (currently it uses as regex, effectively, to match the ccdnum to + the number portion of the photcode). make an explicit entry in + the dvo.config table. + + * clean up the output files (make the output files have names + supplied by the user, etc) + + * I use the string MOSAICNAME to determine the relationship between + chips. this allows me to do the relphot grid process on mosaic + images for which I don't have bilevel astrometry, but where we do, + we could link the mosaic together via the mosaic reference image. + Maybe this is a future upgrade. + + * I think mosaic name can be replaced with camera name: we need to + supply CAMERA in any case, and the photcodes have names of the + form CAMERA.XXX. This is all very ad hoc, but can be effective + for the near term. + +2006.10.17 + + I am working on a partial upgrade of relphot to handle more general + problems than I have treated in the past. In particular, I would + like to be able to have it assign the average magnitudes, regardless + of whether any or all images and/or objects are well-treated for + relative photometry. In fact, I would like it to be able to + determine average magnitudes even for sources which have only + externally supplied data, and thus have no matching images. I would + eventually like to have more flexibility about the filtering which + is performed on the measurements to determine the average + magnitudes. Some of this work will require a bit of a more careful + treatment of the process. + + At the moment, we can consider the relphot process to consist of the + following steps: + + 1) load the complete catalog data + 2) select a subset of the sources: + - appropriate photcode (equiv == selected photcode) + - in time range, if specified + - appropriate dophot type, if specified + - bright enough: + - mag < MAG_LIM + - dMag > SIGMA_LIM + - ImagMin < iMag < ImagMax + - other restrictions + - within valid image region for chip + 3) iterate to a solution for Mcal (image offsets) + - mark specific images as bad + - mark specific sources as bad + - mark specific measurements as bad + - include desired restrictions on Mcal (eg, all chips of mosaic + matched) + - only keep the image offsets + + 4) if the results are to be kept, recalculate the average magnitudes + using the image exclusions and Mcal values determined above + +2006.05.03 + + I am upgrading relphot to accept as input an arbitrary region (using + the standard ohana SkyRegion concept of a RA & DEC bounded patch on + the sky). This modification has a few implications. First, the + analysis is in fact performed on the populated SkyRegions which + overlap the requested area on the sky. + +- select different calibration options +- plot various terms + +- rationalize Mmos & Mcal (sum(Mcal) -> Mmos) + +- options: + + -update: update phot database + -mosaic: apply per-mosaic corrections + -grid: apply X,Y-based offset + -chips: apply per-chip corrections + -trange: select images in time range + + +find_images: selects images in region, filtered by photcode, time range + +bcatalog: selects stars & measurments, filtered by mag limit, photcode, time range + +image: a single ccd +mosaic: a collection of associate ccds (mosaiclink[image]) +grid: x,y magnitude correction grid across mosaic +fit: 2-d fit of mag offsets across field + + + +m = Mrel + Mcal + Mmos + Mxy + Mgrid + +Mrel (average) +Mcal (image) +Mmos (mosaic) +Mxy (Xi, Yi) (Xi,Yi per image ??) +Mgrid (Xm, Ym) + + + measure -> average catalog[i].measure[j].averef + average -> measure catalog[i].average[j].offset, catalog[i].average[j].Nm + + measure -> image image: image[cat, meas] + image -> measure image: catlist[image], measlist[image] + + measure -> mosaic mosaic: mosaic[cat, meas] + mosaic -> measure mosaic: catlist[mosaic], measlist[mosaic] + + measure -> gridbin grid: bin[cat, meas] + gridbin -> measure grid: catlist[bin], measlist[bin] + + measure -> Xi, Yi -> Mxy + + +getMmos: + + for (i = 0; i < Nmosaic; i++) { + + for (j = 0; j < Nmeasure[i]; j++) { + cat = catlist[i][j] + meas = measlist[i][j] + + M = catalog[cat].measure[meas].M + Mcal = applyMcal (cat, meas) + Mrel = applyMrel (cat, meas) + Mxy = applyMxy (cat, meas) + Mgrid = applyMgrid (cat, meas) + + list[n] = M - Mcal - Mrel - Mxy - Mgrid + } + } + + + +excluding data points: + +star: N < 2 +image: + + +-- + +MAG_LIM : keep stars brighter than MAG_LIM (Mcat) +SIGMA_LIM : keep stars with error < SIGMA_LIM +STAR_SCATTER : exclude stars with dM > STAR_SCATTER +IMAGE_SCATTER, NIMAGE_SCATTER : define image scatter limits + + + +load_images - exclude images + + Index: /branches/eam_branches/relphot.20210521/doc/parallel.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/parallel.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/parallel.txt (revision 41602) @@ -0,0 +1,104 @@ + +2014.02.15 + +Nearly done with the relphot mods. some outstanding questions; + + * image vs mosaic? + * how do decide which images need to be shared? + +2014.02.14 + + more relphot notes: + + * I load the catalogs, with a + +2014.02.12 + +I am making progess on the relphot -parallel-regions implementation. some things I need to deal with: + + * assign_images: select mosaics for all images saved for a given host (need to supply them as well) + ** I need to decide on the Image / Mosaic split. If I am calibrating by image, then the image center defines ownership + ** if I am calibrating by exposure, then the mosaic center must define ownership + o determine SkyList covering the images for region host + o tag detections and objects which I own or do not own + o ImageOps.c / matchImage matches detections to images and needs to handle mine / not mine cases + o ditto for matchMosaics + o load_catalogs : I need to make my requested catalogs unique (just add uniquer to load_catalogs.c:130 + o figure out what fields I'm saving in meanmags and how to construct it + -- mag, objID, catID, photcode, (dmag or other stats?) + * from which hosts do I slurp mean mags? + o function to merge a new meanmag array into the existing one + o how to go from (objID,catID) to a given catalog[i].average[j] + +2014.02.06 + +** a single region-level host owns images for which the center lands + in its region. + +** it owns detections which come from images which it owns + +** it owns objects which land in its region + +Extending parallel relphot processing to split the sky (and images) +into regions, each of which runs in parallel at the same time: + +Top Level (relphot -parallel-images): + * define regions of the sky -> hosts + * load images, assign to hosts + * launch region-level jobs on remote hosts + + Region Level (relphot -parallel-images-region) + * load my image subset table + * request objects and detections for my skyregion + * match images & objects, etc + + * update my image parameters + * update my detections + * write out detections + * load detections from my border hosts + * match to my objects + * update my objects + * write out objects + * load objects from my border hosts + * update images + (iterate N times) + * write out image parameters + +Top Level + * read image parameters, update + * update objects + +2012.02.13 + +Image fields needed by relphot_client (saved by ImageSubsetSave): + +imageID +photom_map_id +flags +Mcal +dMcal +tzero +trate + +(if we have USE_GRID == T): +photcode + + +2012.02.10 + +I have put together the start of parallel dvo examples. I have +'dvodist' to distribute the data (and pull it back), and parallel +modifications to 'setphot'. Here is what I need to do to relphot to +make it work in parallel: + +* load_catalogs.c : a parallel version of this would spawn the remote + jobs in -load mode. these should read from their owned tables, + collect the subset of detections (subset naturally generated by + bcatalog), and write the results to local temp fits files. the main + program would wait until the sub programs have been finished and + then load in all of the subset tables. + +* reload_catalogs : a parallel version needs to write the subset image + table and then run the remote jobs. these would read the image + data, and load and update their local database tables. This is + nearly identical to the code in setphot / setphot_client. Index: /branches/eam_branches/relphot.20210521/doc/pv2.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/pv2.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/pv2.txt (revision 41602) @@ -0,0 +1,53 @@ + +2014.07.03 + +NOTE: 2 questions on these measurements: chisq and ubercal dist? + +I think I have worked out all of the issues in the mag / flux +calibrations for stacks and forced warps. It took me a few passes to +get the right combinations of the 3.56 or the 8.9 offset to the mags. +But now, the stack psf mags, warp psf mags, stack flux, warp flux, and +mean psf mags all agree at the 0.1 mmag level. + +I still have some concerns regarding the flags and astrometry. I am +not set on my laptop to test the primary cell stuff (well, I could +probably reproduce it, but easier to wait until I'm home). I need to +review the 'waterfall' and check that the choices make sense (and +write it down for review). + +2014.07.02 + +I am trying to run a (rather artificial) test of relphot +specifically, I have taken a single smf and an associated warp cmf and +modified the headers so I can load them in DVO and run relphot. Some +issues I've hit: + +* needed to specify PHOTCODE for the stack and fforce fake cmfs +* needed to modify IMAGEID and MJD-OBS so the chips associated + correctly with different gpc1 images +* modified relphot/MosaicOps.c to explicitly skip stack and warp by + photcode when matching chips into mosaics (kind of a hack as it + won't work with other cameras) + +2014.07.01 + +thinking about the issues with relphot for the new PV2 / PV3 schema. +I now have 3 separate sets of mean photometry for each of the stages +cam/staticsky/warp. I need to make sure I am handling everthing +needed. + +Calibration sequence: + +* relphot_images / relphot_parallel_images : calculate the zero points + +Loop around with the standard sequence, but only fitting for the mean +camera mags and camera zero points. At the end of the iteration, fit +the stacks and warps to the new mean object mags. + +Some things I've started to worry about: + +1) QSOs & other variables. Am I detecting and avoiding these well +enough? + +2) what is the appropriate ubercal weight (10, 100, 1000?) + Index: /branches/eam_branches/relphot.20210521/doc/timeline.txt =================================================================== --- /branches/eam_branches/relphot.20210521/doc/timeline.txt (revision 41602) +++ /branches/eam_branches/relphot.20210521/doc/timeline.txt (revision 41602) @@ -0,0 +1,36 @@ + +timeline for processing on PV2 (31 remote hosts): + +load images, ready to launch hosts : 160s +remote hosts begin: + * set up mosaic indexes: 388s - 1330s (from client launch time) + * loop 0 done: 1086s - 15870s (~ synchronous, thus clients took ~15ksec to load) + * loop 7 done: 4443s - 19251s (~ 3400sec total after load is done) + * finalized Mcal : 4444s - 19315s + * remote hosts done (5.4hr) + +remote client update Mcal average times: + * load catalog : 5963s (longest one is 11569s) + * make tiny : 113s + * find images: 576s + * set Mrel : 4793s + * save catalog: 2531s + +elapsed time for update Mcal: 51318s (14.3h) + +--> consistent with 3x longest single client times + +end of relphot: 70633s (19.6h) + + +relastro timeline: +on remote hosts: + * load catalog data: 247s - 4444s (loading more detections than relphot), + * wall time of the load is NOT sum(load time), because clients are launched before slurp is started + * update chips: 23981s for last one + ** end of load + analysis ~ 7hr + * analysis takes ~1-2hr + +update time ~ 9hr (compare with ~3-4h for relphot) + +*** the I/O is NOT dominant. Index: /branches/eam_branches/relphot.20210521/include/relphot.h =================================================================== --- /branches/eam_branches/relphot.20210521/include/relphot.h (revision 41602) +++ /branches/eam_branches/relphot.20210521/include/relphot.h (revision 41602) @@ -0,0 +1,858 @@ +# include +# include +# include +# include +# include + +/* # define GRID_V1 */ +# define GRID_V2 +# define NO_IMAGE -100 + +# define ID_SECF_STACK_PRIMARY 0x00004000 + +# define NBOOTSTRAP 100 + +# ifndef MAX_INT +# define MAX_INT 2147483647 +# endif + +// choose off_t or int depending on full-scale relphot analysis resources +// # define IDX_T off_t +# define IDX_T int + +typedef struct { + int minID; + int maxID; + int *index; + int Nindex; + int NINDEX; +} myIndexType; + +typedef enum { + ZPT_STARS, + ZPT_TGROUP, + ZPT_MOSAIC, + ZPT_IMAGES, +} ZptFitModeType; + +typedef enum { + STAGE_CHIP = 0x01, + STAGE_WARP = 0x02, + STAGE_STACK = 0x04, +} RelphotStages; + +typedef enum { + MODE_ERROR = 0, + UPDATE_IMAGES, + UPDATE_AVERAGES, + PARALLEL_REGIONS, + PARALLEL_IMAGES, + APPLY_OFFSETS, + SYNTH_PHOT, +} RelphotMode; + +typedef enum { + MODE_NONE = 0, + MODE_LOAD = 1, + MODE_UPDATE = 2, + MODE_UPDATE_OBJECTS = 3, + MODE_SYNTH_PHOT = 4, +} ModeType; + +// NOTE: this is only used in special cases where we limit photcode ranges +typedef enum { + PS1_none, + PS1_g = 1, + PS1_r = 2, + PS1_i = 3, + PS1_z = 4, + PS1_y = 5, + PS1_w = 6, +} PS1filters; + +typedef struct { + Header PHU; + Header header[5]; // grizy (matches Nsec in photcodes) + Matrix matrix[5]; + Coords coords; + int Nx; + int Ny; +} SynthZeroPoints; + +typedef struct { + unsigned int start; + unsigned int stop; + short photcode; + float McalPSF; + float McalAPER; + float dMcal; + float dMsys; + float stdev; + float dMmin; + float dMmax; + float McalChiSq; + float secz; + float ubercalDist; + unsigned int nFitPhotom; + unsigned int nValPhotom; + unsigned int flags; + char skipCal; // if TRUE, this mosaic is incomplete and should not be calibrated + char inTGroup; + Coords coords; +} Mosaic; + +typedef struct { + short photcode; + float McalPSF; + float McalAPER; + float dKlam; + float dMcal; + float dMsys; + float stdev; + float dMmin; + float dMmax; + float McalChiSq; + unsigned int nFitPhotom; + unsigned int nValPhotom; + unsigned int flags; + + off_t Nimage; + off_t NIMAGE; + off_t *image; + + off_t Nmeasure; + off_t NMEASURE; + off_t *measure; + off_t *catalog; + + off_t Nmosaic; + off_t NMOSAIC; + Mosaic **mosaic; // pointer to the mosaic structures + + void *parent; +} TGroup; + +// we have an array of TGroup times, each pointing to N sets of data values +typedef struct { + unsigned int start; + unsigned int stop; + TGroup *byCode; // each of these contains the collection of images for the time and photcode + int nCode; +} TGTimes; + +typedef enum { + STATS_NONE, + STATS_MEAN, + STATS_MEDIAN, + STATS_WT_MEAN, + STATS_INNER_MEAN, + STATS_INNER_WTMEAN, + STATS_CHI_INNER_MEAN, + STATS_CHI_INNER_WTMEAN +} ListStatsMode; + +typedef struct { + double median; + double mean; + double sigma; + double error; + double chisq; + double min; + double max; + double Upper90; + double Upper80; + double Lower20; + double Lower10; + double total; + int Nmeas; + ListStatsMode statmode; +} StatType; + +typedef struct { + double *flxlist; // list of measure.mag values for a given star + double *errlist; // mag errors for a star + double *wgtlist; // weights to use for mean mags + int *ranking; // weights to use for mean mags + int *measSeq; // weights to use for mean mags + int *msklist; // mask modifications + int Nlist; + + double *values; + double *wtvals; + double *wtlist; + double *ykeep; + double *dykeep; + double *wtkeep; + double *ysample; + double *dysample; + double *wtsample; + double *bvalue; + +} StatDataSet; + +typedef struct { + double *xVector; // complete list of values in independent variable (optional) + double *yVector; // complete list of values available + double *dyVector; // complete list of errors available +} FitDataType; + +// this structure carries the data and pre-allocated arrays for +// 1D fitting with arbitrary order for data with external weights, +// optional priors, irls iterations, and bootstrap resampling +typedef struct { + int order; // order of fit (e.g., y = C0 + C1*x has order = 1) + int nterm; // number of fitted parameters (order + 1) + int mterm; // number of summations needed for OLS (2*order + 1) + + int Nlist; // total number of measurements in list (may be more than the number to use) + int Nbootstrap; // number of bootstrap iterations + int MaxIterations; // number of IRLS iterations + + // input data & parameters: + FitDataType *alldata; // full data set + + double *bPriorValue; // prior value + double *bPriorSigma; // prior sigma + + // internal / temporary arrays: + FitDataType *keepdata; // unclipped data for bootstrap analysis + FitDataType *sample; // sample for a bootstrap iteration + + double *yOffVector; // difference between yFit and yObs + + double *tmpVector; // internal copy of values to use in, e.g., calculation of median + double *wtIRLS; // IRLS weights (distance from model value weighted by standard error) + + double **cArray; + double *sumVector; // pre-allocated to mterm (2*order + 1) + + double **bArray; + double **bSaveArray; + double **bBootArray; // values generated by the bootstrap resampling + + // output results: + double *bSigma; // 1 sigma range of parameters + + double min; // min of bSigma[0] values + double max; // max of bSigma[0] values + double sigma; // (sample) standard deviation of bSigma[0] values + double chisq; // chisq of fit (unmasked values only) + int Nmeas; // number of unmasked values used in fit +} FitDataSet; + +typedef struct { + int Nfew; + int Ncode; + int Nsys; + int Nbad; + int Ncal; + int Nmos; + int Ngrp; + int Ngrid; + + // NOTE: the following arrays are (possibly) pre-allocated and carried down to each + // thread. The psfData are used in all relphot analyses; the others are only used on + // the final output steps. + + int Nsecfilt; + + StatDataSet *psfData; // one is allocated for each primary (average) photcode + StatDataSet *aperData; + StatDataSet *kronData; + + double *psfqf_list; // psfqf for all filters + double *psfqfperf_list; // psfqfperf for all filters + double *stargal_list; // stargal for all filters + uint32_t *photflag_list; // photflags for all filters + + int Nstargal; + int Nphotflags; + + int *havePS1; // this secfilt has synthetic mags + int *haveSYN; // this secfilt has synthetic mags + int *measSYN; // this measurement is the synthetic mag for this secfilt + int *needSYN; // this secfilt mag should use synthetic mags + float *minSYN; // minimum synthetic mag below which synthetic should be forced + + float *psfQfMax; // max psfQf value for this secfilt + float *psfQfPerfMax; // max psfQfperf value for this secfilt + + int *Nmeas; // count of PS1 exposure (chip) measurements for this secfilt + int *NmeasGood; // count of PS1 exposure (chip) measurements for this secfilt + int *Next; // count of PS1 exposure (chip) measurements for this secfilt + int *NexpPS1; // count of PS1 exposure (chip) measurements for this secfilt + int *haveUbercal; // does this secfilt have any ubercal data? + + int *tessID; // tess,proj,skycell to use for warp and diff analysis + int *projID; + int *skycellID; + + float *minUbercalDist; + + StatType psfstats; + StatType apstats; + StatType kronstats; +} SetMrelInfo; + +typedef struct { + float M; + float dM; + float Mchisq; + int Nsec; + unsigned int objID; + unsigned int catID; + int photcode; +} MeanMag; + +typedef struct { + float McalPSF; + float McalAPER; + float dMcal; + float dMagSys; + float McalChiSq; + int nFitPhotom; + int flags; + unsigned int imageID; + short ubercalDist; +} ImageMag; + +typedef struct { + AverageTiny *average; // array of (minimal) average data + MeasureTiny *measure; // array of (minimal) measure data + SecFilt *secfilt; // array of secfilt data (matched to average by Nsecfilt) + off_t Naverage; + off_t Nmeasure; +} BrightCatalog; + +typedef struct { + float McalPSF; + float McalAPER; + float dMcal; + unsigned int imageID; + unsigned int photom_map_id; + unsigned int flags; + int tessID; + int projID; + int skycellID; + unsigned int tzero; + unsigned char trate; + short ubercalDist; +} ImageSubset; + +typedef struct { + Catalog *catalog; // array of catalogs generated + int NCATALOG; // number of catalogs allocated + int Ncatalog; // number of catalogs generated + int Nsecfilt; // number of catalogs generated + off_t *NAVERAGE; // allocated Averages per catalog + off_t *NMEASURE; // allocated Measures per catalog + int *index; // lookup table catID -> catalog[i] + int *catIDs; // lookup table catID <- catalog[i] + int maxID; // max catID value to date +} CatalogSplitter; + +/* global variables set in parameter file */ +# define DVO_MAX_PATH 1024 +char ImageCat[DVO_MAX_PATH]; +char ImageTemplate[DVO_MAX_PATH]; +char CatTemplate[DVO_MAX_PATH]; +char GSCFILE[DVO_MAX_PATH]; +char *CATDIR; +char CATMODE[16]; /* raw, mef, split, mysql */ +char CATFORMAT[16]; /* internal, elixir, loneos, panstarrs */ +char CameraConfig[DVO_MAX_PATH]; +char CAMERA[64]; /* eg, gpc1 */ +char SKY_TABLE[DVO_MAX_PATH]; +int SKY_DEPTH; /** XXX EAM : depth of catalog tables, fix usage */ +char *SYNTH_ZERO_POINTS; + +// globals for parallel region operations +char *REGION_FILE; +char *IMAGE_TABLE; +int REGION_HOST_ID; + +int HOST_ID; +char *HOSTDIR; +char *IMAGES; +char *BCATALOG; +char *BOUNDARY_TREE; +ModeType MODE; + +char *BOUNDARY_TREE; + +// XXX deprecate int SET_MREL_VERSION; +int IS_DIFF_DB; + +double MAG_LIM; +double SIGMA_LIM; +double IMAGE_SCATTER; +double IMAGE_OFFSET; +double NIGHT_SCATTER; +double NIGHT_OFFSET; +double STAR_SCATTER; +double STAR_CHISQ; +double MIN_ERROR; +double IMFIT_SYS_SIGMA_LIM; +double CLOUD_TOLERANCE; + +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; +char *MANUAL_UNIQUER; + +int SKIP_PARALLEL_GROUPS; + +int PARALLEL_REGIONS_MANUAL; + +int NTHREADS; + +int VERBOSE; +int VERBOSE2; +int VERBOSE_IMAGE; + +int MOSAIC_ZEROPT; +int TGROUP_ZEROPT; + +int FREEZE_IMAGES; +int FREEZE_MOSAICS; + +int TGROUP_FIT_AIRMASS; + +int TEST_IMAGE1; +int TEST_IMAGE2; + +int MANUAL_ITERATION; +int NLOOP; +int NGRID; +int RESET; +int RESET_ZEROPTS; +int REPAIR_WARPS; +int PRESERVE_PS1; +int UPDATE; +int SAVE_IMAGE_UPDATES; +int PLOTSTUFF; +int SAVEPLOT; +int SHOW_PARAMS; +char MOSAICNAME[256]; +char STATMODE[32]; +int STAR_BAD; +int MEAS_BAD; +int STAR_TOOFEW; +int GRID_TOOFEW; +int IMAGE_TOOFEW; +double IMAGE_GOOD_FRACTION; +int IMAGE_BAD; +int CALIBRATE_STACKS_AND_WARPS; +int USE_GRID; +int KEEP_UBERCAL; +char *OUTROOT; +char *UPDATE_CATFORMAT; +int PLOTDELAY; +int UpdateAverages; +int ApplyOffsets; +int SyntheticPhotometry; + +int USE_MCAL_PSF_FOR_STACK_APER; + +char *PhotcodeList; + +int RELPHOT_GRID_X; +int RELPHOT_GRID_Y; +int RELPHOT_GRID_BINNING; + +int *photseclist; +int Nphotcodes; +PhotCode **photcodes; +// int PhotSec; +// int PhotNsec; + +PhotCode *refPhotcode; +int USE_REFERENCE_WEIGHT; + +int MaxDensityUse; +double MaxDensityValue; + +int AreaSelect; +double AreaXmin, AreaXmax, AreaYmin, AreaYmax; + +int ImagSelect; +double ImagMin, ImagMax; + +int DophotSelect, DophotValue; + +double PlotMmin, PlotMmax, PlotdMmin, PlotdMmax; +enum {black, white, red, orange, yellow, green, blue, indigo, violet}; + +int TimeSelect; +time_t TSTART, TSTOP; + +SkyRegion UserPatch; +char *UserCatalog; + +enum {TGROUP_ZPT_MODE_NONE, TGROUP_ZPT_MODE_GOOD_NIGHT, TGROUP_ZPT_MODE_ALL, }; +int TGROUP_ZPT_MODE; + +enum {MOSAIC_ZPT_MODE_NONE, MOSAIC_ZPT_MODE_BAD_NIGHT, MOSAIC_ZPT_MODE_GOOD_MOSAIC, MOSAIC_ZPT_MODE_BAD_NIGHT_GOOD_MOSAIC, MOSAIC_ZPT_MODE_ALL, }; +int MOSAIC_ZPT_MODE; + +enum {IMAGE_ZPT_MODE_NONE, IMAGE_ZPT_MODE_BAD_NIGHT, IMAGE_ZPT_MODE_BAD_MOSAIC, IMAGE_ZPT_MODE_BAD_NIGHT_BAD_MOSAIC, IMAGE_ZPT_MODE_ALL, }; +int IMAGE_ZPT_MODE; + +int USE_ALL_IMAGES; +int USE_BASIC_CHECK; +int USE_FULL_OVERLAP; + +RelphotStages STAGES; + +# ifdef GRID_V1 +int setGridMeasure (off_t meas, int cat, double X, double Y); +# endif + +# ifdef GRID_V2 +int setGridMeasure (off_t meas, int cat, double X, double Y, int ccdnum); +# endif + +/*** relphot prototypes ***/ +void ConfigInit PROTO((int *argc, char **argv)); +void GetConfig PROTO((char *config, char *field, char *format, int N, void *ptr)); +char *GetPhotnamebyCode PROTO((PhotCodeData *photcodes, int code)); +void InterpolateGrid PROTO((float *buffer, int Nx, int Ny, Coords *ccd, Coords *gcoords)); +off_t *SelectRefMosaic PROTO((Mosaic **refmosaic, off_t *Nimage)); +RelphotMode args PROTO((int argc, char **argv)); +int args_client PROTO((int argc, char **argv)); +int bcatalog PROTO((Catalog *subcatalog, Catalog *catalog, int Ncat)); +void clean_images PROTO((void)); +void clean_measures PROTO((Catalog *catalog, int Ncatalog, int final, FlatCorrectionTable *flatcorr)); +void clean_mosaics PROTO((void)); +void clean_tgroups PROTO((void)); +void clean_stars PROTO((Catalog *catalog, int Ncatalog)); +int corner_check PROTO((double *x1, double *y1, double *x2, double *y2)); +void dumpGrid PROTO((void)); +void dump_grid PROTO((void)); +int edge_check PROTO((double *x1, double *y1, double *x2, double *y2)); +void findImages PROTO((Catalog *catalog, int Ncatalog, int doImageList)); +int findMosaics PROTO((Catalog *catalog, int Ncatalog, int doMosaicList)); +int findTGroups PROTO((Catalog *catalog, int Ncatalog)); + +void clearImages (void); +void checkImages (char *name); +int dumpMags (FILE *fout, Catalog *catalog, int Ncatalog); + +void makeMosaics (Image *image, off_t Nimage, int mergeMcal); +Mosaic *getMosaicForImage (off_t im); +void setMosaicCenters (Image *image, off_t Nimage); + +void set_db (FITS_DB *in); +int Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2) ; +void TrapSignal (int sig); +void SetProtect (int mode); +int SetSignals (void); + +void freeGridBins PROTO((int Ncatalog)); +void freeImageBins PROTO((int Ncatalog, int doImageList)); +void freeMosaicBins PROTO((int Ncatalog, int doMosaicList)); +void freeTGroupBins PROTO((int Ncatalog)); + +void free_catalogs PROTO((Catalog *catalog, int Ncatalog)); +int gcatalog PROTO((Catalog *catalog, int FINAL)); +Coords *getCoords PROTO((off_t meas, int cat)); +off_t getImageEntry PROTO((off_t meas, int cat)); + +float getMcal PROTO((off_t meas, int cat, dvoMagClassType class)); +float getMflat PROTO((off_t meas, int cat, FlatCorrectionTable *flatcorr, Catalog *catalog)); +float getMgrid PROTO((off_t meas, int cat)); +float getMmos PROTO((off_t meas, int cat)); +float getMgrp PROTO((off_t meas, int cat, float airmass, float *dZpt)); +float getMrel PROTO((Catalog *catalog, off_t meas, int cat, dvoMagClassType class, dvoMagSourceType source)); +short getUbercalDist PROTO((off_t meas, int cat)); +float getCenterOffset PROTO((off_t meas, int cat, Measure *measure, unsigned int *myID)); +Image *getimage PROTO((off_t N)); +Image *getimages PROTO((off_t *N, off_t **LineNumber)); +ImageSubset *getimages_subset PROTO((off_t *N)); +void global_stats PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr, int nloop)); +void initGrid PROTO((int dX, int dY)); +void initGridBins PROTO((Catalog *catalog, int Ncatalog)); +void initImageBins PROTO((Catalog *catalog, int Ncatalog, int doImageList)); +void initImagesSubset PROTO((ImageSubset *input, off_t *line_number, off_t N)); +void initImages PROTO((Image *input, off_t *LineNumber, off_t N)); +void initMosaicBins PROTO((Catalog *catalog, int Ncatalog, int doMosaicList)); +void initMosaicGrid PROTO((Image *image, off_t Nimage)); +void initMosaics PROTO((Image *subset, off_t Nsubset, Image *image, char *inSubset, off_t Nimage)); +void initTGroups PROTO((Image *subset, off_t Nsubset)); +void freeTGroups PROTO((void)); +void initTGroupBins PROTO((Catalog *catalog, int Ncatalog)); +void initMrel PROTO((Catalog *catalog, int Ncatalog)); +RelphotMode initialize PROTO((int argc, char **argv)); +void initialize_client PROTO((int argc, char **argv)); +void liststats_setmode PROTO((StatType *stats, char *strmode)); +int liststats PROTO((double *value, double *dvalue, double *wvalue, int N, StatType *stats)); +int liststats_init PROTO((StatType *stats)); +int liststats_irls PROTO((StatDataSet *dataset, int Npoints, StatType *stats)); +int liststats_fit1d PROTO((double *value, double *err, double *x, int Npts, StatType *stats, double *dk)); +double weight_cauchy PROTO((double x)); +double VectorFractionInterpolate PROTO((double *values, float fraction, int Npoints)); + +// fit1d_irls: +int fit1d_irls PROTO((FitDataSet *dataset, int Npoints)); +void FitDataSetFree PROTO((FitDataSet *dataset)); +void FitDataSetAlloc PROTO((FitDataSet *dataset, int Nmax, int order, int Nbootstrap)); +void FitDataSetAddPriors PROTO((FitDataSet *dataset)); +StatType FitDataSetSoften PROTO((FitDataSet *dataset, int Nvalues)); + +unsigned int *ReadTGroupFile PROTO((FILE *f, int *nelem)); +void loadTGroups PROTO((char *filename)); +void initTGroupsMcal PROTO((void)); +TGroup *getTGroupForImage PROTO((off_t im)); +TGroup *findTGroup PROTO((unsigned int start, int photcode)); + +Catalog *load_catalogs PROTO((SkyList *skylist, int *Ncatalog, int hostID, char *hostpath, char *syncfile)); +Catalog *load_catalogs_parallel PROTO((SkyList *sky, int *Ncatalog, char *syncfile)); + +int load_images PROTO((FITS_DB *db, SkyList *skylist, SkyRegion *region, int unlockImages, int UseAllImages)); +Image *select_images PROTO((SkyList *skylist, Image *timage, off_t Ntimage, char *inSubset, off_t **LineNumber, off_t *Nimage, SkyRegion *region)); + +int main PROTO((int argc, char **argv)); +void mark_images PROTO((Image *image, off_t Nimage, Image *timage, off_t Ntimage)); +void matchImage PROTO((Catalog *catalog, off_t meas, int cat, int doImageList)); +void matchMosaics PROTO((Catalog *catalog, off_t meas, int cat, int doMosaicList)); +void matchTGroups PROTO((Catalog *catalog, off_t meas, int cat)); + +double opening_angle PROTO((double x1, double y1, double x2, double y2, double x3, double y3)); +void plot_chisq PROTO((Catalog *catalog, int Ncatalog)); +void plot_defaults PROTO((Graphdata *graphdata)); +void plot_grid PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +void plot_images PROTO((void)); +void plot_list PROTO((Graphdata *graphdata, double *xlist, double *ylist, int N, char *label, char *format, ...) OHANA_FORMAT(printf, 6, 7) ); +void plot_mosaic_fields PROTO((Catalog *catalog)); +void plot_mosaics PROTO((void)); +void plot_scatter PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr)); +void plot_star_coords PROTO((Catalog *catalog, int Ncatalog)); +void plot_stars PROTO((Catalog *catalog, int Ncatalog)); +void plot_setMcal PROTO((double *list, int Npts)); + +void plot_list_add PROTO((Graphdata *graphdata, double *xlist, double *ylist, int N)); +int get_graph PROTO((int N)); +int open_graph PROTO((int N)); + +void reload_catalogs PROTO((SkyList *skylist, FlatCorrectionTable *flatcorr, int hostID, char *hostpath)); +int reload_catalogs_parallel PROTO((SkyList *sky)); +int reload_images PROTO((FITS_DB *db)); +int setExclusions PROTO((Catalog *catalog, int Ncatalog, int verbose)); +void setMgrid PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +void setMcalFromMosaics PROTO((void)); +void setMcalFromTGroups PROTO((void)); +int setMcalOutput PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr)); + +void setMcal PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +int setMmos PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +int setMgrp PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); + +int setMrel PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr)); +void setMrelFinal PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr, int simpleAverage)); +int setMrelOutput PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr)); + +void setMcalTest PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); + +int setMave PROTO((Catalog *catalog, int Ncatalog)); +void set_ZP PROTO((double ZERO)); +int setrefcode PROTO((Image *image, off_t Nimage)); +void skip_measurements PROTO((Catalog *catalog, int pass, FlatCorrectionTable *flatcorr)); +void sortA PROTO((double *X, int N)); +void sortB PROTO((double *X, double *Y, int N)); +void sortC PROTO((double *X, double *Y, double *F1, double *F2, int N)); +void sortD PROTO((double *X, double *Y, double *Z, int N)); +StatType statsImageM PROTO((Catalog *catalog)); +StatType statsImageN PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +StatType statsImageX PROTO((Catalog *catalog)); +StatType statsImagedM PROTO((Catalog *catalog)); +StatType statsMosaicM PROTO((Catalog *catalog)); +StatType statsMosaicN PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr)); +StatType statsMosaicX PROTO((Catalog *catalog)); +StatType statsMosaicdM PROTO((Catalog *catalog)); +StatType statsTGroupM PROTO((Catalog *catalog)); +StatType statsTGroupX PROTO((Catalog *catalog)); +StatType statsTGroupdM PROTO((Catalog *catalog)); + +StatType statsStarN PROTO((Catalog *catalog, int Ncatalog, int Nsec, int seccode, FlatCorrectionTable *flatcorr)); +StatType statsStarS PROTO((Catalog *catalog, int Ncatalog, int Nsec)); +StatType statsStarX PROTO((Catalog *catalog, int Ncatalog, int Nsec)); +void wcatalog PROTO((Catalog *catalog)); +void wimages PROTO((void)); +void write_coords PROTO((Header *header, Coords *coords)); +int relphot_objects (SkyList *skylist, int hostID, char *hostpath); +int relphot_images (SkyList *skylist); + +void relphot_usage (void); +void relphot_help (int argc, char **argv); + +void relphot_client_usage (void); +void relphot_client_help (int argc, char **argv); + +off_t getImageByID (off_t ID); + +int rationalize_mosaics PROTO((Catalog *catalog, int Ncatalog)); +int LimitDensityCatalog (Catalog *subcatalog, Catalog *catalog); + +BrightCatalog *BrightCatalogLoad(char *filename); +int BrightCatalogSave(char *filename, BrightCatalog *catalog); +BrightCatalog *BrightCatalogMerge (Catalog *catalog, int Ncatalog); +CatalogSplitter *BrightCatalogSplitInit (int Nsecfilt); +int BrightCatalogSplitFree (CatalogSplitter *catalogs); +int BrightCatalogSplit (CatalogSplitter *catalogs, BrightCatalog *bcatalog); + +int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage); +ImageSubset *ImageSubsetLoad(char *filename, off_t *nimage); + +int client_logger_init (char *dirname); +int client_logger_message (char *format,...); + +int MatchImageName (off_t meas, int cat, char *name); +int MatchImageSkycellID (off_t meas, int cat, int myTessID, int myProjectionID, int mySkycellID); +int FindImageSkycellID (off_t meas, int cat, int *myTessID, int *myProjectionID, int *mySkycellID); + +int load_tess (char *treefile); +void free_tess (void); +int get_tess_ids (int *tessID, int *projID, int *skycellID, double ra, double dec); +int TessellationIDsByImageName (int *tessID, int *projID, int *skycellID, char *name); + +// int BoundaryTreePrimaryCell (char *primaryCellName, double ra, double dec); +// int BoundaryTreePrimaryCellIDs (int *projID, int *skycellID, double ra, double dec); + +int print_measure_set_alt (Average *average, SecFilt *secfilt, Measure *measure); +int setMrel_catalog_alt (Catalog *catalog, int Nc, int isSetMrelFinal, FlatCorrectionTable *flatcorr, SetMrelInfo *results, int Nsecfilt); +int setMrelAverageExposure (Catalog *catalog, int cat, off_t ave, int Nsecfilt, int isSetMrelFinal, FlatCorrectionTable *flatcorr, SetMrelInfo *results); +int setMrelAverageStack (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr); +int setMrelAverageForcedWarp (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr, SetMrelInfo *results); + +int setGlobalObjStats (Average *average, Measure *measure); + +void SetMrelInfoInit (SetMrelInfo *results, int allocLists); +void SetMrelInfoFree (SetMrelInfo *results); +void SetMrelInfoReset (SetMrelInfo *results); +void SetMrelInfoResetObject (SetMrelInfo *results); + +void StatDataSetFree (StatDataSet *dataset, int Nsecfilt); +StatDataSet *StatDataSetAlloc (int Nsecfilt, int Nmax); + +int init_synthetic_mags (); +int add_synthetic_mags (AverageTiny *average, SecFilt *secfilt, MeasureTiny *measure, off_t *Nmeasure, off_t *Nm); + +int relphot_parallel_regions (SkyTable *sky); +int relphot_parallel_images (SkyTable *sky); + +int assign_images (FITS_DB *db, RegionHostTable *regionHosts); +int select_images_hostregion (RegionHostTable *hosts, Image *image, off_t Nimage); +int find_host_for_coords (RegionHostTable *regionHosts, double R, double d); +int calculate_image_bounds (Image *image, double *rmin, double *rmax, double *dmin, double *dmax, double Rmid); + +int launch_region_hosts (RegionHostTable *regionHosts); + +Image *ImageTableLoad(char *filename, off_t *nimage); +int ImageTableSave (char *filename, Image *images, off_t Nimages); + +int indexCatalogs (Catalog *catalog, int Ncatalog); +int catID_and_objID_to_seq (int catID, int objID, int *catSeq, off_t *objSeq); +void freeCatalogIndexes (int Ncatalog); + +int check_sync_file (char *filename, int nloop); +int clear_sync_file (char *filename); +int update_sync_file (char *filename, int nloop); +char *make_filename (char *dirname, char *hostname, int hostID, char *tailname); + +int share_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop); +int slurp_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop); +int set_mean_mags (MeanMag *meanmags, AverageTiny *average, SecFilt *secfilt, int Nsec); +MeanMag *merge_mean_mags (MeanMag *target, int *ntarget, MeanMag *source, int Nsource); + +MeanMag *MeanMagLoad(char *filename, off_t *nmeanmags); +int MeanMagSave(char *filename, MeanMag *meanmags, off_t Nmeanmags); + +int share_image_mags (RegionHostTable *regionHosts, int nloop); +int slurp_image_mags (RegionHostTable *regionHosts, int nloop); +int set_image_mags (ImageMag *image_mags, Image *image); +ImageMag *merge_image_mags (ImageMag *target, int *ntarget, ImageMag *source, int Nsource); + +ImageMag *ImageMagLoad(char *filename, off_t *nimage_mags); +int ImageMagSave(char *filename, ImageMag *image_mags, off_t Nimage_mags); + +int markObjects (Catalog *catalog, int Ncatalog); + +// in extra.c +int isMosaicChip (int photcode); + +int isGPC1chip (int photcode); +int isGPC2chip (int photcode); +int isGPC1stack (int photcode); +int isGPC1warp (int photcode); +int isGPC1synth (int photcode); +int whichGPC1filter (int photcode); +int is2MASS (int photcode); +int isTYCHO (int photcode); + +int isHSCchip (int photcode); +int isCFHchip (int photcode); + +int magStatsByRanking (StatDataSet *dataset, StatType *stats); + +int SynthZeroPointsLoad (char *filename); +SynthZeroPoints *SynthZeroPointsGet (); + +int relphot_synthphot (SkyList *sky, int hostID, char *hostpath); +int relphot_synthphot_parallel (SkyList *sky); + +int relphot_synthphot_catalog (Catalog *catalog, SynthZeroPoints *zpts); +int relphot_synthphot_average (Average *average, SecFilt *secfilt, Measure *measure, SynthZeroPoints *zpts); + +void setMeasureRank (Catalog *catalog); +int getImageFlags (off_t meas, int cat); +int getMosaicFlags (off_t meas, int cat); + +void relphot_free (SkyTable *sky, SkyList *skylist); +void freeImages (char *dbImagePtr); +void freeMosaics (void); +void relphot_client_free (SkyTable *sky, SkyList *skylist); +void BrightCatalogFree (BrightCatalog *bcatalog); + +void put_astrom_table (AstromOffsetTable *myTable); +void free_astrom_table (); + +int RepairWarpMeasures (Catalog *catalog); + +int FindWarpGroups (void); +int GetWarpSeq (Image *image, int obstime, unsigned short photcode, double Rave, double Dave, float X, float Y); +void FreeWarpGroups (void); + +myIndexType *myIndexInit (); +int myIndexFree (myIndexType *myIndex); +int myIndexUpdateLimits (myIndexType *myIndex, int value); +int myIndexSetRange (myIndexType *myIndex); +int myIndexSetEntry (myIndexType *myIndex, int value, int entry); +int myIndexGetEntry (myIndexType *myIndex, int value); + +uint64_t CreatePSPSObjectID(double ra, double dec); +uint64_t CreatePSPSStackDetectionID(int sourceID, int imageID, int detID); +uint64_t CreatePSPSDetectionID(double tobs, int ccdid, int detID); + +int MakeStackIndex (void); +void FreeStackGroups (void); +int GetStackSeq (Image *image, double Rstk, double Dstk, unsigned short photcode, float X, float Y); +void sort_by_ra (double *R, double *D, int *I, int *S, int N); + +void RepairWarpMeasuresOpenLogfile (); +void RepairWarpMeasuresCloseLogfile (); + +void dump_tgroups (Catalog *catalog, int Npass); +void dump_catalog (Catalog *catalog, off_t c, int Npass); +void dump_tgroup_imstats (int Npass); + +void SetZptIteration (int current); +int GetZptIteration (void); + +void SetZeroPointModes (void); +int UseStandardOLS (ZptFitModeType mode); +int GetActivePhotcodeIndex (int photcode); + +int save_images_backup (FITS_DB *db); +int save_images_updates (FITS_DB *db); + +int MagResidSave(char *filename, Catalog *catalog); Index: /branches/eam_branches/relphot.20210521/src/BoundaryTreeOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/BoundaryTreeOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/BoundaryTreeOps.c (revision 41602) @@ -0,0 +1,63 @@ +# include "relphot.h" + +static int Ntess = 0; +static TessellationTable *tess = NULL; + +int TessellationIDsByImageName (int *tessID, int *projID, int *skycellID, char *name) { + + int i; + + *tessID = -1; + *projID = -1; + *skycellID = -1; + + if (!tess) return FALSE; + + for (i = 0; i < Ntess; i++) { + // do this with a strhash of some kind? + if (!strncmp (name, tess[i].basename, tess[i].Nbasename)) { + *tessID = i; + if (tess[i].projectIDoff >= 0) { + *projID = atoi(&name[tess[i].projectIDoff]); + } else { + *projID = 0; + } + if (tess[i].skycellIDoff >= 0) { + *skycellID = atoi(&name[tess[i].skycellIDoff]); + } else { + *skycellID = 0; + } + return TRUE; + } + } + return FALSE; +} + +int load_tess (char *tessfile) { + + tess = TessellationTableLoad (tessfile, &Ntess); + if (!tess) { + fprintf (stderr, "failed to load tessellation boundary file %s\n", tessfile); + exit (2); + } + + return TRUE; +} + +int get_tess_ids (int *tessID, int *projID, int *skycellID, double ra, double dec) { + + int status; + + if (!tess) return FALSE; + + double R = ohana_normalize_angle(ra); + status = TessellationPrimaryCellIDs(tess, Ntess, tessID, projID, skycellID, R, dec); + return status; +} + +void free_tess () { + + if (!tess) return; + TessellationTableFree (tess, Ntess); + free (tess); +} Index: /branches/eam_branches/relphot.20210521/src/BrightCatalog.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/BrightCatalog.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/BrightCatalog.c (revision 41602) @@ -0,0 +1,764 @@ +# include "relphot.h" + +// relphot_client reads from the local catalogs and generates a set of bright, subset catalogs +// we are going to save these as a single big FITS file, with only 3 extensions: + +// measure, average, secfilt. these only need to contain the fields relevant to the +// MeasureTiny, AverageTiny, and Secfilt tables, but with cat_id appended to each row so we can +// reassign correctly on read + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// XXX double check the header extname values for each table +// XXX make sure we free things as we can +// XXX make sure we close files as we can +// XXX handle free and close on error return as well +BrightCatalog *BrightCatalogLoad(char *filename) { + + int i, Ncol; + off_t Nrow; + char type[16]; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + header.buffer = NULL; + matrix.buffer = NULL; + ftable.buffer = NULL; + theader.buffer = NULL; + BrightCatalog *catalog = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + goto escape; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + goto escape; + } + + ALLOCATE (catalog, BrightCatalog, 1); + catalog->Naverage = 0; + catalog->Nmeasure = 0; + + ftable.header = &theader; + + // *** Measure Data *** + { + // load data for this header + if (!gfits_load_header (f, &theader)) goto escape; + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) goto escape; + + // need to create and assign to flat-field correction + GET_COLUMN(R, "RA", double); + GET_COLUMN(D, "DEC", double); + GET_COLUMN(M, "MAG_SYS", float); + GET_COLUMN(Mkron, "MAG_KRON", float); + GET_COLUMN(McalPSF, "MCAL_PSF", float); + GET_COLUMN(McalAPER, "MCAL_APER", float); + GET_COLUMN(Mflat, "MAG_FLAT", float); + GET_COLUMN(dM, "MAG_ERR", float); + GET_COLUMN(airmass, "AIRMASS", float); + GET_COLUMN(Xccd, "X_CCD", float); + GET_COLUMN(Yccd, "Y_CCD", float); + GET_COLUMN(dt, "EXPTIME", float); + GET_COLUMN(t, "TIME", int); + GET_COLUMN(averef, "AVE_REF", int); // XXX signed vs unsigned? + GET_COLUMN(imageID, "IMAGE_ID", int); + GET_COLUMN(dbFlags, "DB_FLAGS", int); + GET_COLUMN(photFlags, "PHOT_FLAGS", int); + GET_COLUMN(catID, "CAT_ID", int); + GET_COLUMN(photcode, "PHOTCODE", short); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + MeasureTiny *measure = NULL; + ALLOCATE (measure, MeasureTiny, Nrow); + for (i = 0; i < Nrow; i++) { + measure[i].R = R[i]; + measure[i].D = D[i]; + measure[i].M = M[i]; + measure[i].Mkron = Mkron[i]; + measure[i].McalPSF = McalPSF[i]; + measure[i].McalAPER = McalAPER[i]; + measure[i].Mflat = Mflat[i]; + measure[i].dM = dM[i]; + measure[i].airmass = airmass[i]; + measure[i].Xccd = Xccd[i]; + measure[i].Yccd = Yccd[i]; + measure[i].dt = dt[i]; + measure[i].t = t[i]; + measure[i].averef = averef[i]; + measure[i].imageID = imageID[i]; + measure[i].dbFlags = dbFlags[i]; + measure[i].photFlags = photFlags[i]; + measure[i].catID = catID[i]; + measure[i].photcode = photcode[i]; + measure[i].myDet = FALSE; + } + fprintf (stderr, "loaded data for %lld measure\n", (long long) Nrow); + + free (R ); + free (D ); + free (M ); + free (Mkron ); + free (McalPSF ); + free (McalAPER); + free (Mflat ); + free (dM ); + free (airmass ); + free (Xccd ); + free (Yccd ); + free (dt ); + free (t ); + free (averef ); + free (imageID ); + free (dbFlags ); + free (photFlags ); + free (catID ); + free (photcode); + + catalog->measure = measure; + catalog->Nmeasure = Nrow; + } + + /*** load Average values ***/ + { + + // load data for this header + if (!gfits_load_header (f, &theader)) goto escape; + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) goto escape; + + // need to create and assign to flat-field correction + GET_COLUMN(R, "RA", double); + GET_COLUMN(D, "DEC", double); + GET_COLUMN(Nmeasure, "NMEAS", int); + GET_COLUMN(measureOffset, "MEASURE_OFF", int); + GET_COLUMN(flags, "FLAGS", int); + GET_COLUMN(catID, "CAT_ID", int); + GET_COLUMN(objID, "OBJ_ID", int); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + AverageTiny *average = NULL; + ALLOCATE (average, AverageTiny, Nrow); + for (i = 0; i < Nrow; i++) { + average[i].R = R[i]; + average[i].D = D[i]; + average[i].Nmeasure = Nmeasure[i]; + average[i].measureOffset = measureOffset[i]; + average[i].flags = flags[i]; + average[i].catID = catID[i]; + average[i].objID = objID[i]; + average[i].nOwn = 0; + } + fprintf (stderr, "loaded data for %lld average\n", (long long) Nrow); + + free (R ); + free (D ); + free (Nmeasure ); + free (measureOffset ); + free (flags ); + free (catID ); + free (objID ); + + catalog->average = average; + catalog->Naverage = Nrow; + } + + /*** load Secfilt values ***/ + { + + // load data for this header + if (!gfits_load_header (f, &theader)) goto escape; + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) goto escape; + + // need to create and assign to flat-field correction + GET_COLUMN(M, "MAG", float); + GET_COLUMN(Mkron, "MAG_KRON", float); + GET_COLUMN(dM, "MAG_ERR", float); + GET_COLUMN(Mchisq, "MAG_CHI", float); + GET_COLUMN(flags, "FLAGS", int); + GET_COLUMN(Ncode, "NCODE", short); + GET_COLUMN(Nused, "NUSED", short); + GET_COLUMN(Mmin, "MAG_MIN", float); + GET_COLUMN(Mmax, "MAG_MAX", float); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + SecFilt *secfilt = NULL; + ALLOCATE (secfilt, SecFilt, Nrow); + for (i = 0; i < Nrow; i++) { + secfilt[i].MpsfChp = M[i]; + secfilt[i].MkronChp = Mkron[i]; + secfilt[i].dMpsfChp = dM[i]; + secfilt[i].Mchisq = Mchisq[i]; + secfilt[i].flags = flags[i]; + secfilt[i].Ncode = Ncode[i]; + secfilt[i].Nused = Nused[i]; + secfilt[i].Mmin = Mmin[i]; + secfilt[i].Mmax = Mmax[i]; + } + fprintf (stderr, "loaded data for %lld secfilt\n", (long long) Nrow); + + free (M ); + free (Mkron ); + free (dM ); + free (Mchisq); + free (flags ); + free (Ncode ); + free (Nused ); + free (Mmin ); + free (Mmax ); + catalog->secfilt = secfilt; + // assert Nsecfilt * Naverage = Nrow? + } + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + fclose (f); + + return catalog; + +escape: + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + if (catalog) free (catalog); + + fclose (f); + return NULL; +} + +// we are passed a BrightCatalog structure, write it to a FITS table (3 ext) +int BrightCatalogSave(char *filename, BrightCatalog *catalog) { + + int i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + + /*** MeasureTiny ***/ + { + gfits_create_table_header (&theader, "BINTABLE", "MEASURE_TINY"); + + gfits_define_bintable_column (&theader, "D", "RA", "ra", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "D", "DEC", "dec", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_SYS", "magnitude (sys)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_KRON", "magnitude (sys,kron)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER","magnitude (cal)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_FLAT", "magnitude (flat)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ERR", "magnitude (err)", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "AIRMASS", "airmass", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "X_CCD", "ccd x coord", "pix", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "Y_CCD", "ccd y coord", "pix", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "EXPTIME", "-2.5 * log (exposure time)", "sec", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "TIME", "time of exp", "sec", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "AVE_REF", "pointer to average table", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "DB_FLAGS", "flags", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "PHOT_FLAGS", "photflags", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "CAT_ID", "catalog", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "I", "PHOTCODE", "photcode", NULL, 1.0, FT_BZERO_INT16); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // create intermediate storage arrays + double *R ; ALLOCATE (R , double, catalog->Nmeasure); + double *D ; ALLOCATE (D , double, catalog->Nmeasure); + float *M ; ALLOCATE (M , float, catalog->Nmeasure); + float *Mkron ; ALLOCATE (Mkron , float, catalog->Nmeasure); + float *McalPSF ; ALLOCATE (McalPSF , float, catalog->Nmeasure); + float *McalAPER ; ALLOCATE (McalAPER , float, catalog->Nmeasure); + float *Mflat ; ALLOCATE (Mflat , float, catalog->Nmeasure); + float *dM ; ALLOCATE (dM , float, catalog->Nmeasure); + float *airmass ; ALLOCATE (airmass , float, catalog->Nmeasure); + float *Xccd ; ALLOCATE (Xccd , float, catalog->Nmeasure); + float *Yccd ; ALLOCATE (Yccd , float, catalog->Nmeasure); + float *dt ; ALLOCATE (dt , float, catalog->Nmeasure); + int *t ; ALLOCATE (t , int , catalog->Nmeasure); + int *averef ; ALLOCATE (averef , int , catalog->Nmeasure); + int *imageID ; ALLOCATE (imageID , int , catalog->Nmeasure); + int *dbFlags ; ALLOCATE (dbFlags , int , catalog->Nmeasure); + int *photFlags ; ALLOCATE (photFlags, int , catalog->Nmeasure); + int *catID ; ALLOCATE (catID , int , catalog->Nmeasure); + short *photcode ; ALLOCATE (photcode , short, catalog->Nmeasure); + + // assign the storage arrays + MeasureTiny *measure = catalog->measure; + for (i = 0; i < catalog->Nmeasure; i++) { + R[i] = measure[i].R ; + D[i] = measure[i].D ; + M[i] = measure[i].M ; + Mkron[i] = measure[i].Mkron ; + McalPSF[i] = measure[i].McalPSF ; + McalAPER[i] = measure[i].McalAPER ; + Mflat[i] = measure[i].Mflat ; + dM[i] = measure[i].dM ; + airmass[i] = measure[i].airmass ; + Xccd[i] = measure[i].Xccd ; + Yccd[i] = measure[i].Yccd ; + dt[i] = measure[i].dt ; + t[i] = measure[i].t ; + averef[i] = measure[i].averef ; + catID[i] = measure[i].catID ; + imageID[i] = measure[i].imageID ; + dbFlags[i] = measure[i].dbFlags ; + photFlags[i]= measure[i].photFlags; + photcode[i] = measure[i].photcode ; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "RA", R, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "DEC", D, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MAG_SYS", M, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MAG_KRON", Mkron, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MAG_FLAT", Mflat, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "AIRMASS", airmass, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "X_CCD", Xccd, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "Y_CCD", Yccd, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "EXPTIME", dt, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "TIME", t, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "AVE_REF", averef, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "DB_FLAGS", dbFlags, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "PHOT_FLAGS", photFlags, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "CAT_ID", catID, catalog->Nmeasure); + gfits_set_bintable_column (&theader, &ftable, "PHOTCODE", photcode, catalog->Nmeasure); + + free (R ); + free (D ); + free (M ); + free (Mkron ); + free (McalPSF ); + free (McalAPER); + free (Mflat ); + free (dM ); + free (airmass ); + free (Xccd ); + free (Yccd ); + free (dt ); + free (t ); + free (averef ); + free (imageID ); + free (dbFlags ); + free (photFlags ); + free (catID ); + free (photcode); + + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + gfits_free_header (&theader); + gfits_free_table (&ftable); + } + + /*** AverageTiny ***/ + { + gfits_create_table_header (&theader, "BINTABLE", "AVERAGE_TINY"); + + gfits_define_bintable_column (&theader, "D", "RA", "ra (J2000)", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "D", "DEC", "dec (J2000)", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "NMEAS", "number of measures", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "MEASURE_OFF", "index to measurements", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "FLAGS", "flags", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "CAT_ID", "catalog ref", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "OBJ_ID", "object ref", NULL, 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // create intermediate storage arrays + double *R ; ALLOCATE (R, double, catalog->Naverage); + double *D ; ALLOCATE (D, double, catalog->Naverage); + int *Nmeasure ; ALLOCATE (Nmeasure, int, catalog->Naverage); + int *measureOffset ; ALLOCATE (measureOffset, int, catalog->Naverage); + int *flags ; ALLOCATE (flags, int, catalog->Naverage); + int *catID ; ALLOCATE (catID, int, catalog->Naverage); + int *objID ; ALLOCATE (objID, int, catalog->Naverage); + + // assign the storage arrays + AverageTiny *average = catalog->average; + for (i = 0; i < catalog->Naverage; i++) { + R[i] = average[i].R ; + D[i] = average[i].D ; + Nmeasure[i] = average[i].Nmeasure; + measureOffset[i] = average[i].measureOffset; + flags[i] = average[i].flags; + catID[i] = average[i].catID; + objID[i] = average[i].objID; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "RA", R, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "DEC", D, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "NMEAS", Nmeasure, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "MEASURE_OFF", measureOffset, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "CAT_ID", catID, catalog->Naverage); + gfits_set_bintable_column (&theader, &ftable, "OBJ_ID", objID, catalog->Naverage); + + free (R ); + free (D ); + free (Nmeasure ); + free (measureOffset ); + free (flags ); + free (catID ); + free (objID ); + + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + gfits_free_header (&theader); + gfits_free_table (&ftable); + } + + /*** SecFilt ***/ + { + gfits_create_table_header (&theader, "BINTABLE", "SECFILT"); + + gfits_define_bintable_column (&theader, "E", "MAG", "", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_KRON", "", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ERR", "", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_CHI", "", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "FLAGS", "", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "NCODE", "", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "NUSED", "", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_MIN", "min valid mag", "mag", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_MAX", "max valid mag", "mag", 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // Nsecfilt is number of average filters, Nsec is total number of values + int Nsecfilt = GetPhotcodeNsecfilt(); + int Nsec = Nsecfilt * catalog->Naverage; + + // create intermediate storage arrays + float *M ; ALLOCATE (M , float, Nsec); + float *Mkron ; ALLOCATE (Mkron , float, Nsec); + float *dM ; ALLOCATE (dM , float, Nsec); + float *Mchisq ; ALLOCATE (Mchisq , float, Nsec); + int *flags ; ALLOCATE (flags , int, Nsec); + short *Ncode ; ALLOCATE (Ncode , short, Nsec); + short *Nused ; ALLOCATE (Nused , short, Nsec); + float *Mmin ; ALLOCATE (Mmin , float, Nsec); + float *Mmax ; ALLOCATE (Mmax , float, Nsec); + + // assign the storage arrays + SecFilt *secfilt = catalog->secfilt; + for (i = 0; i < Nsec; i++) { + M [i] = secfilt[i].MpsfChp ; + Mkron [i] = secfilt[i].MkronChp; + dM [i] = secfilt[i].dMpsfChp; + Mchisq[i] = secfilt[i].Mchisq ; + flags [i] = secfilt[i].flags ; + Ncode [i] = secfilt[i].Ncode ; + Nused [i] = secfilt[i].Nused ; + Mmin [i] = secfilt[i].Mmin ; + Mmax [i] = secfilt[i].Mmax ; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "MAG", M , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_KRON", Mkron , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_CHI", Mchisq, Nsec); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags , Nsec); + gfits_set_bintable_column (&theader, &ftable, "NCODE", Ncode , Nsec); + gfits_set_bintable_column (&theader, &ftable, "NUSED", Nused , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_MIN", Mmin , Nsec); + gfits_set_bintable_column (&theader, &ftable, "MAG_MAX", Mmax , Nsec); + + free (M ); + free (Mkron ); + free (dM ); + free (Mchisq ); + free (flags ); + free (Ncode ); + free (Nused ); + free (Mmin ); + free (Mmax ); + + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + gfits_free_header (&theader); + gfits_free_table (&ftable); + } + return TRUE; +} + +// merge a list of catalogs into a single BrightCatalog array set +BrightCatalog *BrightCatalogMerge (Catalog *catalog, int Ncatalog) { + + off_t i, j, k; + + BrightCatalog *bcatalog = NULL; + ALLOCATE (bcatalog, BrightCatalog, 1); + + int Nmeas = 0; + int Naves = 0; + for (i = 0; i < Ncatalog; i++) { + Nmeas += catalog[i].Nmeasure; + Naves += catalog[i].Naverage; + } + + // XXX this prevents different catalogs from having different Nsecfilt values + int Nsecfilt = GetPhotcodeNsecfilt(); + + ALLOCATE (bcatalog[0].measure, MeasureTiny, Nmeas); + ALLOCATE (bcatalog[0].average, AverageTiny, Naves); + ALLOCATE (bcatalog[0].secfilt, SecFilt, Naves*Nsecfilt); + + int Nm = 0; + int Na = 0; + for (i = 0; i < Ncatalog; i++) { + if (!catalog[i].Naverage) continue; + for (j = 0; j < catalog[i].Naverage; j++) { + bcatalog[0].average[Na] = catalog[i].averageT[j]; + for (k = 0; k < Nsecfilt; k++) { + bcatalog[0].secfilt[Nsecfilt*Na + k] = catalog[i].secfilt[Nsecfilt*j + k]; + } + Na++; + assert (Na <= Naves); + } + for (j = 0; j < catalog[i].Nmeasure; j++) { + bcatalog[0].measure[Nm] = catalog[i].measureT[j]; + Nm++; + assert (Nm <= Nmeas); + } + } + bcatalog->Naverage = Na; + bcatalog->Nmeasure = Nm; + return bcatalog; +} + +void BrightCatalogFree (BrightCatalog *bcatalog) { + if (!bcatalog) return; + FREE (bcatalog[0].average); + FREE (bcatalog[0].measure); + FREE (bcatalog[0].secfilt); + FREE (bcatalog); + return; +} + +// distribute a bright catalog across separate catalogs +CatalogSplitter *BrightCatalogSplitInit (int Nsecfilt) { + + int i; + + CatalogSplitter *catalogs = NULL; + + ALLOCATE (catalogs, CatalogSplitter, 1); + + // as we see new BrightCatalogs, we will update maxID and extend this array + catalogs->maxID = 0; + ALLOCATE (catalogs->index, int, catalogs->maxID + 1); + for (i = 0; i <= catalogs->maxID; i++) catalogs->index[i] = -1; + + catalogs->Nsecfilt = Nsecfilt; + + catalogs->Ncatalog = 0; + catalogs->NCATALOG = 16; + catalogs->catalog = NULL; + ALLOCATE (catalogs->catalog, Catalog, catalogs->NCATALOG); + + ALLOCATE (catalogs->catIDs, int, catalogs->NCATALOG); + ALLOCATE (catalogs->NAVERAGE, off_t, catalogs->NCATALOG); + ALLOCATE (catalogs->NMEASURE, off_t, catalogs->NCATALOG); + + for (i = 0; i < catalogs->NCATALOG; i++) { + dvo_catalog_init (&catalogs->catalog[i], TRUE); + catalogs->catIDs[i] = 0; + catalogs->NAVERAGE[i] = 256; + catalogs->NMEASURE[i] = 256; + catalogs->catalog[i].Naverage = 0; + catalogs->catalog[i].Nmeasure = 0; + ALLOCATE (catalogs->catalog[i].averageT, AverageTiny, catalogs->NAVERAGE[i]); + ALLOCATE (catalogs->catalog[i].measureT, MeasureTiny, catalogs->NMEASURE[i]); + ALLOCATE (catalogs->catalog[i].secfilt, SecFilt, catalogs->NAVERAGE[i]*Nsecfilt); + } + return catalogs; +} + +// distribute a bright catalog across separate catalogs +int BrightCatalogSplitFree (CatalogSplitter *catalogs) { + + // XXX don't free the catalogs : + // for (i = 0; i < NCATALOG; i++) { + // free (catalogs->catalog[i].averageT); + // free (catalogs->catalog[i].measureT); + // free (catalogs->catalog[i].secfilt); + // } + // free (catalogs->catalog); + + free (catalogs->catIDs); + free (catalogs->NAVERAGE); + free (catalogs->NMEASURE); + free (catalogs->index); + free (catalogs); + return TRUE; +} + +// distribute a bright catalog across separate catalogs +int BrightCatalogSplit (CatalogSplitter *catalogs, BrightCatalog *bcatalog) { + + int i; + + int D_NCATALOG = 16; + + int Nsecfilt = catalogs->Nsecfilt; + + // find the max value of catID in this BrightCatalog + int catIDmax = 0; + for (i = 0; i < bcatalog->Naverage; i++) { + catIDmax = MAX(catIDmax, bcatalog->average[i].catID); + } + // XXX validate the measure ID range here + + int maxIDold = catalogs->maxID; + catalogs->maxID = MAX (maxIDold, catIDmax); + + // extend the index array and init + REALLOCATE (catalogs->index, int, catalogs->maxID + 1); + for (i = maxIDold + 1; i <= catalogs->maxID; i++) catalogs->index[i] = -1; + + // identify the new catID values + for (i = 0; i < bcatalog->Naverage; i++) { + int catID = bcatalog->average[i].catID; + assert (catID > 0); + assert (catID < catalogs->maxID + 1); + int idx = catalogs->index[catID]; + if (idx != -1) continue; // already have seen this one + + // a new catID value + int Ncat = catalogs->Ncatalog; // the next available slot + catalogs->catIDs[Ncat] = catID; + assert (Ncat >= 0); + assert (Ncat < catalogs->NCATALOG); + + catalogs->catalog[Ncat].Nsecfilt = Nsecfilt; + catalogs->catalog[Ncat].catID = catID; + + catalogs->index[catID] = Ncat; + assert (catID > 0); + assert (catID < catalogs->maxID + 1); + + catalogs->Ncatalog ++; + + if (catalogs->Ncatalog >= catalogs->NCATALOG) { + catalogs->NCATALOG += D_NCATALOG; + + // fprintf (stderr, "realloc catalogs->catalog: old: %llx ", (long long) catalogs->catalog); + REALLOCATE (catalogs->catalog, Catalog, catalogs->NCATALOG); + // fprintf (stderr, "new: %llx - %llx\n", (long long) catalogs->catalog, (long long) (catalogs->catalog + sizeof(Catalog)*catalogs->NCATALOG)); + + // fprintf (stderr, "realloc catalogs->NAVERAGE: old: %llx ", (long long) catalogs->NAVERAGE); + REALLOCATE (catalogs->NAVERAGE, off_t, catalogs->NCATALOG); + // fprintf (stderr, "new: %llx - %llx\n", (long long) catalogs->NAVERAGE, (long long) (catalogs->NAVERAGE + sizeof(off_t)*catalogs->NCATALOG)); + + // fprintf (stderr, "realloc catalogs->NMEASURE: old: %llx ", (long long) catalogs->NMEASURE); + REALLOCATE (catalogs->NMEASURE, off_t, catalogs->NCATALOG); + // fprintf (stderr, "new: %llx - %llx\n", (long long) catalogs->NMEASURE, (long long) (catalogs->NMEASURE + sizeof(off_t)*catalogs->NCATALOG)); + + REALLOCATE (catalogs->catIDs, int, catalogs->NCATALOG); + + int j; + for (j = catalogs->NCATALOG - D_NCATALOG; j < catalogs->NCATALOG; j++) { + dvo_catalog_init (&catalogs->catalog[j], TRUE); + catalogs->catIDs[j] = 0; + catalogs->NAVERAGE[j] = 256; + catalogs->NMEASURE[j] = 256; + catalogs->catalog[j].Naverage = 0; + catalogs->catalog[j].Nmeasure = 0; + ALLOCATE (catalogs->catalog[j].averageT, AverageTiny, catalogs->NAVERAGE[j]); + ALLOCATE (catalogs->catalog[j].measureT, MeasureTiny, catalogs->NMEASURE[j]); + ALLOCATE (catalogs->catalog[j].secfilt, SecFilt, catalogs->NAVERAGE[j]*Nsecfilt); + } + D_NCATALOG = MAX(2000, 2*D_NCATALOG); + } + } + + // each bcatalog (from each host) has a different set of catID ranges + // they also (probably) have contiguous ranges. But, it is a bit tricky to be sure I + // can use that info, so perhaps ignore it + + // assign the averages to the corresponding catalog + for (i = 0; i < bcatalog->Naverage; i++) { + int ID = bcatalog->average[i].catID; + int Nc = catalogs->index[ID]; + assert (Nc > -1); + assert (Nc < catalogs->NCATALOG); + + int Na = catalogs->catalog[Nc].Naverage; + catalogs->catalog[Nc].averageT[Na] = bcatalog->average[i]; + + // secfilt entries are grouped in blocks for each average + int k; + for (k = 0; k < Nsecfilt; k++) { + catalogs->catalog[Nc].secfilt[Na*Nsecfilt + k] = bcatalog->secfilt[i*Nsecfilt + k]; + } + + catalogs->catalog[Nc].Naverage ++; + if (catalogs->catalog[Nc].Naverage >= catalogs->NAVERAGE[Nc]) { + catalogs->NAVERAGE[Nc] += MAX(catalogs->NAVERAGE[Nc], 1024); + REALLOCATE (catalogs->catalog[Nc].averageT, AverageTiny, catalogs->NAVERAGE[Nc]); + REALLOCATE (catalogs->catalog[Nc].secfilt, SecFilt, catalogs->NAVERAGE[Nc]*Nsecfilt); + } + } + + // assign the measures to the corresponding catalog + // XXX what about averef and related links? Do I need them? + for (i = 0; i < bcatalog->Nmeasure; i++) { + int ID = bcatalog->measure[i].catID; + int Nc = catalogs->index[ID]; + assert (Nc > -1); + assert (Nc < catalogs->NCATALOG); + int Na = catalogs->catalog[Nc].Nmeasure; + catalogs->catalog[Nc].measureT[Na] = bcatalog->measure[i]; + catalogs->catalog[Nc].Nmeasure ++; + if (catalogs->catalog[Nc].Nmeasure >= catalogs->NMEASURE[Nc]) { + catalogs->NMEASURE[Nc] += MAX(catalogs->NMEASURE[Nc], 4096); + REALLOCATE (catalogs->catalog[Nc].measureT, MeasureTiny, catalogs->NMEASURE[Nc]); + } + } + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/ConfigInit.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ConfigInit.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ConfigInit.c (revision 41602) @@ -0,0 +1,100 @@ +# include "relphot.h" + +// do not use with %s +# define DefConfig(NAME, FMT, DEF, VAR) { \ + char *status = ScanConfig (config, NAME, FMT, 0, &VAR); \ + if (status == NULL) { VAR = DEF; } \ + } + +void ConfigInit (int *argc, char **argv) { + + double ZERO_POINT; + char *config, *file; + char CatdirPhotcodeFile[256]; + + /*** load configuration info ***/ + file = SelectConfigFile (argc, argv, "ptolemy"); + config = LoadConfigFile (file); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find configuration file %s\n", file); + if (file != (char *) NULL) free (file); + exit (0); + } + if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); + + GetConfig (config, "MAG_LIM", "%lf", 0, &MAG_LIM); + GetConfig (config, "SIGMA_LIM", "%lf", 0, &SIGMA_LIM); + + + GetConfig (config, "STAR_SCATTER", "%lf", 0, &STAR_SCATTER); + + GetConfig (config, "IMAGE_SCATTER", "%lf", 0, &IMAGE_SCATTER); + GetConfig (config, "IMAGE_OFFSET", "%lf", 0, &IMAGE_OFFSET); + + GetConfig (config, "STAR_CHISQ", "%lf", 0, &STAR_CHISQ); + GetConfig (config, "GRID_TOOFEW", "%d", 0, &GRID_TOOFEW); + GetConfig (config, "STAR_TOOFEW", "%d", 0, &STAR_TOOFEW); + GetConfig (config, "IMAGE_TOOFEW", "%d", 0, &IMAGE_TOOFEW); + GetConfig (config, "IMAGE_GOOD_FRACTION", "%lf", 0, &IMAGE_GOOD_FRACTION); + + // force CATDIR to be absolute (so parallel mode will work) + char *tmpcatdir = NULL; + ALLOCATE (tmpcatdir, char, DVO_MAX_PATH); + GetConfig (config, "CATDIR", "%s", 0, tmpcatdir); + CATDIR = abspath (tmpcatdir, DVO_MAX_PATH); + free (tmpcatdir); + + GetConfig (config, "CAMERA", "%s", 0, CAMERA); + + GetConfig (config, "GSCFILE", "%s", 0, GSCFILE); + ScanConfig (config, "CATMODE", "%s", 0, CATMODE); + ScanConfig (config, "CATFORMAT", "%s", 0, CATFORMAT); + + snprintf (ImageCat, DVO_MAX_PATH, "%s/Images.dat", CATDIR); + + DefConfig ("RELPHOT_IMFIT_SYS_SIGMA_LIM", "%lf", 0.01, IMFIT_SYS_SIGMA_LIM); + DefConfig ("NIGHT_SCATTER", "%lf", 0.05, NIGHT_SCATTER); + DefConfig ("NIGHT_OFFSET", "%lf", 0.50, NIGHT_OFFSET); + DefConfig ("SKY_DEPTH", "%d", 2, SKY_DEPTH); + + if (!ScanConfig (config, "SKY_TABLE", "%s", 0, SKY_TABLE)) { + SKY_TABLE[0] = 0; + } + + GetConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + + GetConfig (config, "RELPHOT_GRID_X", "%d", 0, &RELPHOT_GRID_X); + GetConfig (config, "RELPHOT_GRID_Y", "%d", 0, &RELPHOT_GRID_Y); + GetConfig (config, "RELPHOT_GRID_BINNING", "%d", 0, &RELPHOT_GRID_BINNING); + GetConfig (config, "CAMERA_CONFIG", "%s", 0, CameraConfig); + GetConfig (config, "MOSAICNAME", "%s", 0, MOSAICNAME); + + if (*CATMODE == 0) strcpy (CATMODE, "RAW"); + if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR"); + + /* XXX this does not yet write out the master photcode table */ + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } + SetZeroPoint (ZERO_POINT); + + FreeConfigFile(); + free (config); + free (file); + +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} + Index: /branches/eam_branches/relphot.20210521/src/GridOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/GridOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/GridOps.c (revision 41602) @@ -0,0 +1,898 @@ +# include "relphot.h" + +enum { + GRID_FITTED, + GRID_FROZEN, + GRID_REFERENCE, +}; + +static int Ngrid; // number of grid elements (gridX * gridY) +static float *gridM; // magnitude offset for this grid cell +static float *gridS; // stdev of the magnitude offset for this grid cell +static int *gridN; // number of stars used to measure the magnitude offset for this grid cell +static int *gridV; // data mode for this cell: fitted, frozen, reference +static int gridX; // number of grid elements in X direction +static int gridY; // number of grid elements in Y direction + +static int **bin; // link from catalog, measure to grid element +static int **Xmeas; // grid x-coordinate for a measurement +static int **Ymeas; // grid y-coordinate for a measurement + +static int **clist; // link from measurement on a cell to catalog containing measurement +static off_t **mlist; // link from measurement on a cell to measurement in a catalog +static off_t *Nlist; // number of measurements for each grid cell +static off_t *NLIST; // allocated number of measurements for each grid cell + +static struct { + int Nchip; + int Mx, My; /* mosaic size in chips */ + int Nx, Ny; /* chip size in pixels */ + int *Fx, *Fy; /* chip flip */ + int *Ox, *Oy; /* chip offset */ + int *valid; + char **ccdname; +} camera; + +static int *ccdnum; +static char *config; + +void initGrid (int dX, int dY) { + OHANA_UNUSED_PARAM(dX); + OHANA_UNUSED_PARAM(dY); + + int i, N, ccdnum_max, refX, refY, refBin; + char *p, field[64], line[256]; + int *Fx, *Fy; /* chip flip */ + int *Ox, *Oy; /* chip offset */ + char **ccdname; + + /* load camera config file */ + config = LoadConfigFile (CameraConfig); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find camera config file %s\n", CameraConfig); + exit (1); + } + + /* load basic mosaic parameters */ + ScanConfig (config, "NCCD", "%d", 1, &camera.Nchip); + ScanConfig (config, "MOSAIC_X", "%d", 1, &camera.Mx); + ScanConfig (config, "MOSAIC_Y", "%d", 1, &camera.My); + ScanConfig (config, "NAXIS1", "%d", 1, &camera.Nx); + ScanConfig (config, "NAXIS2", "%d", 1, &camera.Ny); + + ScanConfig (config, "REFCELL.X", "%d", 1, &refX); + ScanConfig (config, "REFCELL.Y", "%d", 1, &refY); + + // temporary storage + ALLOCATE (Fx, int, camera.Nchip); + ALLOCATE (Fy, int, camera.Nchip); + ALLOCATE (Ox, int, camera.Nchip); + ALLOCATE (Oy, int, camera.Nchip); + ALLOCATE (ccdname, char *, camera.Nchip); + ALLOCATE (ccdnum, int, camera.Nchip); + + /* load per-chip parameters */ + for (i = 0; i < camera.Nchip; i++) { + ALLOCATE (ccdname[i], char, 256); + sprintf (field, "CCD.%d", i); + ScanConfig (config, field, "%s", 1, line); + // XXX get error status! + + sscanf (line, "%s %d %d %d %d", ccdname[i], &Ox[i], &Oy[i], &Fx[i], &Fy[i]); + + p = ccdname[i]; + while (!isdigit(*p) && *p) p++; + if (*p == 0) continue; + ccdnum[i] = atoi (p); + } + + /* we now have the parameters loaded into a minimal length list; reshuffle to the a list of length 0 - MAX(ccdnum) */ + ccdnum_max = 0; + for (i = 0; i < camera.Nchip; i++) { + ccdnum_max = MAX(ccdnum_max, ccdnum[i]); + } + ccdnum_max ++; + + if (ccdnum_max < camera.Nchip) { + fprintf (stderr, "problem with camera config: duplicate ccd IDs\n"); + exit (1); + } + + if (ccdnum_max > 0x1000) { + fprintf (stderr, "problem with camera config: absurd max ccd ID number %d\n", ccdnum_max); + exit (1); + } + + ALLOCATE (camera.Fx, int, ccdnum_max); + ALLOCATE (camera.Fy, int, ccdnum_max); + ALLOCATE (camera.Ox, int, ccdnum_max); + ALLOCATE (camera.Oy, int, ccdnum_max); + ALLOCATE (camera.valid, int, ccdnum_max); + ALLOCATE (camera.ccdname, char *, ccdnum_max); + + for (i = 0; i < ccdnum_max; i++) { + camera.valid[i] = FALSE; + } + + for (i = 0; i < camera.Nchip; i++) { + N = ccdnum[i]; + camera.Fx[N] = Fx[i]; + camera.Fy[N] = Fy[i]; + camera.Ox[N] = Ox[i]; + camera.Oy[N] = Oy[i]; + camera.ccdname[N] = ccdname[i]; + camera.valid[N] = TRUE; + } + + /* define mosaic 2d correction grid: + * GRID_X is the number of grid pixels per chip in the X direction */ + gridX = RELPHOT_GRID_X * camera.Mx; + gridY = RELPHOT_GRID_Y * camera.My; + Ngrid = gridX * gridY; + + ALLOCATE (gridM, float, Ngrid); + ALLOCATE (gridS, float, Ngrid); + ALLOCATE (gridN, int, Ngrid); + ALLOCATE (gridV, int, Ngrid); + + // the grid bins may have one of three possible states: fitted, frozen, reference + // set the initial values to indicate that the bins are frozen + for (i = 0; i < Ngrid; i++) { + gridM[i] = 0.0; + gridS[i] = 0.0; + gridN[i] = 0; + gridV[i] = GRID_FROZEN; + } + + // refBin is the index of the grid cell which is kept at 0.0 (all others are relative to this) + refBin = refX + refY*gridX; + gridV[refBin] = GRID_REFERENCE; +} + +void initGridBins (Catalog *catalog, int Ncatalog) { + + int i, j; + + if (!USE_GRID) return; + + /* define cat,meas -> grid pointers */ + ALLOCATE (bin, int *, Ncatalog); + ALLOCATE (Xmeas, int *, Ncatalog); + ALLOCATE (Ymeas, int *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (bin[i], int, MAX (catalog[i].Nmeasure, 1)); + ALLOCATE (Xmeas[i], int, MAX (catalog[i].Nmeasure, 1)); + ALLOCATE (Ymeas[i], int, MAX (catalog[i].Nmeasure, 1)); + for (j = 0; j < catalog[i].Nmeasure; j++) bin[i][j] = -1; + } + + /* define grid -> cat,meas pointers */ + ALLOCATE (Nlist, off_t, Ngrid); + ALLOCATE (NLIST, off_t, Ngrid); + ALLOCATE (clist, int *, Ngrid); + ALLOCATE (mlist, off_t *, Ngrid); + + for (i = 0; i < Ngrid; i++) { + Nlist[i] = 0; + NLIST[i] = 100; + ALLOCATE (clist[i], int, NLIST[i]); + ALLOCATE (mlist[i], off_t, NLIST[i]); + } +} + +void freeGridBins (int Ncatalog) { + + int i; + + if (!USE_GRID) return; + + /* define cat,meas -> grid pointers */ + for (i = 0; i < Ncatalog; i++) { + free (bin[i]); + free (Xmeas[i]); + free (Ymeas[i]); + } + free (bin); + free (Xmeas); + free (Ymeas); + + /* define grid -> cat,meas pointers */ + for (i = 0; i < Ngrid; i++) { + free (clist[i]); + free (mlist[i]); + } + free (Nlist); + free (NLIST); + free (clist); + free (mlist); +} + +int setGridMeasure (off_t meas, int cat, double X, double Y, int ccdnum) { + + int ix, iy, Cx, Cy, i; + double x, y; + + /* X, Y are chip coords on chip ccdnum */ + + /* normalize X & Y */ + x = X; + if (camera.Fx[ccdnum]) x = camera.Nx - X; + y = Y; + if (camera.Fy[ccdnum]) y = camera.Ny - Y; + + /* grid coords on the chip */ + Cx = MIN (MAX ((x / camera.Nx) * RELPHOT_GRID_X, 0), RELPHOT_GRID_X - 1); + Cy = MIN (MAX ((y / camera.Ny) * RELPHOT_GRID_Y, 0), RELPHOT_GRID_Y - 1); + + /* coordinates in the grid */ + ix = Cx + camera.Ox[ccdnum]*RELPHOT_GRID_X; + iy = Cy + camera.Oy[ccdnum]*RELPHOT_GRID_Y; + + i = ix + iy*gridX; + + bin[cat][meas] = i; + Xmeas[cat][meas] = x + camera.Ox[ccdnum]*camera.Nx; + Ymeas[cat][meas] = y + camera.Oy[ccdnum]*camera.Ny; + clist[i][Nlist[i]] = cat; + mlist[i][Nlist[i]] = meas; + + // for the moment, add up the total grid count + gridN[i]++; + + Nlist[i] ++; + if (Nlist[i] == NLIST[i]) { + NLIST[i] += 100; + REALLOCATE (clist[i], int, NLIST[i]); + REALLOCATE (mlist[i], off_t, NLIST[i]); + } + return (TRUE); + + fprintf (stderr, "error: star out of grid\n"); + exit (1); +} + +int showGridCount() { + + int ix, iy, i; + + for (iy = 0; iy < gridY; iy++) { + for (ix = 0; ix < gridX; ix++) { + i = ix + iy*gridX; + fprintf (stderr, "%3d ", gridN[i]); + } + fprintf (stderr, "\n"); + } + return (TRUE); +} + +float getMgrid (off_t meas, int cat) { + + int i; + float value; + + if (!USE_GRID) return (0); + i = bin[cat][meas]; + if (i == -1) return (NAN); + + // during the grid annealing process, we skip over grid cells until they have enough + // valid stars to be fitted + if (gridV[i] == GRID_FROZEN) { + return (NAN); + } + + value = gridM[i]; + return (value); +} + +/* direct (non-iterative) solution for Mgrid values for all grid bins */ +void setMgridDirect (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + int **gotstar, **gridmeas; + int i, j, k, Ngood, Nbad, Nmos, Ncal, Nrel, Nsys, Ngrp; + double **A, **B, *Mjx, *Wjx; + float Msys, Mcal, Mmos, Mgrp, Merr, Wsys; + double Mj, Wj; + + if (!USE_GRID) return; + + ALLOCATE (A, double *, Ngrid); + ALLOCATE (B, double *, Ngrid); + for (i = 0; i < Ngrid; i++) { + ALLOCATE (A[i], double, Ngrid); + ALLOCATE (B[i], double, 1); + memset (A[i], 0, Ngrid*sizeof(double)); + memset (B[i], 0, sizeof(double)); + } + + Ngood = Nbad = Ncal = Nmos = Nrel = Nsys = Ngrp = 0; + + ALLOCATE (gotstar, int *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (gotstar[i], int, catalog[i].Naverage); + } + + // set up gridmeas table : grid index for each measurement + ALLOCATE (gridmeas, int *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (gridmeas[i], int, catalog[i].Nmeasure); + for (j = 0; j < catalog[i].Nmeasure; j++) { + gridmeas[i][j] = -1; + } + } + for (i = 0; i < Ngrid; i++) { + for (j = 0; j < Nlist[i]; j++) { + int m, c; + m = mlist[i][j]; + c = clist[i][j]; + gridmeas[c][m] = i; + } + } + + // as we loop over the grid cells, we need to accumulate the values of Wjx for each star + ALLOCATE (Wjx, double, Ngrid); + ALLOCATE (Mjx, double, Ngrid); + + // accumulate the elements of the matrix equation. We have an equation of the form: Ax = B + // where x is the vector of grid cell values G_x (x = 0 - Ngrid), A is an Ngrid x Ngrid matrix, + // and B is an Ngrid vector. For a cell A(x,y), we need the following elements from each + // star which touches grid cell (x): + // + // Mj : sum over all measurements of Msys / dMsys^2 + // Wj : sum over all measurements of 1.0 / dMsys^2 + // Mjx : sum over all measurements which touch cell (x) of Msys / dMsys^2 + // Wjx : sum over all measurements which touch cell (x) of 1.0 / dMsys^2 + + // Mjx and Wjx can be calculated by summing over all measurements which touch the cell + // Mj requires looping over stars which touch (x) + + // this is tricky because we need to know both the measurements which touch a cell + // and the stars for which any measurement touches a cell. we need to accumulate + // sums for each star which touches as cell on both bases. + + int Nsecfilt = GetPhotcodeNsecfilt (); + int thisCode = photcodes[0][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + for (i = 0; i < Ngrid; i++) { + + for (j = 0; j < Ncatalog; j++) { + memset (gotstar[j], 0, catalog[j].Naverage*sizeof(int)); + } + + // we are looping over the stars, but doing so by looping over the set of measurements: + // every star which touches this grid cell has a measurement in Nlist[i] + for (j = 0; j < Nlist[i]; j++) { + + int mx, c, n, m0, Npts; + + mx = mlist[i][j]; + c = clist[i][j]; + n = catalog[c].measureT[mx].averef; + + // if we have already visited this star, skip the stuff below + if (gotstar[c][n]) continue; + gotstar[c][n] = TRUE; + + // skip stars marked as BAD + if (catalog[c].secfilt[n*Nsecfilt+Nsec].flags & STAR_BAD) { + Nrel ++; + continue; + } + + m0 = catalog[c].average[n].measureOffset; + + // we accumuate an entry for each cell + memset (Wjx, 0, Ngrid*sizeof(double)); + memset (Mjx, 0, Ngrid*sizeof(double)); + Npts = Mj = Wj = 0.0; + + // if we have not yet visited this star, accumulate the Mj, Wj entries + for (k = 0; k < catalog[c].average[n].Nmeasure; k++) { + + int m, Ng; + + m = m0 + k; + + // skip measurements marked as BAD + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + Nbad ++; + continue; + } + + // skip images marked as BAD + Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) { + Ncal ++; + continue; + } + + // skip mosaics marked as BAD + Mmos = getMmos (m, c); + if (isnan(Mmos)) { + Nmos ++; + continue; + } + + // skip mosaics marked as BAD + Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) { + Ngrp ++; + continue; + } + + // select the color- and airmass-corrected observed magnitude for this star + // XXX need to be able to turn off the color-correction until initial average mags are found + Msys = PhotCatTiny (&catalog[c].measureT[m], MAG_CLASS_PSF); + if (isnan(Msys)) { + Nsys++; + continue; + } + + // mag-error for this measurement + Merr = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + + // disable Wsys for now + Wsys = TRUE ? 1.0 : 1.0 / SQ(Merr); + + Ng = gridmeas[c][m]; + if (Ng == -1) continue; // skip measurements which do not touch any cell + + Mj += Msys * Wsys; // we are only including measurements touching this cell + Wj += Wsys; // we are only including measurements touching this cell + Npts ++; + Ngood ++; + + Mjx[Ng] += Msys * Wsys; // we are only including measurements touching cell (x) + Wjx[Ng] += Wsys; // we are only including measurements touching cell (x) + } + + // some stars will not have any valid measurements, skip these + if (Npts == 0) continue; + + B[i][0] += Mj*Wjx[i]/Wj - Mjx[i]; + A[i][i] -= Wjx[i]; + for (k = 0; k < Ngrid; k++) { + A[i][k] += Wjx[i]*Wjx[k]/Wj; + // fprintf (stderr, "%3.0f ", Wjx[k]); + } + } + } + + if (1) { + + FILE *f; + Header theader; + Matrix matrix; + + /* we are writing to this file */ + f = fopen ("matrix.fits", "w"); + if (f == (FILE *) NULL) { + fprintf (stderr, "cannot open matrix.fits for output\n"); + return; + } + + /* save grid mag values */ + gfits_init_header (&theader); + theader.Naxes = 2; + theader.Naxis[0] = Ngrid; + theader.Naxis[1] = Ngrid; + theader.bitpix = -32; + gfits_create_Theader (&theader, "IMAGE"); + gfits_modify (&theader, "EXTNAME", "%s", 1, "MATRIX"); + gfits_create_matrix (&theader, &matrix); + for (i = 0; i < Ngrid; i++) { + for (j = 0; j < Ngrid; j++) { + gfits_set_matrix_value (&matrix, i, j, (double) A[i][j]); + } + } + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + + gfits_modify (&theader, "EXTNAME", "%s", 1, "TRPOSE"); + gfits_create_matrix (&theader, &matrix); + for (i = 0; i < Ngrid; i++) { + for (j = 0; j < Ngrid; j++) { + gfits_set_matrix_value (&matrix, i, j, (double) A[j][i]); + } + } + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + fclose (f); + } + + dgaussjordan (A, B, Ngrid, 1); + + fprintf (stderr, "grid cells fitted (Ngood: %d, Nbad: %d, Nmos: %d, Ncal: %d, Nrel: %d, Nsys: %d)\n", Ngood, Nbad, Nmos, Ncal, Nrel, Nsys); + + for (i = 0; i < Ngrid; i++) { + gridM[i] = B[i][0]; + gridS[i] = sqrt(A[i][i]); + gridN[i] = Ngood; + } + + free (Wjx); + free (Mjx); + + for (i = 0; i < Ngrid; i++) { + free (A[i]); + } + free (A); + free (B); + + for (i = 0; i < Ncatalog; i++) { + free (gotstar[i]); + free (gridmeas[i]); + } + free (gotstar); + free (gridmeas); +} + +/* determine Mgrid values for all grid bins */ +void setMgrid (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int i, j, m, c, n, N, Nmax, Nbad, Nmos, Ngrp, Ncal, Nrel, Nsys, Nfit; + double *list, *dlist; + float Msys, Mrel, Mcal, Mmos, Mgrp; + + StatType stats; + liststats_setmode (&stats, "INNER_WTMEAN"); + + if (!USE_GRID) return; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Nmax = Nlist[0]; + for (i = 0; i < Ngrid; i++) { + Nmax = MAX (Nmax, Nlist[i]); + } + ALLOCATE (list, double, Nmax); + ALLOCATE (dlist, double, Nmax); + + Nbad = Ncal = Nmos = Ngrp = Nrel = Nsys = Nfit = 0; + + for (i = 0; i < Ngrid; i++) { + + N = 0; + for (j = 0; j < Nlist[i]; j++) { + + m = mlist[i][j]; + c = clist[i][j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + Nbad ++; + continue; + } + Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) { + Ncal ++; + continue; + } + Mmos = getMmos (m, c); + if (isnan(Mmos)) { + Nmos ++; + continue; + } + Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) { + Ngrp ++; + continue; + } + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(Mrel)) { + Nrel ++; + continue; + } + + n = catalog[c].measureT[m].averef; + Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(Msys)) { + Nsys++; + continue; + } + list[N] = Msys - Mrel - Mcal - Mmos - Mgrp; + dlist[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + N++; + } + + // the reference Cell is forced to have a value of 0.0, and is never changed to GRID_FITTED + if (gridV[i] == GRID_REFERENCE) { + gridM[i] = 0.0; + gridS[i] = 0.0; + gridN[i] = N; + continue; + } + + // until we have enough valid measurements on this grid cell, skip it + if (N < GRID_TOOFEW) { + gridV[i] = GRID_FROZEN; + continue; + } + + liststats (list, dlist, NULL, N, &stats); + gridM[i] = stats.mean; + gridS[i] = stats.sigma; + gridN[i] = N; + gridV[i] = GRID_FITTED; + Nfit++; + } + + fprintf (stderr, "%d of %d grid cells fitted (+ reference cell) (Nbad: %d, Nmos: %d, Ncal: %d, Nrel: %d, Nsys: %d)\n", Nfit, Ngrid, Nbad, Nmos, Ncal, Nrel, Nsys); + + free (list); + free (dlist); +} + +void plot_grid (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int i, j, m, c, n, N, Narea; + float Msys, Mrel, Mcal, Mmos, Mgrp; + double *xlist, *Mlist, *dlist, *ylist; + Graphdata graphdata; + + if (!USE_GRID) return; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + N = 0; + for (i = 0; i < Ngrid; i++) + N += Nlist[i]; + + ALLOCATE (xlist, double, N); + ALLOCATE (ylist, double, N); + ALLOCATE (Mlist, double, N); + ALLOCATE (dlist, double, N); + + Narea = 0; + N = 0; + for (i = 0; i < Ngrid; i++) { + for (j = 0; j < Nlist[i]; j++) { + + m = mlist[i][j]; + c = clist[i][j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + Narea ++; + continue; + } + Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + Mmos = getMmos (m, c); + if (isnan(Mmos)) continue; + Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) continue; + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(Mrel)) continue; + + n = catalog[c].measureT[m].averef; + Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + xlist[N] = Xmeas[c][m]; + ylist[N] = Ymeas[c][m]; + Mlist[N] = Msys - Mrel - Mcal - Mmos - Mgrp; + dlist[N] = Msys - Mrel - Mcal - Mmos - Mgrp - gridM[i]; + N++; + } + } + + fprintf (stderr, "skipped %d meas for area\n", Narea); + + plot_defaults (&graphdata); + graphdata.ymin = PlotdMmin; + graphdata.ymax = PlotdMmax; + plot_list (&graphdata, xlist, Mlist, N, "X vs dM raw", "%s.XdM.png", OUTROOT); + plot_list (&graphdata, xlist, dlist, N, "X vs dM corrected", "%s.XdMf.png", OUTROOT); + plot_list (&graphdata, ylist, dlist, N, "Y vs dM corrected", "%s.YdMf.png", OUTROOT); + + plot_defaults (&graphdata); + plot_list (&graphdata, xlist, ylist, N, "X vs Y", "%s.XY.png", OUTROOT); + + free (ylist); + free (xlist); + free (Mlist); + free (dlist); + +} + +void dump_grid () { + + off_t i, Nimage; + int j, Nbytes, Nformat; + FILE *f; + Header header, theader; + Matrix matrix; + Mosaic *refmosaic; + char *filename; + char formatline[32], key[32], value[64]; + + Nbytes = strlen (OUTROOT) + 6; + ALLOCATE (filename, char, Nbytes); + snprintf (filename, Nbytes, "%s.fits", OUTROOT); + + /* select reference mosaic image */ + // off_t *imlist = SelectRefMosaic (&refmosaic, &Nimage); return value ignored + SelectRefMosaic (&refmosaic, &Nimage); + + /* we are writing to this file */ + f = fopen (filename, "w"); + if (f == (FILE *) NULL) { + fprintf (stderr, "cannot open %s for output\n", filename); + free (filename); + return; + } + + /* create empty phu */ + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + gfits_modify (&header, "NEXTEND", OFF_T_FMT, 1, Nimage + 3); + gfits_modify (&header, "FILTER", "%s", 1, photcodes[0][0].name); // XXXX note that this expects a single photcode, enforced in initialize.d + gfits_modify_alt (&header, "COMMENT", "%S", 1, "Mosaic Photometry Grid Analysis"); + + // we need to add lines to the PHU to identify the camera and format; these are used by the ipp config system + // Note that config must have been loaded (and not freed) above. + ScanConfig (config, "NFORMAT", "%d", 1, &Nformat); + for (i = 1; i <= Nformat; i++) { + ScanConfig (config, "FORMAT", "%s", i, formatline); + sscanf (formatline, "%s %s", key, value); + gfits_modify (&header, key, "%s", 1, value); + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + + /* save grid mag values */ + gfits_init_header (&theader); + theader.Naxes = 2; + theader.Naxis[0] = gridX; + theader.Naxis[1] = gridY; + theader.bitpix = -32; + gfits_create_Theader (&theader, "IMAGE"); + gfits_modify (&theader, "FILTER", "%s", 1, photcodes[0][0].name); + gfits_modify (&theader, "EXTNAME", "%s", 1, "MAG_OFFSET"); + gfits_create_matrix (&theader, &matrix); + for (i = 0; i < gridX; i++) { + for (j = 0; j < gridY; j++) { + gfits_set_matrix_value (&matrix, i, j, (double) gridM[i + j*gridX]); + } + } + write_coords (&theader, &refmosaic[0].coords); + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + + /* save grid Nmeas values */ + gfits_modify (&theader, "EXTNAME", "%s", 1, "NMEAS"); + gfits_modify (&theader, "FILTER", "%s", 1, photcodes[0][0].name); + gfits_create_matrix (&theader, &matrix); + for (i = 0; i < gridX; i++) { + for (j = 0; j < gridY; j++) { + gfits_set_matrix_value (&matrix, i, j, (double) gridN[i + j*gridX]); + } + } + write_coords (&theader, &refmosaic[0].coords); + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + + /* save grid sigma values */ + gfits_modify (&theader, "EXTNAME", "%s", 1, "SIGMA"); + gfits_modify (&theader, "FILTER", "%s", 1, photcodes[0][0].name); + gfits_create_matrix (&theader, &matrix); + for (i = 0; i < gridX; i++) { + for (j = 0; j < gridY; j++) { + gfits_set_matrix_value (&matrix, i, j, (double) gridS[i + j*gridX]); + } + } + write_coords (&theader, &refmosaic[0].coords); + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + + /* calculate value for each CCD pixel, write out CCD images */ + /* grid pixels are tied to detector pixels, but are flipped to match focal plane */ + for (i = 0; i < camera.Nchip; i++) { + int N, ix, iy, x, y, X, Y, bin; + + N = ccdnum[i]; + + gfits_modify (&theader, "EXTNAME", "%s", 1, camera.ccdname[N]); + gfits_modify (&theader, "FILTER", "%s", 1, photcodes[0][0].name); + gfits_modify (&theader, "NX", "%d", 1, camera.Nx); + gfits_modify (&theader, "NY", "%d", 1, camera.Ny); + + theader.Naxis[0] = RELPHOT_GRID_X; + theader.Naxis[1] = RELPHOT_GRID_Y; + gfits_modify (&theader, "NAXIS1", "%d", 1, RELPHOT_GRID_X); + gfits_modify (&theader, "NAXIS2", "%d", 1, RELPHOT_GRID_Y); + gfits_create_matrix (&theader, &matrix); + + for (Y = 0; Y < RELPHOT_GRID_Y; Y++) { + for (X = 0; X < RELPHOT_GRID_X; X++) { + + /* normalize X & Y */ + x = X; + if (camera.Fx[N]) x = RELPHOT_GRID_X - X - 1; + y = Y; + if (camera.Fy[N]) y = RELPHOT_GRID_Y - Y - 1; + + /* coordinates in the grid */ + ix = x + camera.Ox[N]*RELPHOT_GRID_X; + iy = y + camera.Oy[N]*RELPHOT_GRID_Y; + + bin = ix + iy*gridX; + gfits_set_matrix_value (&matrix, X, Y, (double) gridM[bin]); + } + } + gfits_fwrite_header (f, &theader); + gfits_fwrite_matrix (f, &matrix); + gfits_free_matrix (&matrix); + } + + free (filename); +} + +void InterpolateGrid (float *buffer, int Nx, int Ny, Coords *ccd, Coords *gcoords) { + + int i, j; + double x, y, r, d, X, Y, dx, dy; + double V00, V01, V10, V11; + double wV00, wV01, wV10, wV11; + double dV00, dV01, dV10, dV11; + double v1, v2, value; + int ix, iy, N; + + for (i = 0; i < Nx; i++) { + for (j = 0; j < Ny; j++) { + x = i * RELPHOT_GRID_BINNING / 2; + y = j * RELPHOT_GRID_BINNING / 2; + XY_to_RD (&r, &d, x, y, ccd); + RD_to_XY (&X, &Y, r, d, gcoords); + + X = X / RELPHOT_GRID_BINNING; + Y = Y / RELPHOT_GRID_BINNING; + + ix = (int) X; + dx = X - ix; + iy = (int) Y; + dy = Y - iy; + + if (ix < 0) continue; + if (iy < 0) continue; + if (ix >= gridX) continue; + if (iy >= gridY) continue; + + N = ix + iy*gridX; + V00 = gridM[N]; + V10 = gridM[N + 1]; + V01 = gridM[N + gridX]; + V11 = gridM[N + gridX + 1]; + + dV00 = gridS[N]; + dV10 = gridS[N + 1]; + dV01 = gridS[N + gridX]; + dV11 = gridS[N + gridX + 1]; + + wV00 = (dV00 == 0) ? 0.0 : 1 / SQ(dV00); + wV01 = (dV01 == 0) ? 0.0 : 1 / SQ(dV01); + wV10 = (dV10 == 0) ? 0.0 : 1 / SQ(dV10); + wV11 = (dV11 == 0) ? 0.0 : 1 / SQ(dV11); + + v1 = wV00*V00*(1 + dx*dy - dx - dy) + + wV10*V10*(dx - dx*dy) + + wV01*V01*(dy - dx*dy) + + wV11*V11*(dx*dy); + + v2 = wV00*(1 + dx*dy - dx - dy) + + wV10*(dx - dx*dy) + + wV01*(dy - dx*dy) + + wV11*(dx*dy); + + value = v1 / v2; + buffer[j*Nx + i] = value; + } + } +} Index: /branches/eam_branches/relphot.20210521/src/ImageMagIO.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ImageMagIO.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ImageMagIO.c (revision 41602) @@ -0,0 +1,230 @@ +# include "relphot.h" + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// this is nearly identical to the one in 'uniphot/src' used by setphot / setphot_client. +// Here, we use a handful of different columns (if not, we could move to libdvo) +ImageMag *ImageMagLoad(char *filename, off_t *nimage_mags) { + + int i, Ncol; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + *nimage_mags = 0; + ImageMag *image_mags = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return (NULL); + } + fclose (f); + + // we read the entire block of data, then extract the columns, then set the image structure values. + // I free the FITS table data after extracting the colums to avoid having 3 copies in memory. + + char type[16]; + + GET_COLUMN (McalPSF, "MCAL_PSF", float); + GET_COLUMN (McalAPER, "MCAL_APER", float); + GET_COLUMN (dMcal, "MCAL_ERR", float); + GET_COLUMN (dMagSys, "MCAL_SYSERR", float); + GET_COLUMN (McalChiSq, "MCAL_CHISQ", float); + GET_COLUMN (nFitPhotom, "NFIT", int); + GET_COLUMN (flags, "FLAGS", int); + GET_COLUMN (ubercalDist, "UDIST", int); + GET_COLUMN (imageID, "ID", int); + + // free the memory associated with the FITS files + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + ALLOCATE (image_mags, ImageMag, Nrow); + for (i = 0; i < Nrow; i++) { + image_mags[i].McalPSF = McalPSF [i]; + image_mags[i].McalAPER = McalAPER [i]; + image_mags[i].dMcal = dMcal [i]; + image_mags[i].dMagSys = dMagSys [i]; + image_mags[i].McalChiSq = McalChiSq [i]; + image_mags[i].nFitPhotom = nFitPhotom [i]; + image_mags[i].flags = flags [i]; + image_mags[i].ubercalDist = ubercalDist[i]; + image_mags[i].imageID = imageID [i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (McalPSF ); + free (McalAPER ); + free (dMcal ); + free (dMagSys ); + free (McalChiSq ); + free (nFitPhotom ); + free (flags ); + free (ubercalDist); + free (imageID ); + + *nimage_mags = Nrow; + return image_mags; +} + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +int ImageMagSave(char *filename, ImageMag *image_mags, off_t Nimage_mags) { + + int i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_MAGS"); + + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "PSF cal offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER", "APER cal offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_ERR", "cal error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_SYSERR", "systematic error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_CHISQ", "cal chisq", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "NFIT", "number of fitted stars", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "FLAGS", "analysis flags", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "UDIST", "distance to ubercal images", "images", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "ID", "image ID", "unitless", 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + float *McalPSF ; + float *McalAPER ; + float *dMcal ; + float *dMagSys ; + float *McalChiSq ; + int *nFitPhotom ; + int *flags ; + int *ubercalDist ; + int *imageID ; + + // create intermediate storage arrays + ALLOCATE (McalPSF , float, Nimage_mags); + ALLOCATE (McalAPER , float, Nimage_mags); + ALLOCATE (dMcal , float, Nimage_mags); + ALLOCATE (dMagSys , float, Nimage_mags); + ALLOCATE (McalChiSq , float, Nimage_mags); + ALLOCATE (nFitPhotom , int, Nimage_mags); + ALLOCATE (flags , int, Nimage_mags); + ALLOCATE (ubercalDist , int, Nimage_mags); + ALLOCATE (imageID , int, Nimage_mags); + + // assign the storage arrays + for (i = 0; i < Nimage_mags; i++) { + McalPSF [i] = image_mags[i].McalPSF ; + McalAPER [i] = image_mags[i].McalAPER ; + dMcal [i] = image_mags[i].dMcal ; + dMagSys [i] = image_mags[i].dMagSys ; + McalChiSq [i] = image_mags[i].McalChiSq ; + nFitPhotom [i] = image_mags[i].nFitPhotom ; + flags [i] = image_mags[i].flags ; + ubercalDist[i] = image_mags[i].ubercalDist; + imageID [i] = image_mags[i].imageID ; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_SYSERR", dMagSys , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "MCAL_CHISQ", McalChiSq , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "NFIT", nFitPhotom , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags , Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "UDIST", ubercalDist, Nimage_mags); + gfits_set_bintable_column (&theader, &ftable, "ID", imageID , Nimage_mags); + + free (McalPSF ); + free (McalAPER ); + free (dMcal ); + free (dMagSys ); + free (McalChiSq ); + free (nFitPhotom ); + free (flags ); + free (ubercalDist); + free (imageID ); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image_mags file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for image_mags %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for image_mags %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for image_mags %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for image_mags %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file image_mags %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file image_mags %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing image_mags file %s\n", filename); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/ImageOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ImageOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ImageOps.c (revision 41602) @@ -0,0 +1,1299 @@ +# include "relphot.h" + +// the MeasureToImage, ImageToCatalog, and ImageToMeasure arrays are a substantial part of the memory footprint. +// total data volume is currently: +// a) Naverage*sizeof(AverageTiny) [averages] : 32 +// b) Naverage*sizeof(Secfilt)*Nsecfilt [secfilt values] : 8 * 32 +// b) Nmeasure*sizeof(Measure) [measurements] : 72 +// c) Nmeasure*sizeof(IDX_T)*3 [image idx] : 3 * 16 +// d) Nmeasure*sizeof(IDX_T)*3 [mosaic idx] : 3 * 16 +// e) Nimage*sizeof(Image) [image data] : 360 + +// for 3pi analysis in 2012, we have Nmeasure ~ 20 x Naverage +// so, each measurement costs ~14.5B (Ave + Sec) + 72B (Meas) + 96B (idx)! +// we are using off_t (64bit) to avoid the 32bit limit of an int, but +// if we really had 2^31 measurements in a single analysis, we would be using > 350GB of ram... +// until we reach that point, it is sort of silly to use IDX_T = off_t here. + +// in fact, we are safer than this, because the number of detections per table is rarely so large. + +// with IDX_T = int, each measurement costs ~14.5B (Ave + Sec) + 72B (Meas) + 48B (idx)! + +// elsewhere (not in relphot_images), we need to use off_t because a single catalog + +// we have an array of images: image[ImageIndex] (ImageIndex : 0 < Nimage) +static off_t Nimage = 0; // number of available images +static Image *image = NULL; // array of available images + +// if we read only a subset of the rows from the Image FITS, LineNumber tells us to which row +// each image belongs +static off_t *LineNumber; // match of subset to full image table + +// to search by image ID, we sort (imageIDs, imageIdx) by imageIDs to get a sorted index +static off_t *imageIDs; // list of all image IDs +static off_t *imageIdx; // list of index for image IDs + +// QQ // as an alternative, we generate imageSeq, which directly maps imageID -> seq +// QQ static off_t *imageSeq; // list of index for image IDs +// QQ +// QQ // MAX_ID requires 512M to store the image index +// QQ # define MAX_ID 0x8000000 +// QQ static off_t minImageID = MAX_ID; +// QQ static off_t maxImageID = 0; + +// elsewhere, we have loaded a set of catalogs with measures (catalog[cat].measure[meas]) +// each image has N_onImage[ImageIndex] measurements +static off_t *N_onImage; // actual number of measurements on image +static off_t *N_ONIMAGE; // allocated number of measurements on image + +// relationships between the measure,catalog set and the images: +static IDX_T **MeasureToImage = NULL; // image index from measure,catalog : MeasureToImage[cat][meas] = ImageIndex +static IDX_T **ImageToCatalog = NULL; // catalog for given measure on image : ImageCatalog[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex]) +static IDX_T **ImageToMeasure = NULL; // measure for given measure on image : ImageMeasure[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex]) + +// Projection Cell / SkyCell ID : for stack primary detection, we need to know the projection cell and skycell ID for each +// stack image. for now, we generate these ID arrays based on the image names when we load in the image table (initImages). +// When we pass data to the remote clients via the ImageSubset, the projID/skycellID values are carried directly in the table. +int *tessID = NULL; +int *projectID = NULL; +int *skycellID = NULL; + +// MeasureToImage was 'bin' +// ImageToCatalog was 'clist' +// ImageToMeasure was 'mlist' + +// N_onImage was 'Nlist' +// N_ONIMAGE was 'NLIST' + +Image *getimages (off_t *N, off_t **line_number) { + + *N = Nimage; + if (line_number) *line_number = LineNumber; + return (image); +} + +Image *getimage (off_t N) { + return (&image[N]); +} + +off_t *get_N_onImage() { + return N_onImage; +} + +IDX_T **get_ImageToCatalog() { + return ImageToCatalog; +} + +IDX_T **get_ImageToMeasure() { + return ImageToMeasure; +} + +void initImages (Image *input, off_t *line_number, off_t N) { + + off_t i; + + image = input; + LineNumber = line_number; + Nimage = N; + + ALLOCATE (imageIDs, off_t, Nimage); + ALLOCATE (imageIdx, off_t, Nimage); + + // for stack images, assign projection cell ID and skycell ID based on filenames + ALLOCATE (tessID, int, Nimage); + ALLOCATE (projectID, int, Nimage); + ALLOCATE (skycellID, int, Nimage); + + for (i = 0; i < Nimage; i++) { + imageIdx[i] = i; + imageIDs[i] = image[i].imageID; + + tessID[i] = -1; + projectID[i] = -1; + skycellID[i] = -1; + + TessellationIDsByImageName (&tessID[i], &projectID[i], &skycellID[i], image[i].name); + + // QQ minImageID = MIN(minImageID, image[i].imageID); + // QQ maxImageID = MAX(maxImageID, image[i].imageID); + // QQ myAssert (image[i].imageID < MAX_ID, "image IDs too large for index memory"); + } + + // sort the image index by the IDs + // XXX does this break the imageID <-> projectID, etc match? + llsortpair (imageIDs, imageIdx, Nimage); + + // QQ ALLOCATE (imageSeq, off_t, maxImageID + 1); + // QQ for (i = 0; i < maxImageID + 1; i++) { + // QQ imageSeq[i] = -1; // not yet assigned + // QQ } + // QQ + // QQ for (i = 0; i < Nimage; i++) { + // QQ off_t N = image[i].imageID; + // QQ myAssert (imageSeq[N] == -1, "previously assigned"); + // QQ imageSeq[N] = i; + // QQ } +} + +void initImagesSubset (ImageSubset *input, off_t *line_number, off_t N) { + + off_t i; + + // we have been given an ImageSubset array, containing a reduced set of image fields + // create full a Image array and save the needed values + ALLOCATE (image, Image, N); + + ALLOCATE (tessID, int, N); + ALLOCATE (projectID, int, N); + ALLOCATE (skycellID, int, N); + + for (i = 0; i < N; i++) { + image[i].imageID = input[i].imageID ; + image[i].photom_map_id = input[i].photom_map_id; + image[i].flags = input[i].flags ; + image[i].McalPSF = input[i].McalPSF ; + image[i].McalAPER = input[i].McalAPER ; + image[i].dMcal = input[i].dMcal ; + image[i].tzero = input[i].tzero ; + image[i].trate = input[i].trate ; + image[i].ubercalDist = input[i].ubercalDist ; + tessID[i] = input[i].tessID ; + projectID[i] = input[i].projID ; + skycellID[i] = input[i].skycellID ; + } + LineNumber = line_number; + Nimage = N; + + ALLOCATE (imageIDs, off_t, Nimage); + ALLOCATE (imageIdx, off_t, Nimage); + + for (i = 0; i < Nimage; i++) { + imageIdx[i] = i; + imageIDs[i] = image[i].imageID; + } + llsortpair (imageIDs, imageIdx, Nimage); +} + +ImageSubset *getimages_subset (off_t *N) { + + *N = Nimage; + + off_t i; + + // we have been given an ImageSubset array, containing a reduced set of image fields + // create full a Image array and save the needed values + ImageSubset *subset = NULL; + ALLOCATE (subset, ImageSubset, Nimage); + for (i = 0; i < Nimage; i++) { + subset[i].imageID = image[i].imageID ; + subset[i].photom_map_id = image[i].photom_map_id; + subset[i].flags = image[i].flags ; + subset[i].McalPSF = image[i].McalPSF ; + subset[i].McalAPER = image[i].McalAPER ; + subset[i].dMcal = image[i].dMcal ; + subset[i].tzero = image[i].tzero ; + subset[i].trate = image[i].trate ; + subset[i].ubercalDist = image[i].ubercalDist ; + subset[i].tessID = tessID[i]; + subset[i].projID = projectID[i]; + subset[i].skycellID = skycellID[i]; + } + return subset; +} + +off_t getImageByID (off_t ID) { + + // QQ if (imageSeq) { + // QQ myAssert (ID >= minImageID, "oops"); + // QQ myAssert (ID <= maxImageID, "oops"); + // QQ off_t N = imageSeq[ID]; + // QQ return N; + // QQ } + + // we have a pair of vectors (imageIDs, imageIdx) sorted by imageIDs + // use bisection to find the specified image ID + + off_t Nlo, Nhi, N; + + Nlo = 0; Nhi = Nimage; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (imageIDs[N] < ID) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, Nimage); + } + } + + for (N = Nlo; N < Nhi; N++) { + if (imageIDs[N] == ID) + return (imageIdx[N]); + } + + return (-1); +} + +void initImageBins (Catalog *catalog, int Ncatalog, int doImageList) { + + IDX_T i, j; + + ALLOCATE (MeasureToImage, IDX_T *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (MeasureToImage[i], IDX_T, MAX (catalog[i].Nmeasure, 1)); + for (j = 0; j < catalog[i].Nmeasure; j++) MeasureToImage[i][j] = -1; + } + + if (doImageList) { + ALLOCATE (N_onImage, off_t, Nimage); + ALLOCATE (N_ONIMAGE, off_t, Nimage); + ALLOCATE (ImageToCatalog, IDX_T *, Nimage); + ALLOCATE (ImageToMeasure, IDX_T *, Nimage); + + for (i = 0; i < Nimage; i++) { + N_onImage[i] = 0; + N_ONIMAGE[i] = 30; + ALLOCATE (ImageToCatalog[i], IDX_T, N_ONIMAGE[i]); + ALLOCATE (ImageToMeasure[i], IDX_T, N_ONIMAGE[i]); + } + } +} + +void freeImageBins (int Ncatalog, int doImageList) { + + off_t i; + + for (i = 0; i < Ncatalog; i++) { + free (MeasureToImage[i]); + } + free (MeasureToImage); + + if (doImageList) { + for (i = 0; i < Nimage; i++) { + free (ImageToCatalog[i]); + free (ImageToMeasure[i]); + } + free (ImageToCatalog); + free (ImageToMeasure); + free (N_onImage); + free (N_ONIMAGE); + } +} + +void freeImages (char *dbImagePtr) { + + FREE (LineNumber); + + FREE (imageIDs); + FREE (imageIdx); + + FREE (tessID); + FREE (projectID); + FREE (skycellID); + + // we call gfits_db_free as well as this function. sometimes those point at the same + // memory location, in which case we should only do the free once. + if ((void *) dbImagePtr != (void *) image) free (image); + + free_astrom_table(); +} + +/* select all images equivalent to the active photcode set */ +void findImages (Catalog *catalog, int Ncatalog, int doImageList) { + + int Nmatch = 0; + for (int i = 0; i < Ncatalog; i++) { + for (off_t j = 0; j < catalog[i].Nmeasure; j++) { + catalog[i].measureT[j].myDet = FALSE; // a detetion is not mine until proven otherwise + + // skip measurements which do not match one of the requested photcodes ( + // (do we not already exclude in bcatalog -- maybe needed for reload_objects? + int Ns = GetActivePhotcodeIndex (catalog[i].measureT[j].photcode); + if (Ns < 0) continue; + + // if we match one of our images, myDet gets set to TRUE + matchImage (catalog, j, i, doImageList); + Nmatch ++; + } + } + // fprintf (stderr, "matched %d detections to images\n", Nmatch); +} + +int findCCD (off_t idx, off_t meas, int cat, MeasureTiny *measure) { + + int ccdnum; + double X, Y; + char *pname, *filter, *p, base[256]; + + /* identify the ccd on the basis of the photcode name */ + pname = GetPhotcodeNamebyCode (image[idx].photcode); + + // skip measurements which do not match one of the requested photcodes ( + // (do we not already exclude in bcatalog -- maybe needed for reload_objects? + int Ns = GetActivePhotcodeIndex (measure[0].photcode); + if (Ns < 0) return FALSE; + + filter = photcodes[Ns][0].name; + sprintf (base, "%s.%s.", MOSAICNAME, filter); + if (strncmp (pname, base, strlen (base))) return (FALSE); + p = pname + strlen(base); + + /* p is pointing at the DETECTOR ID part of the photcode: CAMERA.FILTER.DETECTOR + for now, we will let this be of the form SSSSnn where SSS is an arbitrary string + and nn is an integer -> ccdnum */ + + /// XXX we have an inconsistency here wrt dvo.layout and dvo.photcode + + while (!isdigit(*p) && *p) p++; + if (*p == 0) return (FALSE); + ccdnum = atoi (p); + + /* ccdnum is an integer, but not necessarily a sequence number. when the camera layout is + constructed, there will be null values for undefined ccdnums */ + + // old code to add this measurement to the grid cell for this chip + // ave = measureT[0].averef; + // ra = catalog[cat].averageT[ave].R - measureT[0].dR / 3600.0; + // dec = catalog[cat].averageT[ave].D - measureT[0].dD / 3600.0; + // RD_to_XY (&X, &Y, ra, dec, &image[i].coords); + + // XXX we can now use these values (but need to be careful about old formats) + X = measure[0].Xccd; + Y = measure[0].Yccd; + setGridMeasure (meas, cat, X, Y, ccdnum); + + return (TRUE); +} + +void matchImage (Catalog *catalog, off_t meas, int cat, int doImageList) { + + off_t idx, ID; + int status; + MeasureTiny *measure; + + measure = &catalog[cat].measureT[meas]; + + ID = measure[0].imageID; + idx = getImageByID (ID); + if (idx == -1) { + if (VERBOSE2) fprintf (stderr, "can't match detection to image?\n"); + return; + } + catalog[cat].measureT[meas].myDet = TRUE; + + if (USE_GRID) { + status = findCCD (idx, meas, cat, measure); + if (!status) { + if (VERBOSE2) fprintf (stderr, "failed to determine CCD for "OFF_T_FMT", %d\n", meas, cat); + return; + } + } + + // index for (catalog, measure) -> image + MeasureToImage[cat][meas] = idx; + + if (doImageList) { + // index for image, Nentry -> catalog + ImageToCatalog[idx][N_onImage[idx]] = cat; + + // index for image, Nentry -> measure + ImageToMeasure[idx][N_onImage[idx]] = meas; + N_onImage[idx] ++; + + if (N_onImage[idx] == N_ONIMAGE[idx]) { + N_ONIMAGE[idx] += 30; + REALLOCATE (ImageToCatalog[idx], IDX_T, N_ONIMAGE[idx]); + REALLOCATE (ImageToMeasure[idx], IDX_T, N_ONIMAGE[idx]); + } + } + + return; +} + +off_t getImageEntry (off_t meas, int cat) { + + off_t i; + + if (!MeasureToImage) return -1; + + i = MeasureToImage[cat][meas]; + return (i); +} + +int getImageFlags (off_t meas, int cat) { + + off_t i; + + if (!MeasureToImage) return 0; + + i = MeasureToImage[cat][meas]; + + return (image[i].flags); +} + +// returns image.McalPSF or image.McalAPER +// NOTE: static flat-field component is included in measure.Mflat +float getMcal (off_t meas, int cat, dvoMagClassType class) { + + off_t i; + + i = MeasureToImage[cat][meas]; + if (i == -1) return NAN; + + if (image[i].flags & IMAGE_BAD) return NAN; + + switch (class) { + case MAG_CLASS_PSF: + return image[i].McalPSF; + case MAG_CLASS_APER: + case MAG_CLASS_KRON: + return image[i].McalAPER; + default: + return NAN; + } + return NAN; // should not be able to reach here +} + +short getUbercalDist (off_t meas, int cat) { + + off_t i; + short distance; + + if (!MeasureToImage) return -1; + + i = MeasureToImage[cat][meas]; + if (i == -1) return (1000); + + if (image[i].flags & IMAGE_BAD) return (1000); + distance = image[i].ubercalDist; // was dummy3 in structure + return (distance); +} + +float getCenterOffset (off_t meas, int cat, Measure *measure, unsigned int *myID) { + + off_t i; + float distance; + + if (!MeasureToImage) return -1; + + i = MeasureToImage[cat][meas]; + if (i == -1) return (1000); + + float Xcenter = 0.5*image[i].NX; + float Ycenter = 0.5*image[i].NY; + + *myID = image[i].imageID; + + distance = hypot (measure[0].Xccd - Xcenter, measure[0].Yccd - Ycenter); + return (distance); +} + +int MatchImageName (off_t meas, int cat, char *name) { + + off_t i; + + if (!name) return FALSE; + if (!name[0]) return FALSE; + + if (!MeasureToImage) return FALSE; + + i = MeasureToImage[cat][meas]; + if (i == -1) return FALSE; + + // this is a bit crude: stack image names are of the form: + // RINGS.V3.skycell.1495.027.sky.191211.stk.988232.cmf. + + // the primaryCell has a name of the form RINGS.V3.skycell.1495 or RINGS.V3.skycell.1495.027 + // (if we use projection or skycell as the primary) + + if (!strncmp(image[i].name, name, strlen(name))) return TRUE; + return FALSE; +} + +int MatchImageSkycellID (off_t meas, int cat, int myTessID, int myProjectionID, int mySkycellID) { + + off_t i; + + if (!MeasureToImage) return FALSE; + + i = MeasureToImage[cat][meas]; + if (i == -1) return FALSE; + + if (tessID[i] == -1) return FALSE; + if (projectID[i] == -1) return FALSE; + if (skycellID[i] == -1) return FALSE; + + if (tessID[i] != myTessID) return FALSE; + if (projectID[i] != myProjectionID) return FALSE; + if (skycellID[i] != mySkycellID) return FALSE; + + return TRUE; +} + +int FindImageSkycellID (off_t meas, int cat, int *myTessID, int *myProjectionID, int *mySkycellID) { + + off_t i; + + if (!MeasureToImage) return FALSE; + + if (meas < 0) return FALSE; + + i = MeasureToImage[cat][meas]; + if (i == -1) return FALSE; + + if (tessID[i] == -1) return FALSE; + if (projectID[i] == -1) return FALSE; + if (skycellID[i] == -1) return FALSE; + + *myTessID = tessID[i]; + *myProjectionID = projectID[i]; + *mySkycellID = skycellID[i]; + + return TRUE; +} + +float getMflat (off_t meas, int cat, FlatCorrectionTable *flatcorr, Catalog *catalog) { + OHANA_UNUSED_PARAM(flatcorr); + + off_t i = MeasureToImage[cat][meas]; + if (i == -1) return (NAN); + + float offset = catalog[cat].measureT[meas].Mflat; + if (!isfinite(offset)) { + offset = 0.0; + } + + // to do this, I need to pass in the catalog and flatcorr pointers + // int flat_id = image[i].photom_map_id; + // if (flat_id) { + // offset = FlatCorrectionOffset (flatcorr, flat_id, catalog[cat].measureT[meas].Xccd, catalog[cat].measureT[meas].Yccd); + // } + + return (offset); +} + +Coords *getCoords (off_t meas, int cat) { + + off_t i; + + i = MeasureToImage[cat][meas]; + if (i == -1) return (NULL); + return (&image[i].coords); +} + +/* determine Mcal values for all images */ +void setMcal (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t i, j, m, c, n; + int mark, Nfew, Nbad, Nmos, Ngrp, Nrel, Ngrid, Nsys; + + if (IMAGE_ZPT_MODE == IMAGE_ZPT_MODE_NONE) return; + + fprintf (stderr, "limiting negative clouds to %f\n", CLOUD_TOLERANCE); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + off_t Nmax = 0; + for (i = 0; i < Nimage; i++) { + Nmax = MAX (Nmax, N_onImage[i]); + } + + // we are making a 0-order fit and not doing bootstrap analysis: + FitDataSet psfStars, kronStars, brightStars; + FitDataSetAlloc (&psfStars, Nmax, 0, 0); + FitDataSetAlloc (&kronStars, Nmax, 0, 0); + FitDataSetAlloc (&brightStars, Nmax, 0, 0); + + // until the analysis has converged a bit, do not use the IRLS analysis + // default is MaxIterations = 10 + if (UseStandardOLS(ZPT_IMAGES)) { + brightStars.MaxIterations = 0; + kronStars.MaxIterations = 0; + psfStars.MaxIterations = 0; + } + + Nfew = Nbad = Nmos = Ngrp = Ngrid = Nrel = Nsys = 0; + + int Ncalibrated = 0; + for (i = 0; i < Nimage; i++) { + + if (image[i].imageID == TEST_IMAGE1) { + fprintf (stderr, "test image 1\n"); + } + if (image[i].imageID == TEST_IMAGE2) { + fprintf (stderr, "test image 1\n"); + } + + if (image[i].photcode == 0) continue; // skip the PHU images + + int badNight = (image[i].flags & ID_IMAGE_NIGHT_POOR); + int badMosaic = (image[i].flags & ID_IMAGE_MOSAIC_POOR); + int isMosaic = isMosaicChip(image[i].photcode); + int useMcal = TRUE; + + // unset at start by default (set if actually used below) + image[i].flags &= ~ID_IMAGE_IMAGE_PHOTCAL; + + // if requested, freeze mosaic chips: + if (FREEZE_IMAGES && isMosaic) continue; + + // in BAD_NIGHT mode, we fit ONLY images in bad nights + if ((IMAGE_ZPT_MODE == IMAGE_ZPT_MODE_BAD_NIGHT)) { + if (!badNight) useMcal = FALSE; // do not fit images from good nights + } + + // in BAD_NIGHT_BAD_MOSAIC mode, we fit ONLY bad mosaics in bad nights + if ((IMAGE_ZPT_MODE == IMAGE_ZPT_MODE_BAD_NIGHT_BAD_MOSAIC)) { + if (!badNight) useMcal = FALSE; // do not fit images from good nights + if (!badMosaic && isMosaic) useMcal = FALSE; // skip good mosaics (but not good images) + } + + // in BAD_MOSAIC mode, we fit bad mosaics ignoring night state + if ((IMAGE_ZPT_MODE == IMAGE_ZPT_MODE_BAD_MOSAIC)) { + if (!badMosaic && isMosaic) useMcal = FALSE; // skip good mosaics (but not bad images) + } + + // UBERCAL image: if this is an ubercal image, set minUbercalDist to 0: + // we optionally do not recalibrate images with UBERCAL zero points + if (image[i].flags & ID_IMAGE_PHOTOM_UBERCAL) { + image[i].ubercalDist = 0; // was dummy3 + if (KEEP_UBERCAL) continue; + } + + int minUbercalDist = 1000; + + off_t Nref = 0; // number of stars used to measure McalPSF + int Nkron = 0; // number of stars to measure McalAPER + int Nbright = 0; // number of stars to measure the bright-end scatter + + if (N_onImage[i] == 0) { + fprintf (stderr, "image missing detections? %s %d "OFF_T_FMT"\n", image[i].name, image[i].nstar, N_onImage[i]); + continue; + } + + for (j = 0; j < N_onImage[i]; j++) { + + m = ImageToMeasure[i][j]; + c = ImageToCatalog[i][j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + Nbad++; + continue; + } + float Mmos = getMmos (m, c); + if (isnan(Mmos)) { + Nmos ++; + continue; + } + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); // ignore error for now? + if (isnan(Mgrp)) { + Ngrp ++; + continue; + } + float Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) { + Ngrid++; + continue; + } + + // Mrel* is the average magnitude for this star. For PS1 stacks, we have too much + // PSF variability. We need to calibrate the PSF magnitudes separately from the + // Aperture-like magnitues. (We have an option to use the kron magnitudes or the + // other apertures here). I basically need to do this analysis separately for each + // magnitude type + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { + Nrel ++; + continue; + } + + // image.Mcal is not supposed to include the flat-field correction, so we need to + // apply that offset as well here for this image (in other words, each detection is + // being compared to the model, excluding the zero point, Mcal. The model includes + // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) + + float Mflat = getMflat (m, c, flatcorr, catalog); + + n = catalog[c].measureT[m].averef; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { + Nsys++; + continue; + } + + float Moff = Mmos + Mgrp + Mgrid - Mflat; + + PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode); + if (!code) goto skip; + if (code->equiv < 1) goto skip; + int Nsec = GetPhotcodeNsec (code->equiv); + if (Nsec == -1) goto skip; + minUbercalDist = MIN (catalog[c].secfilt[n*Nsecfilt + Nsec].ubercalDist, minUbercalDist); + + skip: + psfStars.alldata-> yVector[Nref] = MsysPSF - MrelPSF - Moff; + psfStars.alldata->dyVector[Nref] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + + float MrelKron = getMrel (catalog, m, c, MAG_CLASS_KRON, MAG_SRC_CHP); + float MsysKron = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_KRON); + + if (isfinite(MrelKron) && isfinite(MsysKron)) { + kronStars.alldata-> yVector[Nkron] = MsysKron - MrelKron - Moff; + kronStars.alldata->dyVector[Nkron] = psfStars.alldata->dyVector[Nref]; + Nkron ++; + } + + if ((image[i].imageID == TEST_IMAGE1) || (image[i].imageID == TEST_IMAGE2)) { + fprintf (stderr, "%1d, %3d : %3d, %3d : %10.6f %10.6f : %6.3f %6.3f %6.3f %6.3f %6.3f : %6.3f\n", (int) i, (int) j, (int) c, (int) m, catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysKron, MrelKron, Mmos, Mgrid, Mflat, kronStars.alldata->yVector[Nkron]); + } + + if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) { + brightStars.alldata-> yVector[Nbright] = psfStars.alldata-> yVector[Nref]; + brightStars.alldata->dyVector[Nbright] = psfStars.alldata->dyVector[Nref]; + Nbright ++; + } + Nref++; + } + /* N_onImage[i] is all measurements, N is good measurements */ + + // if (VERBOSE2) fprintf (stderr, "meas skipped: (Nbad: %d, Nmos: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", Nbad, Nmos, Ngrid, Nrel, Nsys); + + /* too few good measurements or too many bad measurements */ + mark = (Nref < IMAGE_TOOFEW) || (Nref < IMAGE_GOOD_FRACTION*N_onImage[i]); + if (mark) { + image[i].flags |= ID_IMAGE_PHOTOM_FEW; + image[i].dMcal = NAN; + image[i].McalChiSq = NAN; + image[i].nFitPhotom = 0; + Nfew ++; + } else { + image[i].flags &= ~ID_IMAGE_PHOTOM_FEW; + } + if (mark) continue; + + // use liststats to find the 20-pct, median, 80-pct points + StatType stats; + liststats_setmode (&stats, "MEDIAN"); + liststats (psfStars.alldata->yVector, NULL, NULL, Nref, &stats); + double altSigma = (stats.Upper80 - stats.Lower20) / 1.6; // 20% to 80% encompasses 60% of the values, corresponds to the range (-0.85 sigma : +0.85 sigma) + + // soften the individual errors with 10% of the scatter above + for (j = 0; j < Nref; j++) { + double newSigma = hypot(psfStars.alldata->dyVector[j], 0.1*altSigma); + psfStars.alldata->dyVector[j] = newSigma; + } + + // soften the errors based on the scatter + FitDataSetSoften (&psfStars, Nref); + + fit1d_irls (&psfStars, Nref); + fit1d_irls (&kronStars, Nkron); + fit1d_irls (&brightStars, Nbright); + + // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not) + if (useMcal) { + image[i].McalPSF = psfStars.bSaveArray[0][0]; + image[i].McalAPER = kronStars.bSaveArray[0][0]; + image[i].flags |= ID_IMAGE_IMAGE_PHOTCAL; // set this flag (unset by default) + } else { + image[i].McalPSF = 0.0; + image[i].McalAPER = 0.0; + } + + // record statistics on the fit regardless if it is used or not + image[i].dMcal = psfStars.bSigma[0]; + image[i].nFitPhotom = psfStars.Nmeas; + image[i].McalChiSq = psfStars.chisq; + Ncalibrated ++; + + // bright end scatter + image[i].dMagSys = brightStars.sigma; + + if ((image[i].imageID == TEST_IMAGE1) || (image[i].imageID == TEST_IMAGE2)) { + fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].McalAPER, image[i].dMcal); + } + + if (PLOTSTUFF) { + fprintf (stderr, "Mcal for : %s : %7.4f %7.4f\n", image[i].name, image[i].McalAPER, image[i].dMcal); + plot_setMcal (psfStars.alldata-> yVector, Nref); + } + + // minUbercalDist calculated here is the min value for any star owned by this image + // since this particular image is tied to that star, bump its distance by 1 + image[i].ubercalDist = minUbercalDist + 1; + } + + fprintf (stderr, "%d images calibrated\n", Ncalibrated); + fprintf (stderr, "%d images marked having too few measurements (Nbad: %d, Nmos: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", Nfew, Nbad, Nmos, Ngrid, Nrel, Nsys); + + FitDataSetFree (&brightStars); + FitDataSetFree (&kronStars); + FitDataSetFree (&psfStars); + + return; +} + +/* determine McalTEST values for all images -- this is not used to set measure.Mcal, but only to test */ +void setMcalTest (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t i, j, m, c, n; + int Nfew, Nbad, Nmos, Ngrp, Nrel, Ngrid, Nsys; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + off_t Nmax = 0; + for (i = 0; i < Nimage; i++) { + Nmax = MAX (Nmax, N_onImage[i]); + } + + // we are making a 0-order fit and not doing bootstrap analysis: + FitDataSet psfStars; + FitDataSetAlloc (&psfStars, Nmax, 0, 0); + + // for testing, need to allow this to be optional + if (FALSE && UseStandardOLS(ZPT_IMAGES)) { + psfStars.MaxIterations = 0; + } + psfStars.MaxIterations = 0; + + Nfew = Nbad = Nmos = Ngrp = Ngrid = Nrel = Nsys = 0; + + int Ncalibrated = 0; + for (i = 0; i < Nimage; i++) { + if (image[i].photcode == 0) continue; // skip the PHU images + + off_t Nref = 0; // number of stars used to measure McalPSF + + if (N_onImage[i] == 0) { + // fprintf (stderr, "image missing detections? %s %d "OFF_T_FMT"\n", image[i].name, image[i].nstar, N_onImage[i]); + continue; + } + + for (j = 0; j < N_onImage[i]; j++) { + + m = ImageToMeasure[i][j]; + c = ImageToCatalog[i][j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + Nbad++; + continue; + } + float Mmos = getMmos (m, c); + if (isnan(Mmos)) { + Nmos ++; + continue; + } + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); // ignore error for now? + if (isnan(Mgrp)) { + Ngrp ++; + continue; + } + float Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) { + Ngrid++; + continue; + } + + // MrelPSF is the average magnitude for this star. + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { + Nrel ++; + continue; + } + + // image.Mcal is not supposed to include the flat-field correction, so we need to + // apply that offset as well here for this image (in other words, each detection is + // being compared to the model, excluding the zero point, Mcal. The model includes + // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) + + float Mflat = getMflat (m, c, flatcorr, catalog); + + // get the PSF magnitude for thie measurement, with airmass slope applied + n = catalog[c].measureT[m].averef; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { + Nsys++; + continue; + } + + float Moff = Mmos + Mgrp + Mgrid - Mflat; + + psfStars.alldata-> yVector[Nref] = MsysPSF - MrelPSF - Moff; + psfStars.alldata->dyVector[Nref] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + Nref++; + } + + if (Nref < 5) { + // fprintf (stderr, "very few detections? %s %d "OFF_T_FMT"\n", image[i].name, image[i].nstar, Nref); + // continue; + image[i].McalAPER = NAN; + image[i].dMcal = NAN; + image[i].nFitPhotom = Nref; + image[i].McalChiSq = NAN; + continue; + } + + // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not) + fit1d_irls (&psfStars, Nref); + image[i].McalAPER = psfStars.bSaveArray[0][0]; + image[i].dMcal = psfStars.bSigma[0]; + image[i].nFitPhotom = psfStars.Nmeas; + image[i].McalChiSq = psfStars.chisq; + Ncalibrated ++; + } + + fprintf (stderr, "%d images calibrated\n", Ncalibrated); + fprintf (stderr, "%d images marked having too few measurements (Nbad: %d, Nmos: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", Nfew, Nbad, Nmos, Ngrid, Nrel, Nsys); + + FitDataSetFree (&psfStars); + return; +} + +/* mark image if: abs(Mcal) too large, dMcal too large */ +void clean_images () { + + int mark, Nmark; + off_t i, N; + double *mlist, *slist, *dlist; + double MaxOffset, MaxScatter, MedOffset; + + // if (FREEZE_IMAGES) return; + + if (VERBOSE) fprintf (stderr, "marking poor images\n"); + + ALLOCATE (mlist, double, Nimage); + ALLOCATE (slist, double, Nimage); + ALLOCATE (dlist, double, Nimage); + + // measure stats for Mcal and dMcal + for (i = N = 0; i < Nimage; i++) { + if (image[i].flags & IMAGE_BAD) continue; + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + mlist[N] = image[i].McalPSF; + slist[N] = image[i].dMcal; + dlist[N] = 1; + N++; + } + + // use a straight mean to find the global image statistics (no weighting) + StatType stats; + liststats_setmode (&stats, "MEAN"); + + liststats (mlist, dlist, NULL, N, &stats); + MaxOffset = MAX (IMAGE_OFFSET, 3*stats.sigma); + MedOffset = stats.median; + + liststats (slist, dlist, NULL, N, &stats); + MaxScatter = MAX (IMAGE_SCATTER, 2*stats.median); + fprintf (stderr, "Mrel: %f, dMrel: %f, Max Scatter: %f, Max Offset: %f\n", MedOffset, stats.median, MaxScatter, MaxOffset); + + Nmark = 0; + for (i = 0; i < Nimage; i++) { + // if we are keeping ubercal sacrosanct, then we should not be allowed to break them... + if (KEEP_UBERCAL && (image[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + mark = FALSE; + image[i].flags &= ~ID_IMAGE_PHOTOM_POOR; + mark = (image[i].dMcal > MaxScatter) || (fabs(image[i].McalPSF - MedOffset) > MaxOffset); + if (mark) { + Nmark ++; + image[i].flags |= ID_IMAGE_PHOTOM_POOR; + } else { + image[i].flags &= ~ID_IMAGE_PHOTOM_POOR; + } + } + + fprintf (stderr, "%d images marked poor\n", Nmark); + free (mlist); + free (slist); + free (dlist); +} + +static int setMcal_init_done = FALSE; +void plot_setMcal (double *list, int Npts) { + + off_t i; + double *xlist; + Graphdata graphdata; + + if (!PLOTSTUFF) return; + + if (Npts == 0) { + // fprintf (stderr, "cannot set image Mcal values yet (nan Mave)\n"); + return; + } + + plot_defaults (&graphdata); + graphdata.xmin = -0.01; + graphdata.xmax = +1.01; + graphdata.ymin = -0.21; + graphdata.ymax = +0.21; + + if (!setMcal_init_done) { + int kapa = get_graph(0); + if (kapa < 1) { + kapa = open_graph(0); + if (!kapa) return; + } + + KapaClearSections (kapa); + KapaClearPlots (kapa); + KapaSetLimits (kapa, &graphdata); + KapaSetFont (kapa, "helvetica", 14); + KapaBox (kapa, &graphdata); + setMcal_init_done = TRUE; + } + + ALLOCATE (xlist, double, Npts); + for (i = 0; i < Npts; i++) { + xlist[i] = i / (float) Npts; + } + + plot_list_add (&graphdata, xlist, list, Npts); + + free (xlist); +} + +void plot_images () { + + off_t i, bin; + double *xlist, *Mlist, *dlist; + Graphdata graphdata; + + // if (FREEZE_IMAGES) return; + + ALLOCATE (xlist, double, Nimage); + ALLOCATE (dlist, double, Nimage); + ALLOCATE (Mlist, double, Nimage); + + /**** dMcal vs airmass ****/ + float minAirmass = 1000.0; + float maxAirmass = 0.0; + float minMcal = +100.0; + float maxMcal = -100.0; + float mindMcal = +100.0; + float maxdMcal = -100.0; + + int Nplot = 0; + + for (i = 0; i < Nimage; i++) { + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + Mlist[Nplot] = image[i].McalPSF; + dlist[Nplot] = image[i].dMcal; + xlist[Nplot] = image[i].secz; + minAirmass = MIN (image[i].secz, minAirmass); + maxAirmass = MAX (image[i].secz, maxAirmass); + minMcal = MIN (image[i].McalPSF, minMcal); + maxMcal = MAX (image[i].McalPSF, maxMcal); + mindMcal = MIN (image[i].dMcal, mindMcal); + maxdMcal = MAX (image[i].dMcal, maxdMcal); + + Nplot ++; + } + + float AirmassRange = MAX(1.2*(maxAirmass - minAirmass), 0.25); + float AirmassCenter = 0.5*(maxAirmass + minAirmass); + + float McalRange = MAX(1.2*(maxMcal - minMcal), 0.21); + float McalCenter = 0.5*(maxMcal + minMcal); + + float dMcalRange = MAX(1.2*(maxdMcal - mindMcal), 0.21); + float dMcalCenter = 0.5*(maxdMcal + mindMcal); + + plot_defaults (&graphdata); + graphdata.xmin = AirmassCenter - 0.5*AirmassRange; + graphdata.xmax = AirmassCenter + 0.5*AirmassRange; + graphdata.ymin = McalCenter - 0.5*McalRange; + graphdata.ymax = McalCenter + 0.5*McalRange; + plot_list (&graphdata, xlist, Mlist, Nimage, "airmass vs Mcal", "%s.airmass.png", OUTROOT); + + plot_defaults (&graphdata); + graphdata.xmin = McalCenter - 0.5*McalRange; + graphdata.xmax = McalCenter + 0.5*McalRange; + graphdata.ymin = dMcalCenter - 0.5*dMcalRange; + graphdata.ymax = dMcalCenter + 0.5*dMcalRange; + plot_list (&graphdata, Mlist, dlist, Nimage, "Mcal vs dMcal", "%s.Mcal.dMcal.png", OUTROOT); + +# define NBIN 200 + REALLOCATE (xlist, double, NBIN); + REALLOCATE (Mlist, double, NBIN); + + /**** dMcal histgram ****/ + for (i = 0; i < NBIN; i++) xlist[i] = 0.00025*i; + bzero (Mlist, NBIN*sizeof(double)); + + for (i = 0; i < Nimage; i++) { + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + bin = image[i].dMcal / 0.00025; + bin = MAX (0, MIN (NBIN - 1, bin)); + Mlist[bin] += 1.0; + } + + plot_defaults (&graphdata); + graphdata.style = 1; + graphdata.xmin = dMcalCenter - 0.5*dMcalRange; + graphdata.xmax = dMcalCenter + 0.5*dMcalRange; + plot_list (&graphdata, xlist, Mlist, NBIN, "dMcal hist", "%s.dMcalhist.png", OUTROOT); + + free (dlist); + free (xlist); + free (Mlist); +} + +StatType statsImageN (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t i, j, m, c, n, N; + double *list, *dlist; + float Mcal, Mmos, Mgrp, Mgrid; + + StatType stats; + bzero (&stats, sizeof (StatType)); + + // we no longer blindly apply FREEZE_IMAGES to all images, only to mosaics + // if (FREEZE_IMAGES) return (stats); + + ALLOCATE (list, double, Nimage); + ALLOCATE (dlist, double, Nimage); + + n = 0; + for (i = 0; i < Nimage; i++) { + if (image[i].flags & IMAGE_BAD) continue; + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + N = 0; + for (j = 0; j < N_onImage[i]; j++) { + + m = ImageToMeasure[i][j]; + c = ImageToCatalog[i][j]; + + Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + Mmos = getMmos (m, c); + if (isnan(Mmos)) continue; + Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) continue; + Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) continue; + N++; + } + list[n] = N; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsImageX (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i, n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + // if (FREEZE_IMAGES) return (stats); + + ALLOCATE (list, double, Nimage); + ALLOCATE (dlist, double, Nimage); + + n = 0; + for (i = 0; i < Nimage; i++) { + + if (image[i].flags & IMAGE_BAD) continue; + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + list[n] = image[i].McalChiSq; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsImageM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i, n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + // if (FREEZE_IMAGES) return (stats); + + ALLOCATE (list, double, Nimage); + ALLOCATE (dlist, double, Nimage); + + n = 0; + for (i = 0; i < Nimage; i++) { + + if (image[i].flags & IMAGE_BAD) continue; + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + list[n] = image[i].McalPSF; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsImagedM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i, n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + // if (FREEZE_IMAGES) return (stats); + + ALLOCATE (list, double, Nimage); + ALLOCATE (dlist, double, Nimage); + + n = 0; + for (i = 0; i < Nimage; i++) { + + if (image[i].flags & IMAGE_BAD) continue; + + if (FREEZE_IMAGES && isMosaicChip(image[i].photcode)) continue; + + list[n] = image[i].dMcal; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +void clearImages (void) { + image = NULL; +} Index: /branches/eam_branches/relphot.20210521/src/ImageSubset.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ImageSubset.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ImageSubset.c (revision 41602) @@ -0,0 +1,256 @@ +# include "relphot.h" + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// this is nearly identical to the one in 'uniphot/src' used by setphot / setphot_client. +// Here, we use a handful of different columns (if not, we could move to libdvo) +ImageSubset *ImageSubsetLoad(char *filename, off_t *nimage) { + + int i, Ncol; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + *nimage = 0; + ImageSubset *image = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + gfits_free_header (&theader); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + return (NULL); + } + fclose (f); + + // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values. + // this means I need 3 copies in memory at some point. ugh. + + char type[16]; + + GET_COLUMN (McalPSF, "MCAL_PSF", float); + GET_COLUMN (McalAPER, "MCAL_APER", float); + GET_COLUMN (dMcal, "MCAL_ERR", float); + GET_COLUMN (imageID, "IMAGE_ID", int); + GET_COLUMN (map, "PHOTOM_MAP", int); + GET_COLUMN (flags, "FLAGS", int); + GET_COLUMN (tessID, "TESS_ID", int); + GET_COLUMN (projID, "PROJ_ID", int); + GET_COLUMN (skycellID, "SKYCELL_ID", int); + GET_COLUMN (tzero, "TZERO", int); + GET_COLUMN (trate, "TRATE", short); + GET_COLUMN (ucdist, "UBERCAL_DIST", short); + + // XXX free the fits table data here + + ALLOCATE (image, ImageSubset, Nrow); + for (i = 0; i < Nrow; i++) { + image[i].McalPSF = McalPSF[i]; + image[i].McalAPER = McalAPER[i]; + image[i].dMcal = dMcal[i]; + image[i].imageID = imageID[i]; + image[i].photom_map_id = map[i]; + image[i].flags = flags[i]; + image[i].tessID = tessID[i]; + image[i].projID = projID[i]; + image[i].skycellID = skycellID[i]; + image[i].tzero = tzero[i]; + image[i].trate = trate[i]; + image[i].ubercalDist = ucdist[i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (McalPSF); + free (McalAPER); + free (dMcal); + free (imageID); + free (map); + free (flags); + free (tessID); + free (projID); + free (skycellID); + free (tzero); + free (trate); + free (ucdist); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + *nimage = Nrow; + return image; +} + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage) { + + int i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_SUBSET"); + + gfits_define_bintable_column (&theader, "E", "MCAL_PSF", "zero point offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_APER", "zero point offset", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MCAL_ERR", "zero point error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image ID", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "PHOTOM_MAP", "map", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "FLAGS", "flags", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "TESS_ID", "ID", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "PROJ_ID", "ID", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "SKYCELL_ID", "ID", NULL, 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "TZERO", "exposure start", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "I", "TRATE", "tti rate", NULL, 1.0, FT_BZERO_INT16); + gfits_define_bintable_column (&theader, "I", "UBERCAL_DIST", "ubercal distance", NULL, 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + float *McalPSF, *McalAPER, *dMcal; + unsigned int *imageID, *map, *flags, *tzero; + int *tessID, *projID, *skycellID; + unsigned short *trate; + short *ucdist; + + // create intermediate storage arrays + ALLOCATE (McalPSF, float, Nimage); + ALLOCATE (McalAPER, float, Nimage); + ALLOCATE (dMcal, float, Nimage); + ALLOCATE (imageID, unsigned int, Nimage); + ALLOCATE (map, unsigned int, Nimage); + ALLOCATE (flags, unsigned int, Nimage); + ALLOCATE (tessID, int, Nimage); + ALLOCATE (projID, int, Nimage); + ALLOCATE (skycellID, int, Nimage); + ALLOCATE (tzero, unsigned int, Nimage); + ALLOCATE (trate, unsigned short, Nimage); + ALLOCATE (ucdist, short, Nimage); + + // assign the storage arrays + for (i = 0; i < Nimage; i++) { + McalPSF[i] = image[i].McalPSF; + McalAPER[i] = image[i].McalAPER; + dMcal[i] = image[i].dMcal; + imageID[i] = image[i].imageID; + map[i] = image[i].photom_map_id; + flags[i] = image[i].flags; + tzero[i] = image[i].tzero; + trate[i] = image[i].trate; + ucdist[i] = image[i].ubercalDist; + tessID[i] = image[i].tessID; + projID[i] = image[i].projID; + skycellID[i] = image[i].skycellID; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "MCAL_PSF", McalPSF, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_APER", McalAPER, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MCAL_ERR", dMcal, Nimage); + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PHOTOM_MAP", map, Nimage); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags, Nimage); + gfits_set_bintable_column (&theader, &ftable, "TESS_ID", tessID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PROJ_ID", projID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "SKYCELL_ID", skycellID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "TZERO", tzero, Nimage); + gfits_set_bintable_column (&theader, &ftable, "TRATE", trate, Nimage); + gfits_set_bintable_column (&theader, &ftable, "UBERCAL_DIST", ucdist, Nimage); + + free (McalPSF); + free (McalAPER); + free (dMcal); + free (imageID); + free (map); + free (flags); + free (tessID); + free (projID); + free (skycellID); + free (tzero); + free (trate); + free (ucdist); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for image subset %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for image subset %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for image subset %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for image subset %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing image subset file %s\n", filename); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/ImageTable.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ImageTable.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ImageTable.c (revision 41602) @@ -0,0 +1,74 @@ +# include "relphot.h" + +Image *ImageTableLoad(char *filename, off_t *nimage) { + + int status; + off_t Nimage; + FITS_DB db; + + gfits_db_init (&db); + db.mode = dvo_catalog_catmode (CATMODE); + db.format = dvo_catalog_catformat (CATFORMAT); + + /* lock and load the image db table */ + status = dvo_image_lock (&db, filename, 60.0, LCK_SOFT); + if (!status) { + Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + } + + // if the file is missing, db.dbstate will have a value of either: + // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE) + if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) { + Shutdown ("ERROR: database %s contains no image data", CATDIR); + } + + // read data from Image.dat file + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + + // convert database table to internal structure (binary to Image) + // 'image' points to the same memory as db->ftable->buffer + Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder); + if (!image) { + Shutdown ("ERROR: failed to read images"); + } + + // i want to keep the image data, but free everything else + db.ftable.buffer = NULL; + gfits_db_free (&db); + + // XXX do not make the chip match -- we have not loaded the PHU entries (and do not need them here) + // BuildChipMatch (image, Nimage); + + *nimage = Nimage; + return image; +} + +int ImageTableSave (char *filename, Image *images, off_t Nimages) { + + int status; + FITS_DB db; + + // setup image table format and lock + gfits_db_init (&db); + db.mode = dvo_catalog_catmode (CATMODE); + db.format = dvo_catalog_catformat (CATFORMAT); + + status = dvo_image_lock (&db, filename, 60.0, LCK_XCLD); // shorter timeout? + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + + // load or create the image table + if (db.dbstate != LCK_EMPTY) { + Shutdown ("image catalog already exists %s", db.filename); + } + + dvo_image_create (&db, GetZeroPoint()); + + /* add the new images and save */ + dvo_image_addrows (&db, images, Nimages); + dvo_image_update (&db, VERBOSE); + dvo_image_unlock (&db); + + gfits_db_free (&db); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/MagResidSave.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/MagResidSave.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/MagResidSave.c (revision 41602) @@ -0,0 +1,290 @@ +# include "relphot.h" + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +typedef struct { + float Mresid; + float dMresid; + float Mchisq; + float Mmedian; + float Maltsig; + int Nfit; +} MagResidResult; + +MagResidResult MagResidCalc (Catalog *catalog, Image *image, FitDataSet *psfStars, int Nsecfilt, int N_onImage, IDX_T *ImageToMeasure, IDX_T *ImageToCatalog); + +off_t *get_N_onImage(); +IDX_T **get_ImageToCatalog(); +IDX_T **get_ImageToMeasure(); + +int MagResidSave(char *filename, Catalog *catalog) { + + int i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + off_t Nimage; + Image *image = getimages (&Nimage, NULL); + off_t *N_onImage = get_N_onImage(); + + // find the maximum number of stars (needed to allocate the psfStars structure) + off_t Nmax = 0; + for (i = 0; i < Nimage; i++) { + Nmax = MAX (Nmax, N_onImage[i]); + } + + // we are making a 0-order fit and not doing bootstrap analysis + FitDataSet psfStars; + FitDataSetAlloc (&psfStars, Nmax, 0, 0); + // By default, we use IRLS; if MaxIterations = 0, then we use OLS + // psfStars.MaxIterations = 0; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "MAG_PSF_RESID"); + + gfits_define_bintable_column (&theader, "D", "RA", "ra", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "D", "DEC", "dec", "degree", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_RESID", "magnitude residual", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ERROR", "magnitude error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_CHISQ", "magnitude resid chisq", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_MEDIAN", "median magnitude residual", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ALTSIG", "68 percentile range as sigma", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "AIRMASS", "", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "D", "MJD", "", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "EXPTIME", "", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "PHOTCODE", "", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "FLAGS", "image flags", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "NFIT", "number of stars fitted", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "NMEAS", "number of stars on image", NULL, 1.0, FT_BZERO_INT32); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // create intermediate storage arrays + ALLOCATE_PTR (Rs, double, Nimage); + ALLOCATE_PTR (Ds, double, Nimage); + + ALLOCATE_PTR (Mresid, float, Nimage); + ALLOCATE_PTR (dMresid, float, Nimage); + ALLOCATE_PTR (Mchisq, float, Nimage); + ALLOCATE_PTR (Mmedian, float, Nimage); + ALLOCATE_PTR (Maltsig, float, Nimage); + ALLOCATE_PTR (airmass, float, Nimage); + ALLOCATE_PTR (mjd, double, Nimage); + ALLOCATE_PTR (exptime, float, Nimage); + ALLOCATE_PTR (photcode, int, Nimage); + ALLOCATE_PTR (flags, int, Nimage); + ALLOCATE_PTR (Nfit, int, Nimage); + ALLOCATE_PTR (Nmeas, int, Nimage); + + // XXXX fill in the vectors here + int Nsecfilt = GetPhotcodeNsecfilt (); + IDX_T **ImageToCatalog = get_ImageToCatalog(); + IDX_T **ImageToMeasure = get_ImageToMeasure(); + + for (i = 0; i < Nimage; i++) { + double Ro, Do; + XY_to_RD (&Ro, &Do, (double) 0.5*image[i].NX, (double) 0.5*image[i].NY, &image[i].coords); + + Rs[i] = Ro; + Ds[i] = Do; + Mresid[i] = NAN; + Mchisq[i] = NAN; + dMresid[i] = NAN; + Mmedian[i] = NAN; + Maltsig[i] = NAN; + Nfit[i] = 0; + Nmeas[i] = N_onImage[i]; + airmass[i] = image[i].secz; + mjd[i] = ohana_sec_to_mjd (image[i].tzero); + exptime[i] = image[i].exptime; + photcode[i] = image[i].photcode; + flags[i] = image[i].flags; + + if (image[i].photcode == 0) continue; // skip the PHU images + if (N_onImage[i] == 0) continue; + + MagResidResult result = MagResidCalc (catalog, &image[i], &psfStars, Nsecfilt, N_onImage[i], ImageToMeasure[i], ImageToCatalog[i]); + + Mresid[i] = result.Mresid; + Mchisq[i] = result.Mchisq; + dMresid[i] = result.dMresid; + Mmedian[i] = result.Mmedian; + Maltsig[i] = result.Maltsig; + Nfit[i] = result.Nfit; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "RA", Rs, Nimage); + gfits_set_bintable_column (&theader, &ftable, "DEC", Ds, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MAG_RESID", Mresid, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MAG_ERROR", dMresid, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MAG_CHISQ", Mchisq, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MAG_MEDIAN", Mmedian, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MAG_ALTSIG", Maltsig, Nimage); + gfits_set_bintable_column (&theader, &ftable, "AIRMASS", airmass, Nimage); + gfits_set_bintable_column (&theader, &ftable, "MJD", mjd, Nimage); + gfits_set_bintable_column (&theader, &ftable, "EXPTIME", exptime, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PHOTCODE", photcode, Nimage); + gfits_set_bintable_column (&theader, &ftable, "FLAGS", flags, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NFIT", Nfit, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NMEAS", Nmeas, Nimage); + + free (Rs); + free (Ds); + free (Mresid ); + free (dMresid ); + free (Mchisq ); + free (Mmedian ); + free (Maltsig ); + free (airmass ); + free (mjd ); + free (exptime ); + free (photcode); + free (flags ); + free (Nfit ); + free (Nmeas ); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open meanmag file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for meanmags %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for meanmags %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for meanmags %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for meanmags %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing meanmags file %s\n", filename); + + return TRUE; +} + +// determine Mag residual for this image +MagResidResult MagResidCalc (Catalog *catalog, Image *image, FitDataSet *psfStars, int Nsecfilt, int N_onImage, IDX_T *ImageToMeasure, IDX_T *ImageToCatalog) { + + off_t Nref = 0; // number of stars used to measure McalPSF + + MagResidResult result; + result.Mresid = NAN; + result.dMresid = NAN; + result.Mchisq = NAN; + result.Mmedian = NAN; + result.Maltsig = NAN; + result.Nfit = 0; + + for (off_t j = 0; j < N_onImage; j++) { + + off_t m = ImageToMeasure[j]; + off_t c = ImageToCatalog[j]; + + // XXX drop this? + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) continue; + + float Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + + float Mmos = getMmos (m, c); + if (isnan(Mmos)) continue; + + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); // ignore error for now? + if (isnan(Mgrp)) continue; + + float Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) continue; + + // image.Mcal is not supposed to include the flat-field correction, so we need to + // apply that offset as well here for this image (in other words, each detection is + // being compared to the model, excluding the zero point, Mcal. The model includes + // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) + + // float Mflat = getMflat (m, c, flatcorr, catalog); + + // MrelPSF is the average magnitude for this star. + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) continue; + + // get the PSF magnitude for thie measurement, with airmass slope applied + off_t n = catalog[c].measureT[m].averef; + + // MsysPSF is the specific measurement for this star on this image + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) continue; + + // XXX skip Mflat for now + // float Moff = Mcal + Mmos + Mgrp + Mgrid - Mflat; + + float Moff = Mcal + Mmos + Mgrp + Mgrid; + + // Mrel is the true average apparent magnitude of this star + // Msys is the observed apparent magnitude, with nominal corrections, ie, the instrumental magnitude plus a constant + // yVector (dMag) = Msys - Mrel - Moff + // as the clouds come and go, Mrel is constant. As the clouds increase, the observed + // star is fainter, so Msys gets larger, and the value of Moff also gets larger to compensate + // in other words: + // Mapp = Minst + ZP + // is equivalent to: + // Mrel = Msys - Moff, so a larger Moff means a smaller ZP and increase clouds (decreased transmission) + psfStars->alldata-> yVector[Nref] = MsysPSF - MrelPSF - Moff; + psfStars->alldata->dyVector[Nref] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + Nref++; + } + + if (Nref < 5) { + result.Mresid = NAN; + result.dMresid = NAN; + result.Mchisq = NAN; + result.Mmedian = NAN; + result.Maltsig = NAN; + result.Nfit = Nref; + return result; + } + + StatType stats = FitDataSetSoften (psfStars, Nref); + double altSigma = (stats.Upper80 - stats.Lower20) / 1.6; + + // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not) + fit1d_irls (psfStars, Nref); + result.Mresid = psfStars->bSaveArray[0][0]; + result.dMresid = psfStars->bSigma[0]; + result.Nfit = psfStars->Nmeas; + result.Mchisq = psfStars->chisq; + result.Mmedian = stats.median; + result.Maltsig = altSigma; + + return result; +} Index: /branches/eam_branches/relphot.20210521/src/MeanMagIO.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/MeanMagIO.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/MeanMagIO.c (revision 41602) @@ -0,0 +1,200 @@ +# include "relphot.h" + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// this is nearly identical to the one in 'uniphot/src' used by setphot / setphot_client. +// Here, we use a handful of different columns (if not, we could move to libdvo) +MeanMag *MeanMagLoad(char *filename, off_t *nmeanmags) { + + int i, Ncol; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + *nmeanmags = 0; + MeanMag *meanmags = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return (NULL); + } + fclose (f); + + // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values. + // this means I need 3 copies in memory at some point. ugh. + + char type[16]; + + GET_COLUMN (M, "MAG", float); + GET_COLUMN (dM, "MAG_ERR", float); + GET_COLUMN (Mchisq, "MAG_CHISQ", float); + GET_COLUMN (Nsec, "NSEC", int); + GET_COLUMN (objID, "OBJ_ID", int); + GET_COLUMN (catID, "CAT_ID", int); + + // free the memory associated with the FITS files + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + ALLOCATE (meanmags, MeanMag, Nrow); + for (i = 0; i < Nrow; i++) { + meanmags[i].M = M [i]; + meanmags[i].dM = dM [i]; + meanmags[i].Mchisq = Mchisq [i]; + meanmags[i].Nsec = Nsec [i]; + meanmags[i].objID = objID [i]; + meanmags[i].catID = catID [i]; + } + fprintf (stderr, "loaded data for %lld objects (* filters)\n", (long long) Nrow); + + free (M ); + free (dM ); + free (Mchisq); + free (Nsec ); + free (objID ); + free (catID ); + + *nmeanmags = Nrow; + return meanmags; +} + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +int MeanMagSave(char *filename, MeanMag *meanmags, off_t Nmeanmags) { + + int i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "MEANMAGS"); + + gfits_define_bintable_column (&theader, "E", "MAG", "mean magnitude", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_ERR", "mean magnitude error", "magnitudes", 1.0, 0.0); + gfits_define_bintable_column (&theader, "E", "MAG_CHISQ", "mean magnitude chisq", "unitless", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "NSEC", "secfilt sequence", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "OBJ_ID", "object ID", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "CAT_ID", "catalog ID", NULL, 1.0, FT_BZERO_INT32); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + float *M, *dM, *Mchisq; + int *Nsec; + unsigned int *objID, *catID; + + // create intermediate storage arrays + ALLOCATE (M, float, Nmeanmags); + ALLOCATE (dM, float, Nmeanmags); + ALLOCATE (Mchisq, float, Nmeanmags); + ALLOCATE (Nsec, int, Nmeanmags); + ALLOCATE (objID, unsigned int, Nmeanmags); + ALLOCATE (catID, unsigned int, Nmeanmags); + + // assign the storage arrays + for (i = 0; i < Nmeanmags; i++) { + M [i] = meanmags[i].M ; + dM [i] = meanmags[i].dM ; + Mchisq[i] = meanmags[i].Mchisq; + Nsec [i] = meanmags[i].Nsec ; + objID [i] = meanmags[i].objID ; + catID [i] = meanmags[i].catID ; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "MAG", M, Nmeanmags); + gfits_set_bintable_column (&theader, &ftable, "MAG_ERR", dM, Nmeanmags); + gfits_set_bintable_column (&theader, &ftable, "MAG_CHISQ", Mchisq, Nmeanmags); + gfits_set_bintable_column (&theader, &ftable, "NSEC", Nsec, Nmeanmags); + gfits_set_bintable_column (&theader, &ftable, "OBJ_ID", objID, Nmeanmags); + gfits_set_bintable_column (&theader, &ftable, "CAT_ID", catID, Nmeanmags); + + free (M ); + free (dM ); + free (Mchisq); + free (Nsec ); + free (objID ); + free (catID ); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open meanmag file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for meanmags %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for meanmags %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for meanmags %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for meanmags %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file meanmags %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing meanmags file %s\n", filename); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/MosaicOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/MosaicOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/MosaicOps.c (revision 41602) @@ -0,0 +1,2065 @@ +# include "relphot.h" +off_t findMosaic (unsigned int *startTimes, off_t Nmosaic, unsigned int start); +int save_test_mosaic_measures (FILE *fout, int Nmos, Catalog *catalog, FlatCorrectionTable *flatcorr); + +// see discussion in ImagesOps.c re: IDX_T + +// array of mosaic definition structures +static off_t Nmosaic; +static Mosaic *mosaic; + +// relationships between the mosaics and their associated images +static off_t *MosaicN_Image; // number of images associated with the given mosaic +static off_t **MosaicToImage; // list of images associated with the given mosaic + +// mosaic index for given image : ImageToMosaic[ImageIndex] = MosaicIndex (ImageIndex : 0 < Nimage) +static off_t *ImageToMosaic; + +// elsewhere, we have loaded a set of catalogs with measures (catalog[cat].measure[meas]) +// each mosaic has N_onMosaic[MosaicIndex] measurements +static off_t *N_onMosaic; // actual number of measurements on mosaic +static off_t *N_ONMOSAIC; // allocated number of measurements on mosaic + +// relationships between the measure,catalog set and the mosaics: +static off_t **MeasureToMosaic; // Mosaic index from measure,catalog : MeasureToMosaic[cat][meas] = MosaicIndex +static off_t **MosaicToCatalog; // catalog for given measure on mosaic : MosaicCatalog[MosaicIndex][i] = cat (i : 0 < NonMosaic[MosaicIndex]) +static off_t **MosaicToMeasure; // measure for given measure on mosaic : MosaicMeasure[MosaicIndex][i] = cat (i : 0 < NonMosaic[MosaicIndex]) + +// MeasureToMosaic was 'bin' +// MosaicToCatalog was 'clist' +// MosaicToMeasure was 'mlist' + +// N_onMosaic was 'Nlist' +// N_ONMOSAIC was 'NLIST' + +// ImageToMosaic was 'mosimage' + +// MosaicN_image was 'Nimlist' +// MosaicToImage was 'imlist' + +void sort_times (unsigned int *T, int N) { + +# define SWAPFUNC(A,B){ unsigned int tmp; \ + tmp = T[A]; T[A] = T[B]; T[B] = tmp; \ +} +# define COMPARE(A,B)(T[A] < T[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +/* find mosaic frames (unique time periods & photcode name matches mosaic) */ +void initMosaics_old (Image *image, off_t Nimage) { + + off_t i, j, status, found, NMOSAIC, *MosaicN_IMAGE; + unsigned int start, stop; + char *pname; + + if (!MOSAIC_ZEROPT) return; + + Nmosaic = 0; + NMOSAIC = 10; + ALLOCATE (mosaic, Mosaic, NMOSAIC); + + ALLOCATE (MosaicToImage, off_t *, NMOSAIC); + ALLOCATE (MosaicN_Image, off_t, NMOSAIC); + ALLOCATE (MosaicN_IMAGE, off_t, NMOSAIC); + + ALLOCATE (ImageToMosaic, off_t, Nimage); // mosaic to which image belongs + + /* a 'mosaic' in relphot is (unlike relastro) a virtual concept: there is no + * entry in the image table that represents this mosaic. Instead, it is an + * internal construct that defines a group of related images + */ + + /* generate list of unique mosaics */ + for (i = 0; i < Nimage; i++) { + ImageToMosaic[i] = -1; + + /* select valid mosaic images by photcode */ + pname = GetPhotcodeNamebyCode (image[i].photcode); + status = strncmp (pname, MOSAICNAME, strlen (MOSAICNAME)); + if (status) continue; + + /* set image time range */ + start = image[i].tzero - MAX(0.01*image[i].trate*image[i].NY, 1); + stop = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1); + + /* find existing mosaic with this time range */ + found = FALSE; + for (j = 0; !found && (j < Nmosaic); j++) { + if (stop < mosaic[j].start) continue; + if (start > mosaic[j].stop) continue; + found = TRUE; + + // add reference from image to mosaic + ImageToMosaic[i] = j; + + /* add image to mosaic image list */ + MosaicToImage[j][MosaicN_Image[j]] = i; + MosaicN_Image[j] ++; + if (MosaicN_Image[j] == MosaicN_IMAGE[j]) { + MosaicN_IMAGE[j] += 10; + REALLOCATE (MosaicToImage[j], off_t, MosaicN_IMAGE[j]); + } + + } + if (found) continue; + + /* a new mosaic, define ranges -- preserve the original values incase this image is not used */ + mosaic[Nmosaic].start = start; + mosaic[Nmosaic].stop = stop; + mosaic[Nmosaic].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[Nmosaic].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[Nmosaic].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[Nmosaic].dMsys = 0.0; + mosaic[Nmosaic].McalChiSq = 0.0;// NAN or 0.0? + mosaic[Nmosaic].flags = image[i].flags; + mosaic[Nmosaic].secz = image[i].secz; + mosaic[Nmosaic].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); + + // XXX do we need to do something about flag consistency across a mosaic? + + /* add image to mosaic image list */ + MosaicN_IMAGE[Nmosaic] = 10; + MosaicN_Image[Nmosaic] = 1; + ALLOCATE (MosaicToImage[Nmosaic], off_t, MosaicN_IMAGE[Nmosaic]); + MosaicToImage[Nmosaic][0] = i; + + // add reference from image to mosaic + ImageToMosaic[i] = Nmosaic; + + Nmosaic ++; + if (Nmosaic == NMOSAIC) { + NMOSAIC += 10; + REALLOCATE (mosaic, Mosaic, NMOSAIC); + REALLOCATE (MosaicToImage, off_t *, NMOSAIC); + REALLOCATE (MosaicN_Image, off_t, NMOSAIC); + REALLOCATE (MosaicN_IMAGE, off_t, NMOSAIC); + } + } + + // free this or not? + free (MosaicN_IMAGE); + + initMosaicGrid (image, Nimage); + + fprintf (stderr, "matched %d images to %d mosaics\n", (int) Nimage, (int) Nmosaic); + return; +} + +/* find mosaic frames (unique time periods) (NOTE : we do NOT require matching photcodes...) + this function will also identify the images NOT in the subset which belong to a selected mosaic + */ +void initMosaics (Image *subset, off_t Nsubset, Image *image, char *inSubset, off_t Nimage) { + + off_t i, j, found, NMOSAIC, *MosaicN_IMAGE; + unsigned int start, stop, *startTimes, *startTimesMosaic; + + if (!MOSAIC_ZEROPT) return; + + INITTIME; + + /* a 'mosaic' in relphot is (unlike relastro) a virtual concept: there is no + * entry in the image table that represents this mosaic. Instead, it is an + * internal construct that defines a group of related images + */ + + // generate a list of all subset image start times + ALLOCATE (startTimes, unsigned int, Nsubset); + int Nmoschip = 0; + for (i = 0; i < Nsubset; i++) { + if (!isMosaicChip(subset[i].photcode)) continue; + startTimes[Nmoschip] = subset[i].tzero; + Nmoschip ++; + } + sort_times (startTimes, Nmoschip); + MARKTIME("create array of all image obstimes: %f sec\n", dtime); + + Nmosaic = 0; + NMOSAIC = 1000; + ALLOCATE (startTimesMosaic, unsigned int, NMOSAIC); + startTimesMosaic[0] = startTimes[0]; + + // generate a list of the unique start times (these define the mosaics) + for (i = 0; i < Nmoschip; i++) { + myAssert (startTimes[i] >= startTimesMosaic[Nmosaic], "times out of order?"); + if (startTimes[i] == startTimesMosaic[Nmosaic]) continue; + Nmosaic ++; + if (Nmosaic >= NMOSAIC) { + NMOSAIC += 1000; + REALLOCATE (startTimesMosaic, unsigned int, NMOSAIC); + } + startTimesMosaic[Nmosaic] = startTimes[i]; + } + Nmosaic ++; + MARKTIME("create subset array of mosaic obstimes: %f sec\n", dtime); + + // now I have a list of uniq start times, and they are in order + // create the mosaic arrays for these times + ALLOCATE (mosaic, Mosaic, Nmosaic); + + ALLOCATE (MosaicToImage, off_t *, Nmosaic); + ALLOCATE (MosaicN_Image, off_t, Nmosaic); + ALLOCATE (MosaicN_IMAGE, off_t, Nmosaic); + + for (i = 0; i < Nmosaic; i++) { + /* a new mosaic, define ranges */ + mosaic[i].start = startTimesMosaic[i]; + mosaic[i].stop = 0; + mosaic[i].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[i].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0;// NAN or 0.0? + mosaic[i].flags = 0; + mosaic[i].secz = NAN; + mosaic[i].photcode = 0; + mosaic[i].skipCal = FALSE; + mosaic[i].inTGroup = FALSE; // not (yet?) assigned to a TGroup + + memset (&mosaic[i].coords, 0, sizeof(Coords)); + + MosaicN_IMAGE[i] = 10; + MosaicN_Image[i] = 0; + ALLOCATE (MosaicToImage[i], off_t, MosaicN_IMAGE[i]); + MosaicToImage[i][0] = -1; + } + + int Nskip, Nmark; + Nskip = Nmark = 0; + // find any mosaics (startTimesMosaic) which match unselected images + for (i = 0; i < Nimage; i++) { + if (inSubset[i]) { + Nskip ++; + continue; + } + + if (!isMosaicChip(image[i].photcode)) continue; + + /* set image time range */ + start = image[i].tzero; + stop = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1); + + /* find a matching mosaic */ + j = findMosaic(startTimesMosaic, Nmosaic, start); + if (j != -1) { + // mark this mosaic as bad + mosaic[j].skipCal = TRUE; + Nmark ++; + } + } + fprintf (stderr, "%d total images, %d overlap skyregion & match selection criteria, %d do not overlap, but match overlapping mosaics\n", (int) Nimage, (int) Nskip, (int) Nmark); + MARKTIME("find unselected images matching selected mosaics: %f sec\n", dtime); + + ALLOCATE (ImageToMosaic, off_t, Nsubset); // mosaic to which image belongs + + // assign each image to a mosaic + int Nsimple = 0; + for (i = 0; i < Nsubset; i++) { + ImageToMosaic[i] = -1; + + if (!isMosaicChip(subset[i].photcode)) { + Nsimple ++; + continue; + } + + start = subset[i].tzero; + stop = subset[i].tzero + MAX(1.01*subset[i].trate*subset[i].NY, 1); + + j = findMosaic(startTimesMosaic, Nmosaic, start); + if (j == -1) { + fprintf (stderr, "programming error? all subset images should belong to a mosaic\n"); + abort(); + } + + // add reference from image to mosaic + ImageToMosaic[i] = j; + + // have we already found this mosaic? + found = (MosaicN_Image[j] > 0); + + /* add image to mosaic image list */ + MosaicToImage[j][MosaicN_Image[j]] = i; + MosaicN_Image[j] ++; + if (MosaicN_Image[j] == MosaicN_IMAGE[j]) { + MosaicN_IMAGE[j] += 10; + REALLOCATE (MosaicToImage[j], off_t, MosaicN_IMAGE[j]); + } + if (found) continue; + + /* a new mosaic, define ranges */ + if (mosaic[j].start != start) { + fprintf (stderr, "error?\n"); + abort(); + } + mosaic[j].stop = stop; + + mosaic[j].McalPSF = 0.0; + mosaic[j].McalAPER = 0.0; + mosaic[j].dMcal = 0.0; + mosaic[j].McalChiSq = 0.0; + + mosaic[j].nFitPhotom = 0; + mosaic[j].nValPhotom = 0; + + mosaic[j].dMsys = subset[i].dMagSys; + mosaic[j].flags = subset[i].flags; + mosaic[j].secz = subset[i].secz; + mosaic[j].photcode = GetPhotcodeEquivCodebyCode (subset[i].photcode); + } + + // free this or not? + free (MosaicN_IMAGE); + free (startTimes); + free (startTimesMosaic); + + initMosaicGrid (subset, Nsubset); + + fprintf (stderr, "matched %d images to %d mosaics, %d simple chips not matched to mosaics\n", (int) (Nsubset - Nsimple), (int) Nmosaic, (int) Nsimple); + return; +} + +/* find mosaic frames (unique time periods) (NOTE : require gpc1 chips, which is pretty limiting) + if mergeMcal is TRUE, values will be saved on Mosaic.Mcal + */ +void makeMosaics (Image *image, off_t Nimage, int mergeMcal) { + + off_t i, j, found, NMOSAIC, *MosaicN_IMAGE; + unsigned int start, stop, *startTimes, *startTimesMosaic; + + if (!MOSAIC_ZEROPT) return; + + INITTIME; + + /* a 'mosaic' in relphot is (unlike relastro) a virtual concept: there is no + * entry in the image table that represents this mosaic. Instead, it is an + * internal construct that defines a group of related images + */ + + // generate a list of all image start times + ALLOCATE (startTimes, unsigned int, Nimage); + int Nmoschip = 0; + for (i = 0; i < Nimage; i++) { + if (!isMosaicChip(image[i].photcode)) continue; + startTimes[Nmoschip] = image[i].tzero; + Nmoschip ++; + } + sort_times (startTimes, Nmoschip); + MARKTIME("create array of all image obstimes: %f sec\n", dtime); + + Nmosaic = 0; + NMOSAIC = 1000; + ALLOCATE (startTimesMosaic, unsigned int, NMOSAIC); + startTimesMosaic[0] = startTimes[0]; + + // generate a list of the unique start times (these define the mosaics) + for (i = 0; i < Nmoschip; i++) { + myAssert (startTimes[i] >= startTimesMosaic[Nmosaic], "times out of order?"); + if (startTimes[i] == startTimesMosaic[Nmosaic]) continue; + Nmosaic ++; + if (Nmosaic >= NMOSAIC) { + NMOSAIC += 1000; + REALLOCATE (startTimesMosaic, unsigned int, NMOSAIC); + } + startTimesMosaic[Nmosaic] = startTimes[i]; + } + Nmosaic ++; + MARKTIME("create array of mosaic obstimes: %f sec\n", dtime); + + // now I have a list of uniq start times, and they are in order + // create the mosaic arrays for these times + ALLOCATE (mosaic, Mosaic, Nmosaic); + + ALLOCATE (MosaicToImage, off_t *, Nmosaic); + ALLOCATE (MosaicN_Image, off_t, Nmosaic); + ALLOCATE (MosaicN_IMAGE, off_t, Nmosaic); + + // init the mosaic array values + for (i = 0; i < Nmosaic; i++) { + mosaic[i].start = startTimesMosaic[i]; + mosaic[i].stop = 0; + mosaic[i].McalPSF = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].McalAPER = 0.0; // note : mosaic stores only offsets relative to the original image values + mosaic[i].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0;// NAN or 0.0? + mosaic[i].flags = 0; + mosaic[i].secz = NAN; + mosaic[i].photcode = 0; + mosaic[i].skipCal = FALSE; + + memset (&mosaic[i].coords, 0, sizeof(Coords)); + + MosaicN_IMAGE[i] = 10; + MosaicN_Image[i] = 0; + ALLOCATE (MosaicToImage[i], off_t, MosaicN_IMAGE[i]); + MosaicToImage[i][0] = -1; + } + + ALLOCATE (ImageToMosaic, off_t, Nimage); // mosaic to which image belongs + + // assign each image to a mosaic + int Nsimple = 0; + for (i = 0; i < Nimage; i++) { + ImageToMosaic[i] = -1; + + if (!isMosaicChip(image[i].photcode)) { + Nsimple ++; + continue; + } + + start = image[i].tzero; + stop = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1); + + j = findMosaic(startTimesMosaic, Nmosaic, start); + if (j == -1) { + fprintf (stderr, "programming error? all image images should belong to a mosaic\n"); + abort(); + } + + // add reference from image to mosaic + ImageToMosaic[i] = j; + + // have we already found this mosaic? + found = (MosaicN_Image[j] > 0); + + /* add image to mosaic image list */ + MosaicToImage[j][MosaicN_Image[j]] = i; + MosaicN_Image[j] ++; + if (MosaicN_Image[j] == MosaicN_IMAGE[j]) { + MosaicN_IMAGE[j] += 10; + REALLOCATE (MosaicToImage[j], off_t, MosaicN_IMAGE[j]); + } + if (found) continue; + + /* a new mosaic, define ranges */ + if (mosaic[j].start != start) { + fprintf (stderr, "error?\n"); + abort(); + } + mosaic[j].stop = stop; + mosaic[j].McalPSF = 0.0; + mosaic[j].McalAPER = 0.0; + mosaic[j].dMcal = 0.0; + mosaic[j].McalChiSq = 0.0; + mosaic[j].dMsys = image[i].dMagSys; + mosaic[j].flags = image[i].flags; + mosaic[j].secz = image[i].secz; + mosaic[j].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode); + } + MARKTIME("assign images to mosaic: %f sec\n", dtime); + + // free this or not? + free (MosaicN_IMAGE); + free (startTimes); + free (startTimesMosaic); + + if (mergeMcal) { + initMosaicGrid (image, Nimage); + } + + fprintf (stderr, "matched %d images to %d mosaics\n", (int) Nimage, (int) Nmosaic); + return; +} + +Mosaic *getMosaicForImage (off_t im) { + + if (im < 0) return NULL; + if (!ImageToMosaic) return NULL; + + off_t m = ImageToMosaic[im]; + if (m < 0) return NULL; + if (m >= Nmosaic) return NULL; + + return (&mosaic[m]); +} + +// use bisection to find the overlapping mosaic (returns exact match) +// startTimes is a sorted, unique list of times +// start might not be in the list +off_t findMosaic (unsigned int *startTimes, off_t Nmosaic, unsigned int start) { + + off_t Nlo, Nhi, N; + + // find the last mosaic before start + Nlo = 0; // first valid startTimes value + Nhi = Nmosaic - 1; // last valid startTimes value + + // if start is not in this range, return -1 + if (start < startTimes[Nlo]) return (-1); + if (start > startTimes[Nhi]) return (-1); + + while (Nhi - Nlo > 4) { + N = 0.5*(Nlo + Nhi); + if (startTimes[N] < start) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N, Nmosaic - 1); + } + } + // we now have : startTimes[Nlo] < start <= starTimes[Nhi] + + // find a matched mosaic starting from Nlo, or return -1 + for (N = Nlo; N <= Nhi; N++) { + if (startTimes[N] == start) return N; + } + return (-1); +} + +void setMosaicCenters (Image *image, off_t Nimage) { + OHANA_UNUSED_PARAM(Nimage); + + /* find max dR, dD range for all mosaics */ + /* define mosaic.coords to cover dR, dD */ + /* send results to initGridBins */ + + off_t i, j, m, NX, NY, NC, Nc; + double R, D, Rmid, Dmid; + double *Rc, *Dc; + + NC = 100; + ALLOCATE (Rc, double, NC); + ALLOCATE (Dc, double, NC); + + for (i = 0; i < Nmosaic; i++) { + Nc = 0; + Rmid = Dmid = NAN; + for (j = 0; j < MosaicN_Image[i]; j++) { + m = MosaicToImage[i][j]; + + NX = image[m].NX; + NY = image[m].NY; + if (!XY_to_RD (&R, &D, 0.5*NX, 0.5*NY, &image[m].coords)) continue; + R = ohana_normalize_angle_to_midpoint (R, 180.0); + + // Exclude images with crazy astrometry + // XXX NOTE : this is gpc1-specific + { + double dP1 = hypot(image[m].coords.pc1_1, image[m].coords.pc1_2); + double dP2 = hypot(image[m].coords.pc2_1, image[m].coords.pc2_2); + if (fabs(dP1 - 1.0) > 0.02) continue; + if (fabs(dP2 - 1.0) > 0.02) continue; + + double X00, Y00, X10, Y10, X01, Y01; + XY_to_LM (&X00, &Y00, 0.0, 0.0, &image[m].coords); + XY_to_LM (&X10, &Y10, image[m].NX, 0.0, &image[m].coords); + XY_to_LM (&X01, &Y01, 0.0, image[m].NY, &image[m].coords); + double dS0 = hypot ((X00 - X10), (Y00 - Y10)); + double dS1 = hypot ((X00 - X01), (Y00 - Y01)); + if (dS0 > 6000) continue; + if (dS1 > 6500) continue; + } + + Rc[Nc] = R; + Dc[Nc] = D; + Nc ++; + if (Nc >= NC) { + NC += 100; + REALLOCATE (Rc, double, NC); + REALLOCATE (Dc, double, NC); + } + } + + if (Nc > 0) { + dsort (Rc, Nc); + if (Rc[Nc-1] - Rc[0] > 180.0) { + // in our list, Rc is in the range 0.0 to 360.0. + // any mosaic which is close to the 0.0, 360.0 boundary may have some on + // one side or the other. count how many have values more than Rc[0] + 180. + // if more than half are at the large end, re-normalize to that range + int Nbig = 0; + for (j = 1; j < Nc; j++) { + if (Rc[j] - Rc[0] > 180.0) Nbig ++; + } + if (Nbig > 0.5*Nc) { + for (j = 0; j < Nc; j++) { + Rc[j] = ohana_normalize_angle_to_midpoint (Rc[j], 360.0); + } + dsort (Rc, Nc); + } else if (Nbig > 0) { + for (j = 0; j < Nc; j++) { + Rc[j] = ohana_normalize_angle_to_midpoint (Rc[j], 0.0); + } + dsort (Rc, Nc); + } + } + dsort (Dc, Nc); + + Rmid = Rc[(int)(0.5*Nc)]; + Dmid = Dc[(int)(0.5*Nc)]; + } + + InitCoords (&mosaic[i].coords, "DEC--TAN"); + mosaic[i].coords.crval1 = Rmid; + mosaic[i].coords.crval2 = Dmid; + mosaic[i].coords.cdelt1 = 1.0 / 3600.0; + mosaic[i].coords.cdelt2 = 1.0 / 3600.0; + + mosaic[i].McalPSF = 0.0; + mosaic[i].McalAPER = 0.0; + mosaic[i].dMcal = 0.0; + mosaic[i].dMsys = 0.0; + mosaic[i].McalChiSq = 0.0; + } + return; +} + +void initMosaicGrid (Image *image, off_t Nimage) { + OHANA_UNUSED_PARAM(Nimage); + + /* find max dR, dD range for all mosaics */ + /* define mosaic.coords to cover dR, dD */ + /* send results to initGridBins */ + + off_t i, j, m, NX, NY; + int dXmax, dYmax; + double dS, dX, dY; + double R, D, Rmin, Rmax, Dmin, Dmax; + double McalPSF, McalAPER, dMcal, McalChiSq; + + fprintf (stderr, "*** moving Mcal from image.Mcal to mosaic.Mcal ***\n"); + + dXmax = dYmax = 0.0; + for (i = 0; i < Nmosaic; i++) { + Dmin = Rmin = 360.0; + Dmax = Rmax = -360.0; + dS = 0.0; + McalPSF = McalAPER = dMcal = McalChiSq = 0; + for (j = 0; j < MosaicN_Image[i]; j++) { + m = MosaicToImage[i][j]; + NX = image[m].NX; + NY = image[m].NY; + dS += hypot (image[m].coords.cdelt1*image[m].coords.pc1_1, image[m].coords.cdelt1*image[m].coords.pc2_1); + + OhanaProjection proj = GetProjection (image[m].coords.ctype); + if ((proj == PROJ_WRP) && !image[m].coords.mosaic) { + XY_to_LM (&R, &D, 0.0, 0.0, &image[m].coords); + } else { + XY_to_RD (&R, &D, 0.0, 0.0, &image[m].coords); + } + Rmin = MIN (Rmin, R); + Rmax = MAX (Rmax, R); + Dmin = MIN (Dmin, D); + Dmax = MAX (Dmax, D); + + if ((proj == PROJ_WRP) && !image[m].coords.mosaic) { + XY_to_LM (&R, &D, (double) NX, 0.0, &image[m].coords); + } else { + XY_to_RD (&R, &D, (double) NX, 0.0, &image[m].coords); + } + Rmin = MIN (Rmin, R); + Rmax = MAX (Rmax, R); + Dmin = MIN (Dmin, D); + Dmax = MAX (Dmax, D); + + if ((proj == PROJ_WRP) && !image[m].coords.mosaic) { + XY_to_LM (&R, &D, (double) NX, (double) NY, &image[m].coords); + } else { + XY_to_RD (&R, &D, (double) NX, (double) NY, &image[m].coords); + } + Rmin = MIN (Rmin, R); + Rmax = MAX (Rmax, R); + Dmin = MIN (Dmin, D); + Dmax = MAX (Dmax, D); + + if ((proj == PROJ_WRP) && !image[m].coords.mosaic) { + XY_to_LM (&R, &D, 0.0, (double) NY, &image[m].coords); + } else { + XY_to_RD (&R, &D, 0.0, (double) NY, &image[m].coords); + } + Rmin = MIN (Rmin, R); + Rmax = MAX (Rmax, R); + Dmin = MIN (Dmin, D); + Dmax = MAX (Dmax, D); + + // XXX : this probably does not handle mosaics at RA = 0,360 well + + /* we are using mosaic.Mcal, not image.Mcal. reset image.Mcal */ + + // XXX: how does this work with UBERCAL? We want to keep the Mcal values supplied by ubercal, but + // solve for a single offset for each exposure (Mosaic.Mcal). + // we also want to keep the flat-field terms for each exposure (regardless of ubercal or not) + // if it helps, note that ubercal uses a single zp per exposure, so the mean of those values is the same as the value + + if (!isfinite(image[m].McalPSF)) { + image[m].McalPSF = 0.0; + image[m].McalAPER = 0.0; + image[m].dMcal = 0.0; + image[m].McalChiSq = 0.0; + fprintf (stderr, "warning: resetting NAN value for Mcal %s\n", image[m].name); + } + + McalPSF += image[m].McalPSF; + McalAPER += image[m].McalAPER; + dMcal += image[m].dMcal; + McalChiSq += image[m].McalChiSq; + + image[m].McalPSF = 0.0; + image[m].McalAPER = 0.0; + image[m].dMcal = NAN; + image[m].McalChiSq = NAN; + + } + dS /= MosaicN_Image[i]; + + InitCoords (&mosaic[i].coords, "DEC--TAN"); + mosaic[i].coords.crval1 = Rmin; + mosaic[i].coords.crval2 = Dmin; + mosaic[i].coords.cdelt1 = dS; + mosaic[i].coords.cdelt2 = dS; + + RD_to_XY (&dX, &dY, Rmax, Dmax, &mosaic[i].coords); + + mosaic[i].McalPSF = McalPSF / MosaicN_Image[i]; + mosaic[i].McalAPER = McalAPER / MosaicN_Image[i]; + mosaic[i].dMcal = dMcal / MosaicN_Image[i]; + mosaic[i].McalChiSq = McalChiSq / MosaicN_Image[i]; + } + if (!USE_GRID) return; + + dXmax = MAX (dXmax, dX); + dYmax = MAX (dYmax, dY); + initGrid (dXmax, dYmax); + return; +} + +void setMcalFromMosaics () { + + off_t i, j, im, Nimage; + Image *image; + + if (!MOSAIC_ZEROPT) return; + + image = getimages (&Nimage, NULL); + + fprintf (stderr, "*** return Mcal from mosaic.Mcal to image.Mcal ***\n"); + + // copy the mosaic results to the images. set the mosaic Mcal to 0.0 since we have moved its + // impact to the images + for (i = 0; i < Nmosaic; i++) { + for (j = 0; j < MosaicN_Image[i]; j++) { + im = MosaicToImage[i][j]; + if (mosaic[i].flags & ID_IMAGE_MOSAIC_PHOTCAL) { + image[im].McalPSF = mosaic[i].McalPSF; + image[im].McalAPER = mosaic[i].McalAPER; + image[im].dMcal = mosaic[i].dMcal; + image[im].McalChiSq = mosaic[i].McalChiSq; + image[im].ubercalDist = mosaic[i].ubercalDist; + image[im].dMagSys = mosaic[i].dMsys; + image[im].nFitPhotom = mosaic[i].nFitPhotom; + } + image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_POOR); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_MOSAIC_POOR); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_TGROUP_PHOTCAL); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_MOSAIC_PHOTCAL); + image[im].flags |= (mosaic[i].flags & ID_IMAGE_IMAGE_PHOTCAL); + } + mosaic[i].McalPSF = 0.0; + mosaic[i].McalAPER = 0.0; + } +} + +void markBadMosaic (int myMosaic) { + + off_t Nimage; + Image *image; + + // this mosaic has been identified as poor. + mosaic[myMosaic].flags |= ID_IMAGE_MOSAIC_POOR; + + image = getimages (&Nimage, NULL); + + // all images should be marked as coming from a bad mosaic + // these will be fitted independently. Make the starting solution consistent by + // setting the image McalPSF for the current best fit from the mosaic + + for (off_t i = 0; i < MosaicN_Image[myMosaic]; i++) { + off_t im = MosaicToImage[myMosaic][i]; + image[im].flags |= ID_IMAGE_MOSAIC_POOR; + image[im].McalPSF = mosaic[myMosaic].McalPSF; + image[im].McalAPER = mosaic[myMosaic].McalAPER; + } + + mosaic[myMosaic].McalPSF = 0.0; + mosaic[myMosaic].McalAPER = 0.0; +} + +void markGoodMosaic (int myMosaic) { + + off_t Nimage; + Image *image; + + // this mosaic has been identified as good. + mosaic[myMosaic].flags &= ~ID_IMAGE_MOSAIC_POOR; + + image = getimages (&Nimage, NULL); + + // all images should be marked as NOT coming from a bad mosaic + for (off_t i = 0; i < MosaicN_Image[myMosaic]; i++) { + off_t im = MosaicToImage[myMosaic][i]; + image[im].flags &= ~ID_IMAGE_MOSAIC_POOR; + } +} + +void initMosaicBins (Catalog *catalog, int Ncatalog, int doMosaicList) { + + off_t i, j; + + /* measure -> mosaic */ + if (!MOSAIC_ZEROPT) return; + + ALLOCATE (MeasureToMosaic, off_t *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (MeasureToMosaic[i], off_t, MAX (catalog[i].Nmeasure, 1)); + for (j = 0; j < catalog[i].Nmeasure; j++) MeasureToMosaic[i][j] = -1; + } + + if (doMosaicList) { + /* mosaic -> measure */ + ALLOCATE (N_onMosaic, off_t, Nmosaic); + ALLOCATE (N_ONMOSAIC, off_t, Nmosaic); + ALLOCATE (MosaicToCatalog, off_t *, Nmosaic); + ALLOCATE (MosaicToMeasure, off_t *, Nmosaic); + + for (i = 0; i < Nmosaic; i++) { + N_onMosaic[i] = 0; + N_ONMOSAIC[i] = 100; + ALLOCATE (MosaicToCatalog[i], off_t, N_ONMOSAIC[i]); + ALLOCATE (MosaicToMeasure[i], off_t, N_ONMOSAIC[i]); + } + } +} + +void freeMosaicBins (int Ncatalog, int doMosaicList) { + + off_t i; + + /* measure -> mosaic */ + if (!MOSAIC_ZEROPT) return; + + for (i = 0; i < Ncatalog; i++) { + free (MeasureToMosaic[i]); + } + free (MeasureToMosaic); + + if (doMosaicList) { + /* mosaic -> measure */ + for (i = 0; i < Nmosaic; i++) { + free (MosaicToCatalog[i]); + free (MosaicToMeasure[i]); + } + free (N_onMosaic); + free (N_ONMOSAIC); + free (MosaicToCatalog); + free (MosaicToMeasure); + } +} + +void freeMosaics () { + + if (!MOSAIC_ZEROPT) return; + + for (int i = 0; i < Nmosaic; i++) { + FREE (MosaicToImage[i]); + } + + FREE (mosaic); + FREE (MosaicToImage); + FREE (MosaicN_Image); + FREE (ImageToMosaic); // mosaic to which image belongs +} + +int findMosaics (Catalog *catalog, int Ncatalog, int doMosaicList) { + + if (!MOSAIC_ZEROPT) return (FALSE); + // if we are calibrating by mosaic, redefine myDet == on one of my mosaics + + int Nmatch = 0; + for (int i = 0; i < Ncatalog; i++) { + for (off_t j = 0; j < catalog[i].Nmeasure; j++) { + catalog[i].measureT[j].myDet = FALSE; // a detetion is not mine until proven otherwise + + if (TimeSelect) { + if (catalog[i].measureT[j].t < TSTART) continue; + if (catalog[i].measureT[j].t > TSTOP) continue; + } + + int Ns = GetActivePhotcodeIndex (catalog[i].measureT[j].photcode); + if (Ns < 0) continue; + matchMosaics (catalog, j, i, doMosaicList); + Nmatch ++; + } + } + // fprintf (stderr, "matched %d detections to mosaics\n", Nmatch); + return (TRUE); +} + +void matchMosaics (Catalog *catalog, off_t meas, int cat, int doMosaicList) { + + off_t idx, ID, mosID; + MeasureTiny *measure; + + measure = &catalog[cat].measureT[meas]; + + ID = measure[0].imageID; + idx = getImageByID (ID); + if (idx == -1) { + if (VERBOSE2) fprintf (stderr, "missed measurement "OFF_T_FMT", %d\n", meas, cat); + return; + } + + mosID = ImageToMosaic[idx]; + if (mosID < 0) { + // Image *image = getimage(idx); + // fprintf (stderr, "unmatched image %s\n", image[0].name); + // skip measurements from simple chips (not matched to a mosaic by definition) + return; + } + + // test to check we got the right match: + { + Image *image = getimage(idx); + // XXX we are now matching with just tzero. be careful for cameras with drift + // unsigned int imageStart = image[0].tzero - MAX(0.01*image[0].trate*image[0].NY, 1); + unsigned int imageStart = image[0].tzero; + if (imageStart != mosaic[mosID].start) { + fprintf (stderr, "error in image to mosaic match\n"); + abort(); + } + } + + // this measurement is on one of my mosaics, mark it as mine. + catalog[cat].measureT[meas].myDet = TRUE; + MeasureToMosaic[cat][meas] = mosID; + + if (doMosaicList) { + MosaicToCatalog[mosID][N_onMosaic[mosID]] = cat; + MosaicToMeasure[mosID][N_onMosaic[mosID]] = meas; + N_onMosaic[mosID] ++; + + if (N_onMosaic[mosID] == N_ONMOSAIC[mosID]) { + N_ONMOSAIC[mosID] += 100; + REALLOCATE (MosaicToCatalog[mosID], off_t, N_ONMOSAIC[mosID]); + REALLOCATE (MosaicToMeasure[mosID], off_t, N_ONMOSAIC[mosID]); + } + } + return; +} + +float getMmos (off_t meas, int cat) { + + off_t i; + float value; + + if (!MOSAIC_ZEROPT) return (0); + + // unassigned measurements belong to simple chips + i = MeasureToMosaic[cat][meas]; + if (i == -1) return (0.0); + + if (mosaic[i].flags & IMAGE_BAD) return (NAN); + value = mosaic[i].McalPSF; + return (value); +} + +int getMosaicFlags (off_t meas, int cat) { + + if (!MOSAIC_ZEROPT) return (0); + + // unassigned measurements belong to simple chips + int i = MeasureToMosaic[cat][meas]; + if (i == -1) return (0); + + return (mosaic[i].flags); +} + +typedef struct { + int Nfew; + int Nbad; + int Ncal; + int Ngrp; + int Ngrid; + int Nrel; + int Nsys; + int Nskip; + off_t Nmax; + FitDataSet psfStars; + FitDataSet kronStars; + FitDataSet brightStars; +} SetMmosInfo; + +enum {THREAD_RUN, THREAD_DONE}; + +typedef struct { + int entry; + int state; + Catalog *catalog; + Image *image; + FlatCorrectionTable *flatcorr; + SetMmosInfo info; +} ThreadInfo; + +int setMmos_mosaic (Mosaic *mosaic, off_t Nmos, Image *image, Catalog *catalog, SetMmosInfo *info, FlatCorrectionTable *flatcorr); +void *setMmos_worker (void *data); +int setMmos_threaded (Catalog *catalog, FlatCorrectionTable *flatcorr); + +void SetMmosInfoInit (SetMmosInfo *info, off_t Nmax, int allocLists) { + info->Nfew = 0; + info->Nbad = 0; + info->Ncal = 0; + info->Nrel = 0; + info->Ngrid = 0; + info->Nskip = 0; + info->Nsys = 0; + + info->Nmax = Nmax; + + if (allocLists) { + // we can only fit the zero point for individual exposures, not the airmass + FitDataSetAlloc (&info->psfStars, Nmax, 0, 0); + FitDataSetAlloc (&info->kronStars, Nmax, 0, 0); + FitDataSetAlloc (&info->brightStars, Nmax, 0, 0); + + // until the analysis has converged a bit, do not use the IRLS analysis + if (UseStandardOLS(ZPT_MOSAIC)) { + info->brightStars.MaxIterations = 0; + info->kronStars.MaxIterations = 0; + info->psfStars.MaxIterations = 0; + } + } +} + +void SetMmosInfoFree (SetMmosInfo *info) { + FitDataSetFree (&info->brightStars); + FitDataSetFree (&info->kronStars); + FitDataSetFree (&info->psfStars); +} + +void SetMmosInfoAccum (SetMmosInfo *summary, SetMmosInfo *results) { + summary->Nfew += results->Nfew ; + summary->Nsys += results->Nsys ; + summary->Nbad += results->Nbad ; + summary->Ncal += results->Ncal ; + summary->Nrel += results->Nrel ; + summary->Ngrid += results->Ngrid; + summary->Nskip += results->Nskip; +} + +static int npass_output = 0; + +// mutex to lock setMmos_worker operations +static pthread_mutex_t setMmos_mutex = PTHREAD_MUTEX_INITIALIZER; +static int nextMosaic = 0; + +// we have an array of mosaics (mosaic, Nmosaic). we need to hand out mosaics one at a time to +// the worker threads as they need +off_t getNextMosaicForThread () { + + pthread_mutex_lock (&setMmos_mutex); + if (nextMosaic >= Nmosaic) { + pthread_mutex_unlock (&setMmos_mutex); + return (-1); + } + int thisMosaic = nextMosaic; + nextMosaic ++; + + pthread_mutex_unlock (&setMmos_mutex); + return (thisMosaic); +} + +int setMmos (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t i, N, Nmax; + Image *image; + + if (!MOSAIC_ZEROPT) return FALSE; + if (FREEZE_MOSAICS) return FALSE; + + if (MOSAIC_ZPT_MODE == MOSAIC_ZPT_MODE_NONE) return FALSE; + + // plots cannot be done in a threaded context (the plot commands collide) + // so do not run setMmos in threaded mode if PLOTSTUFF is set + if (NTHREADS && !PLOTSTUFF) { + int status = setMmos_threaded (catalog, flatcorr); + return status; + } + + image = getimages (&N, NULL); + + fprintf (stderr, "limiting negative clouds to %f\n", CLOUD_TOLERANCE); + + Nmax = 0; + for (i = 0; i < Nmosaic; i++) { + Nmax = MAX (Nmax, N_onMosaic[i]); + } + + SetMmosInfo info; + SetMmosInfoInit (&info, Nmax, TRUE); + + // int savePlotDelay = PLOTDELAY; + // if (PLOTSTUFF) PLOTDELAY = 0.0; + + for (i = 0; i < Nmosaic; i++) { + setMmos_mosaic (&mosaic[i], i, image, catalog, &info, flatcorr); + } + SetMmosInfoFree (&info); + + if (PLOTSTUFF) { + fprintf (stdout, "press return\n"); + if (fscanf (stdin, "%*c") != 1) fprintf (stderr, "\n"); + } + + npass_output ++; + + fprintf (stderr, "%d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", info.Nfew, info.Nbad, info.Ncal, info.Ngrid, info.Nrel, info.Nsys); + + return (TRUE); +} + +// 'mosaic' is a pointer to the current mosaic of interest (Nmos) +int setMmos_mosaic (Mosaic *myMosaic, off_t Nmos, Image *image, Catalog *catalog, SetMmosInfo *info, FlatCorrectionTable *flatcorr) { + + off_t j, NimageReal; + + FitDataSet *psfStars = &info->psfStars; + FitDataSet *kronStars = &info->kronStars; + FitDataSet *brightStars = &info->brightStars; + + assert (Nmos >= 0); + assert (Nmos < Nmosaic); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + int minUbercalDist = 1000; + + // unset this flag at start (set below if zeropoint is calculated) + myMosaic->flags &= ~ID_IMAGE_MOSAIC_PHOTCAL; // unset this flag + + // calculate the statistics for both good and bad exposures, but only set Mmos if the rules allow + // we only skip the statistics for nights with too few measurements or exposures + + // if we are fitting TGroup zero points, when we identify the bad TGroups, we proceed + // to fit the mosaics which are from the bad TGroups + + // if we are fitting Mosaic zero points, when we identify the bad nights, we proceed + // to fit the images which are from the bad Mosaics + + int badNight = (myMosaic[0].flags & ID_IMAGE_NIGHT_POOR); + int badMosaic = (myMosaic[0].flags & ID_IMAGE_MOSAIC_POOR); + int useMmos = TRUE; + + // in BAD_NIGHT mode, we fit ONLY mosaics in bad nights (do not fit mosaics from good nights) + if (MOSAIC_ZPT_MODE == MOSAIC_ZPT_MODE_BAD_NIGHT) { + if (!badNight) useMmos = FALSE; + } + + // in BAD_NIGHT_GOOD_MOSAIC mode, we fit ONLY good mosaics in bad nights + if ((MOSAIC_ZPT_MODE == MOSAIC_ZPT_MODE_BAD_NIGHT_GOOD_MOSAIC)) { + if (!badNight) useMmos = FALSE; // do not fit good nights + if (badMosaic) useMmos = FALSE; // do not fit bad mosaics + } + + // in GOOD_MOSAIC mode, we fit good mosaics ignoring night state + // This case should only be used when not fitting nights / tgroups + if ((MOSAIC_ZPT_MODE == MOSAIC_ZPT_MODE_GOOD_MOSAIC)) { + if (badMosaic) useMmos = FALSE; // do not fit bad mosaics + } + + // Image *imageReal = getimages (&NimageReal, NULL); returned pointer is not used + getimages (&NimageReal, NULL); + + // UBERCAL image: if this is an ubercal image, set minUbercalDist to 0: + // we optionally do not recalibrate images with UBERCAL zero points + if (myMosaic[0].flags & ID_IMAGE_PHOTOM_UBERCAL) { + myMosaic[0].ubercalDist = 0; + // propagate ubercalDist to the images + for (j = 0; j < MosaicN_Image[Nmos]; j++) { + off_t im = MosaicToImage[Nmos][j]; + assert (im < NimageReal); + assert (im >= 0); + image[im].ubercalDist = myMosaic[0].ubercalDist; + // fprintf (stderr, "%d %d %d\n", (int) i, (int) im, image[im].ubercalDist); + } + // XXX if KEEP_UBERCAL, calculate stats but do not change Mmos + if (KEEP_UBERCAL) return TRUE; + } + + // do not modify the calibration for mosaics with partial images loaded (skipCal TRUE) + if (myMosaic[0].skipCal) { + info->Nskip ++; + return TRUE; + } + + int testImage = FALSE; + // testImage |= (abs(myMosaic[0].start - 1324104046) < 10); + + FILE *fout = NULL; + if (testImage) { + char filename[64]; + snprintf (filename, 64, "test.%05d.%02d.dat", (int) Nmos, npass_output); + fout = fopen (filename, "w"); + save_test_mosaic_measures (fout, Nmos, catalog, flatcorr); + fclose (fout); + } + + // number of stars to measure the bright-end scatter + int Nbright = 0; + + int N = 0; + for (j = 0; j < N_onMosaic[Nmos]; j++) { + + off_t m = MosaicToMeasure[Nmos][j]; + off_t c = MosaicToCatalog[Nmos][j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + info->Nbad ++; + continue; + } + float Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) { + info->Ncal++; + continue; + } + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) { + info->Ngrp++; + continue; + } + float Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) { + info->Ngrid ++; + continue; + } + + // Mrel* is the average magnitude for this star. For PS1 stacks, we have too much + // PSF variability. We need to calibrate the PSF magnitudes separately from the + // Aperture-like magnitues. (We have an option to use the kron magnitudes or the + // other apertures here). I basically need to do this analysis separately for each + // magnitude type + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { + info->Nrel ++; + continue; + } + float MrelKron = getMrel (catalog, m, c, MAG_CLASS_KRON, MAG_SRC_CHP); + + // image.Mcal is not supposed to include the flat-field correction, so we need to + // apply that offset as well here for this image (in other words, each detection is + // being compared to the model, excluding the zero point, Mcal. The model includes + // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) + + float Mflat = getMflat (m, c, flatcorr, catalog); + + off_t n = catalog[c].measureT[m].averef; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { + info->Nsys++; + continue; + } + float MsysKron = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_KRON); + + PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode); + if (!code) goto skip; + if (code->equiv < 1) goto skip; + int Nsec = GetPhotcodeNsec (code->equiv); + if (Nsec == -1) goto skip; + minUbercalDist = MIN (catalog[c].secfilt[n*Nsecfilt + Nsec].ubercalDist, minUbercalDist); + + skip: + assert (N < info->Nmax); + assert (N >= 0); + assert (Nbright < info->Nmax); + assert (Nbright >= 0); + + float Moff = Mcal + Mgrp + Mgrid - Mflat; + + psfStars->alldata-> yVector[N] = MsysPSF - MrelPSF - Moff; + psfStars->alldata->dyVector[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + + kronStars->alldata-> yVector[N] = MsysKron - MrelKron - Moff; + kronStars->alldata->dyVector[N] = psfStars->alldata->dyVector[N]; + + if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) { + brightStars->alldata-> yVector[Nbright] = psfStars->alldata-> yVector[N]; + brightStars->alldata->dyVector[Nbright] = psfStars->alldata->dyVector[N]; + Nbright ++; + } + N++; + + } + /* N_onMosaic[Nmos] is all measurements, N is good measurements */ + + /* skip mosaics with too few good measurements or too many bad measurements */ + int mark = (N < IMAGE_TOOFEW) || (N < IMAGE_GOOD_FRACTION*N_onMosaic[Nmos]); + if (mark) { + myMosaic->flags |= ID_IMAGE_PHOTOM_FEW; + myMosaic->McalPSF = 0.0; + myMosaic->McalAPER = 0.0; + myMosaic->dMcal = NAN; + myMosaic->stdev = NAN; + myMosaic->dMmin = NAN; + myMosaic->dMmax = NAN; + myMosaic->McalChiSq = NAN; + myMosaic->nFitPhotom = 0; + myMosaic->nValPhotom = N; + info->Nfew ++; + if (testImage) { + fprintf (stderr, "NOTE: *** marked test image poor : %d %d %d***\n", (int) N, (int) IMAGE_TOOFEW, (int) (IMAGE_GOOD_FRACTION*N_onMosaic[Nmos])); + } + return TRUE; // skip mosaics with too few good measurements + } else { + myMosaic[0].flags &= ~ID_IMAGE_PHOTOM_FEW; + } + + // soften the errors based on the scatter + FitDataSetSoften (psfStars, N); + + fit1d_irls (psfStars, N); + fit1d_irls (brightStars, Nbright); + fit1d_irls (kronStars, N); + + if (useMmos) { + myMosaic->McalPSF = psfStars->bSaveArray[0][0]; + myMosaic->McalAPER = kronStars->bSaveArray[0][0]; + myMosaic->flags |= ID_IMAGE_MOSAIC_PHOTCAL; // set this flag + } else { + myMosaic->McalPSF = 0.0; + myMosaic->McalAPER = 0.0; + } + + // for now, I have no reason to measure these separately for camera-level images + myMosaic->dMcal = psfStars->bSigma[0]; + myMosaic->stdev = psfStars->sigma; + myMosaic->McalChiSq = psfStars->chisq; + myMosaic->nFitPhotom = psfStars->Nmeas; + myMosaic->nValPhotom = N; + + // bright end scatter + myMosaic->dMsys = brightStars->sigma; // stdev of bright star mags + + if (testImage) { + fprintf (stderr, "test image %d (%d) %f %f %d ... ", (int) Nmos, myMosaic->start, myMosaic->McalPSF, myMosaic->dMcal, myMosaic->nFitPhotom); + fprintf (stderr, "%f %f : %f\n", myMosaic[0].McalPSF, myMosaic[0].dMsys, myMosaic[0].McalChiSq); + } + + if (PLOTSTUFF) { + fprintf (stderr, "Mmos: %6.3f +/- %6.3f %5d %5d | %s\n", myMosaic->McalPSF, myMosaic->dMcal, myMosaic->nFitPhotom, N, image[MosaicToImage[Nmos][0]].name); + plot_setMcal (psfStars->alldata->yVector, N); + } + + // minUbercalDist calculated here is the min value for any star owned by this image + // since this particular image is tied to that star, bump its distance by 1 + myMosaic[0].ubercalDist = minUbercalDist + 1; + + // propagate ubercalDist to the images + for (j = 0; j < MosaicN_Image[Nmos]; j++) { + off_t im = MosaicToImage[Nmos][j]; + image[im].ubercalDist = myMosaic[0].ubercalDist; + } + + fprintf (stderr, "MOSAIC time %.6f photcode %d Mcal: %f, dMcal: %f, chisq: %f, %d of %d, useMmos: %d\n", + ohana_sec_to_mjd(myMosaic[0].start), myMosaic[0].photcode, psfStars->bSaveArray[0][0], myMosaic[0].dMcal, myMosaic[0].McalChiSq, myMosaic[0].nFitPhotom, myMosaic[0].nValPhotom, useMmos); + + return TRUE; +} + +int save_test_mosaic_measures (FILE *fout, int Nmos, Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int Nsecfilt = GetPhotcodeNsecfilt (); + + for (int j = 0; j < N_onMosaic[Nmos]; j++) { + + off_t m = MosaicToMeasure[Nmos][j]; + off_t c = MosaicToCatalog[Nmos][j]; + + // NOTE : we are only using Mcal == McalPSF in this function; we set McalAPER to McalPSF + float Mcal = getMcal (m, c, MAG_CLASS_PSF); + // float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + float Mgrid = getMgrid (m, c); + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + float Mflat = getMflat (m, c, flatcorr, catalog); + + off_t n = catalog[c].measureT[m].averef; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + float delta = MsysPSF - MrelPSF - Mcal - Mgrid + Mflat; + + int isBad = (catalog[c].measureT[m].dbFlags & MEAS_BAD); + + fprintf (fout, "%f %f : %f %f %f %f %f : %f %d\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysPSF, MrelPSF, Mcal, Mgrid, Mflat, delta, isBad); + } + return TRUE; +} + +int setMmos_threaded (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int i; + off_t N; + + Image *image = getimages (&N, NULL); + + fprintf (stderr, "limiting negative clouds to %f\n", CLOUD_TOLERANCE); + + off_t Nmax = 0; + for (i = 0; i < Nmosaic; i++) { + Nmax = MAX (Nmax, N_onMosaic[i]); + } + + SetMmosInfo summary; + SetMmosInfoInit (&summary, Nmax, FALSE); + + pthread_attr_t attr; + pthread_attr_init (&attr); + pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED); + + pthread_t *threads; + ALLOCATE (threads, pthread_t, NTHREADS); + + ThreadInfo *threadinfo; + ALLOCATE (threadinfo, ThreadInfo, NTHREADS); + + // each time this function is called, we cycle through the available mosaics + // make sure we start at 0 + nextMosaic = 0;; + + // launch N worker threads + for (i = 0; i < NTHREADS; i++) { + threadinfo[i].entry = i; + threadinfo[i].state = THREAD_RUN; + threadinfo[i].catalog = catalog; + threadinfo[i].image = image; + threadinfo[i].flatcorr = flatcorr; + + // we do NOT allocate the arrays here, we only supply basic info (Nmax, Nloop) used in + // the threads to allocate the arrays and set the MaxIterations + SetMmosInfoInit (&threadinfo[i].info, Nmax, FALSE); + pthread_create (&threads[i], NULL, setMmos_worker, &threadinfo[i]); + } + pthread_attr_destroy (&attr); + + // wait until all threads have finished + while (1) { + int allDone = TRUE; + for (i = 0; i < NTHREADS; i++) { + if (threadinfo[i].state == THREAD_RUN) allDone = FALSE; + } + if (allDone) { + break; + } + usleep (500000); + } + + // all threads are done, free the threads array and grab the info + free (threads); + + // report stats & summary from the threads + for (i = 0; i < NTHREADS; i++) { + fprintf (stderr, "setMmos thread %d : %d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n", + i, + threadinfo[i].info.Nfew, + threadinfo[i].info.Nbad, + threadinfo[i].info.Ncal, + threadinfo[i].info.Ngrid, + threadinfo[i].info.Nrel, + threadinfo[i].info.Nsys, + threadinfo[i].info.Nskip); + SetMmosInfoAccum (&summary, &threadinfo[i].info); + } + fprintf (stderr, "total : %d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n", + summary.Nfew, + summary.Nbad, + summary.Ncal, + summary.Ngrid, + summary.Nrel, + summary.Nsys, + summary.Nskip); + free (threadinfo); + + npass_output ++; + + return TRUE; +} + +void *setMmos_worker (void *data) { + + ThreadInfo *threadinfo = data; + + SetMmosInfo results; + SetMmosInfoInit (&results, threadinfo->info.Nmax, TRUE); + + while (1) { + + off_t i = getNextMosaicForThread(); + if (i == -1) { + threadinfo->state = THREAD_DONE; + return NULL; + } + + Catalog *catalog = threadinfo->catalog; + FlatCorrectionTable *flatcorr = threadinfo->flatcorr; + Image *image = threadinfo->image; + + setMmos_mosaic (&mosaic[i], i, image, catalog, &results, flatcorr); + SetMmosInfoAccum (&threadinfo->info, &results); + } + + SetMmosInfoFree (&results); + return NULL; +} + +// When we rationalize the images/mosaics, we are driving the negative cloud images back +// to 0.0. At the same time, we make a guess to the effective impact on all other images, +// driven by the coupling of common stars. +// XXX this function is no longer used because we force significantly negative clouds to 0.0 +int rationalize_mosaics (Catalog *catalog, int Ncatalog) { + + double *imageOffset, **starOffset; + int **starNcount, *seclist; + int **Slist, *NSlist, *NSLIST; + int i, j, k, m, nMos, Ns, found; + + off_t Nimage; + Image *image; + + // set a test value for now + float CLOUD_TOLERANCE = 0.025; + + if (!MOSAIC_ZEROPT) return (FALSE); + if (FREEZE_MOSAICS) return (FALSE); + + image = getimages (&Nimage, NULL); + + ALLOCATE (imageOffset, double, Nmosaic); + ALLOCATE ( Slist, int *, Nmosaic); // array of calibrated star indexes on this mosaic + ALLOCATE (NSlist, int, Nmosaic); // number of stars on mosaic + ALLOCATE (NSLIST, int, Nmosaic); // number of Slist entries allocated + memset (Slist, 0, Nmosaic*sizeof(int *)); + + // find the images / mosaics with negative clouds and save their offset + for (i = 0; i < Nmosaic; i++) { + + NSlist[i] = 0; + NSLIST[i] = 100; + ALLOCATE (Slist[i], int, NSLIST[i]); + + imageOffset[i] = 0.0; + + if (VERBOSE2 && (fabs(mosaic[i].McalPSF) < CLOUD_TOLERANCE)) { + fprintf (stderr, "cloud-free: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); + } + if (VERBOSE2 && (mosaic[i].McalPSF < -CLOUD_TOLERANCE)) { + imageOffset[i] = -mosaic[i].McalPSF; + // NOTE the negative sign: down below, we are going to add in the negative of McalPSF + // to this image, and the propagated mean values for other images + fprintf (stderr, "anti-clouds: %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); + } + if (VERBOSE2 && (mosaic[i].McalPSF > CLOUD_TOLERANCE)) { + fprintf (stderr, "cloudy : %s : %f\n", image[MosaicToImage[i][0]].name, mosaic[i].McalPSF); + } + } + + int Nsecfilt = GetPhotcodeNsecfilt (); + ALLOCATE (seclist, int, Nphotcodes); + for (Ns = 0; Ns < Nphotcodes; Ns ++) { + int thisCode = photcodes[Ns][0].code; + seclist[Ns] = GetPhotcodeNsec(thisCode); + } + + // allocate an array for star offsets + int Nstars = 0; + int NSTARS = 1000; + ALLOCATE (starOffset, double *, NSTARS); + ALLOCATE (starNcount, int *, NSTARS); + memset (starOffset, 0, NSTARS*sizeof(double *)); + memset (starNcount, 0, NSTARS*sizeof(int *)); + + // find the mean offset for each star + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + ALLOCATE (starOffset[Nstars], double, Nphotcodes); + ALLOCATE (starNcount[Nstars], int, Nphotcodes); + memset (starOffset[Nstars], 0, Nphotcodes*sizeof(double)); + memset (starNcount[Nstars], 0, Nphotcodes*sizeof(int)); + + m = catalog[i].averageT[j].measureOffset; + + // determine the mosaic for each measurement + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + + // skip unused measurements + if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue; + + // skip unused measurements + + // XXX replace with: + // Ns = GetActivePhotcodeIndex (photcode); + // if (Ns < 0) continue; + + int Nsec; + int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); + found = FALSE; + for (Ns = 0; !found && (Ns < Nphotcodes); Ns ++) { + if (ecode == photcodes[Ns][0].code) { + found = TRUE; + break; + } + } + if (!found) continue; + Nsec = seclist[Ns]; + + // bad stars for this secfilt + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + + // skip REF measurements (not tied to an image) + if (getImageEntry (m, i) < 0) continue; + + // find the source of this measurement (skip unassigned measurements) + nMos = MeasureToMosaic[i][m]; + if (nMos == -1) continue; + + // XXX how can this not be true? + if (mosaic[nMos].photcode != ecode) { + fprintf (stderr, "*"); + } + + assert (Ns >= 0); + assert (Ns < Nphotcodes); + + // accumulate the offsets from the negative cloud images (others have 0.0 value) + starOffset[Nstars][Ns] += imageOffset[nMos]; + starNcount[Nstars][Ns] ++; + + // record the mosaic->star reference + Slist[nMos][NSlist[nMos]] = Nstars; + NSlist[nMos] ++; + if (NSlist[nMos] == NSLIST[nMos]) { + NSLIST[nMos] += 100; + REALLOCATE (Slist[nMos], int, NSLIST[nMos]); + } + } + Nstars ++; + if (Nstars == NSTARS) { + NSTARS += 1000; + REALLOCATE (starOffset, double *, NSTARS); + REALLOCATE (starNcount, int *, NSTARS); + memset (&starOffset[NSTARS-1000], 0, 1000*sizeof(double *)); + memset (&starNcount[NSTARS-1000], 0, 1000*sizeof(int *)); + } + } + } + + // find the mean offset of the images without negative clouds + for (i = 0; i < Nmosaic; i++) { + + found = FALSE; + for (Ns = 0; !found && (Ns < Nphotcodes); Ns ++) { + if (mosaic[i].photcode == photcodes[Ns][0].code) { + found = TRUE; + break; + } + } + if (!found) { + fprintf (stderr, "invalid photcode for mosaic?\n"); + abort(); + } + + // a negative cloud image (cloud: McalPSF > 0; anti-clouds: McalPSF < 0; imageOffset = -McalPSF) + if (imageOffset[i] > 0.0) continue; + + // we need to actually have cross-references to count + if (NSlist[i] < 2) continue; + + float dM = 0.0; + for (j = 0; j < NSlist[i]; j++) { + k = Slist[i][j]; + if (starNcount[k][Ns] > 1) { + dM += (starOffset[k][Ns] / starNcount[k][Ns]); + } + } + imageOffset[i] = dM / NSlist[i]; + // fprintf (stderr, "adjust image: %s : (%f %d) : %f\n", image[MosaicToImage[i][0]].name, dM, NSlist[i], imageOffset[i]); + } + + // for (i = 0; i < Nmosaic; i++) { + // fprintf (stderr, "correction: %s : %f\n", image[MosaicToImage[i][0]].name, imageOffset[i]); + // } + + // apply all offset values to the mosaics + // find the images / mosaics with negative clouds and save their offset + for (i = 0; i < Nmosaic; i++) { + mosaic[i].McalPSF += imageOffset[i]; + } + + for (i = 0; i < Nstars; i++) { + free (starOffset[i]); + free (starNcount[i]); + } + free (starOffset); + free (starNcount); + + free (seclist); + free (NSlist); + free (NSLIST); + for (i = 0; i < Nmosaic; i++){ + free (Slist[i]); + } + free (Slist); + free (imageOffset); + return (TRUE); +} + +StatType statsMosaicM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i; + int n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!MOSAIC_ZEROPT) return (stats); + + ALLOCATE (list, double, Nmosaic); + ALLOCATE (dlist, double, Nmosaic); + + n = 0; + for (i = 0; i < Nmosaic; i++) { + if (mosaic[i].flags & IMAGE_BAD) continue; + if (mosaic[i].skipCal) continue; + if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + list[n] = mosaic[i].McalPSF; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsMosaicdM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i, n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!MOSAIC_ZEROPT) return (stats); + if (FREEZE_MOSAICS) return (stats); + + ALLOCATE (list, double, Nmosaic); + ALLOCATE (dlist, double, Nmosaic); + + n = 0; + for (i = 0; i < Nmosaic; i++) { + if (mosaic[i].flags & IMAGE_BAD) continue; + if (mosaic[i].skipCal) continue; + if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + list[n] = mosaic[i].dMcal; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsMosaicN (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t i, j, m, c, n, N; + double *list, *dlist; + float Mcal, Mgrp, Mgrid, Mrel; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!MOSAIC_ZEROPT) return (stats); + if (FREEZE_MOSAICS) return (stats); + + ALLOCATE (list, double, Nmosaic); + ALLOCATE (dlist, double, Nmosaic); + + n = 0; + for (i = 0; i < Nmosaic; i++) { + if (mosaic[i].flags & IMAGE_BAD) continue; + if (mosaic[i].skipCal) continue; + if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + N = 0; + for (j = 0; j < N_onMosaic[i]; j++) { + + m = MosaicToMeasure[i][j]; + c = MosaicToCatalog[i][j]; + + Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + if (isnan(Mgrp)) continue; + Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) continue; + Mrel = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(Mrel)) continue; + N++; + } + list[n] = N; + dlist[n] = 1; + n++; + } + // fprintf (stderr, "Nmosaic: "OFF_T_FMT", n: "OFF_T_FMT"\n", Nmosaic, n); + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsMosaicX (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + off_t i, n; + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!MOSAIC_ZEROPT) return (stats); + if (FREEZE_MOSAICS) return (stats); + + ALLOCATE (list, double, Nmosaic); + ALLOCATE (dlist, double, Nmosaic); + + n = 0; + for (i = 0; i < Nmosaic; i++) { + if (mosaic[i].flags & IMAGE_BAD) continue; + if (mosaic[i].skipCal) continue; + if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + list[n] = mosaic[i].McalChiSq; + dlist[n] = 1; + n++; + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +/* mark mosaic if: abs(Mcal - ) too large, dMcal too large */ + +static float MinMaxChiSq = NAN; +static float MinMaxScatter = NAN; + +void clean_mosaics () { + + double *mlist, *slist; + + if (!MOSAIC_ZEROPT) return; + if (FREEZE_MOSAICS) return; + + if (VERBOSE) fprintf (stderr, "marking poor mosaics\n"); + + ALLOCATE (mlist, double, Nmosaic); + ALLOCATE (slist, double, Nmosaic); + + // Measure the good/bad statistics using mosaics which are actually calibrated. + // But they could be applied to all mosaics below (allowing bad mosaics to become good) + // However, for now, we will NOT allow mosaics to be redeemed (bad -> good) + + int N = 0; + for (int i = 0; i < Nmosaic; i++) { + if (!(mosaic[i].flags & ID_IMAGE_MOSAIC_PHOTCAL)) continue; + if (mosaic[i].skipCal) continue; + mlist[N] = mosaic[i].McalChiSq; + slist[N] = mosaic[i].stdev; + N++; + } + + StatType stats; + liststats_setmode (&stats, "MEAN"); + + liststats (mlist, NULL, NULL, N, &stats); + float ChiSqUpper90 = stats.Upper90; + if (isnan (MinMaxChiSq)) MinMaxChiSq = 2.0*stats.median; + float MaxChiSq = MAX (MinMaxChiSq, ChiSqUpper90); + + liststats (slist, NULL, NULL, N, &stats); + float ScatterUpper90 = stats.Upper90; + if (isnan (MinMaxScatter)) MinMaxScatter = 2.0*stats.median; + float MaxScatter = MAX (MinMaxScatter, ScatterUpper90); + + fprintf (stderr, "MOSAIC: Max ChiSq: %f, Max Scatter: %f | ChiSquare Upper 90: %f, Scatter Upper 90: %f\n", MaxChiSq, MaxScatter, ChiSqUpper90, ScatterUpper90); + + int Ntotal = 0, Npoor = 0, Nmark = 0, Nscatter = 0, Nfew = 0, Nchisq = 0; + for (int i = 0; i < Nmosaic; i++) { + // if we are keeping ubercal sacrosanct, then we should not be allowed to break them... + if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue; + + Ntotal ++; + + if (mosaic[i].skipCal) continue; + + int mark = FALSE; + + // we do not allow bad nights to be redeemed + if (mosaic[i].flags & ID_IMAGE_MOSAIC_POOR) { + Npoor ++; + continue; + } + + if (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW) { + mark = TRUE; + Nfew ++; + } + if (mosaic[i].stdev > MaxScatter) { + mark = TRUE; + Nscatter ++; + } + if (mosaic[i].McalChiSq > MaxChiSq) { + mark = TRUE; + Nchisq ++; + } + if (mark) { + Nmark ++; + markBadMosaic(i); // mark the images associated with a bad night + } else { + markGoodMosaic(i); // mark the images associated with a good night + } + } + + fprintf (stderr, "%d + %d of %d mosaics marked poor (%d scatter, %d chisq, %d few)\n", Nmark, Npoor, Ntotal, Nscatter, Nchisq, Nfew); + + free (mlist); + free (slist); +} + +void plot_mosaic_fields (Catalog *catalog) { + + off_t i, j, m, c, N, Nimage; + double *xlist, *ylist; + char string[64]; + Graphdata graphdata; + + if (!MOSAIC_ZEROPT) return; + + // Image *image = getimages (&Nimage, NULL); returned value ignored + getimages (&Nimage, NULL); + + N = 0; + for (i = 0; i < Nmosaic; i++) { + N = MAX (N, N_onMosaic[i]); + } + + ALLOCATE (xlist, double, N); + ALLOCATE (ylist, double, N); + + for (i = 0; i < Nmosaic; i++) { + N = 0; + for (j = 0; j < N_onMosaic[i]; j++) { + + m = MosaicToMeasure[i][j]; + c = MosaicToCatalog[i][j]; + + if (catalog[c].measureT[m].dbFlags & (ID_MEAS_AREA | ID_MEAS_NOCAL)) continue; + + // ave = catalog[c].measureT[m].averef; + xlist[N] = catalog[c].measureT[m].R; + ylist[N] = catalog[c].measureT[m].D; + N++; + } + + sprintf (string, "Mosaic "OFF_T_FMT, i); + plot_defaults (&graphdata); + plot_list (&graphdata, xlist, ylist, N, string, NULL); + } + + free (ylist); + free (xlist); +} + +void plot_mosaics () { + + off_t i, bin; + double *xlist, *Mlist, *dlist; + Graphdata graphdata; + + if (!MOSAIC_ZEROPT) return; + if (FREEZE_MOSAICS) return; + + ALLOCATE (xlist, double, Nmosaic); + ALLOCATE (dlist, double, Nmosaic); + ALLOCATE (Mlist, double, Nmosaic); + + for (i = 0; i < Nmosaic; i++) { + Mlist[i] = mosaic[i].McalPSF; + dlist[i] = mosaic[i].dMcal; + xlist[i] = mosaic[i].secz; + } + + plot_defaults (&graphdata); + graphdata.xmin = 0.95; + graphdata.xmax = 2.50; + graphdata.ymin = PlotdMmin; + graphdata.ymax = PlotdMmax; + plot_list (&graphdata, xlist, Mlist, Nmosaic, "airmass vs Mcal", "%s.airmass.png", OUTROOT); + plot_defaults (&graphdata); + graphdata.size = 1.5; + graphdata.ptype = 7; + plot_list (&graphdata, Mlist, dlist, Nmosaic, "Mcal vs dMcal", "%s.MdM.png", OUTROOT); + +# define NBIN 200 + REALLOCATE (xlist, double, NBIN); + REALLOCATE (Mlist, double, NBIN); + + /**** dMcal histgram ****/ + for (i = 0; i < NBIN; i++) xlist[i] = 0.00005*i; + bzero (Mlist, NBIN*sizeof(double)); + for (i = 0; i < Nmosaic; i++) { + bin = mosaic[i].dMcal / 0.00005; + bin = MAX (0, MIN (NBIN - 1, bin)); + Mlist[bin] += 1.0; + } + plot_defaults (&graphdata); + graphdata.style = 1; + plot_list (&graphdata, xlist, Mlist, NBIN, "dMcal hist", "%s.dMcalhist.png", OUTROOT); + + free (dlist); + free (xlist); + free (Mlist); +} + +off_t *SelectRefMosaic (Mosaic **refmosaic, off_t *Nimage) { + + off_t i, Imax, Nmax; + + Imax = 0; + Nmax = MosaicN_Image[0]; + for (i = 0; i < Nmosaic; i++) { + if (MosaicN_Image[i] > Nmax) { + Imax = i; + Nmax = MosaicN_Image[i]; + } + } + + *refmosaic = &mosaic[Imax]; + *Nimage = Nmax; + return (MosaicToImage[Imax]); +} Index: /branches/eam_branches/relphot.20210521/src/ReadTGroupFile.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ReadTGroupFile.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ReadTGroupFile.c (revision 41602) @@ -0,0 +1,100 @@ +# include "relphot.h" + +# define NBUFFER 0x10000 +# define IsCSV FALSE +# define COLUMN_NUMBER 1 + +unsigned int *ReadTGroupFile (FILE *f, int *nelem) { + + // we allocate one extra byte into which we never read so there will always be a NULL terminating the string + ALLOCATE_PTR (buffer, char, NBUFFER + 1); + bzero (buffer, NBUFFER + 1); + + int Nstart = 0; // location of the last valid byte in the buffer (start filling here) + int EndOfFile = FALSE; + + // int TimeFormat = TIME_MJD; + // time_t TimeReference = 0; + + int Nelem = 0; + int NELEM = 1000; + ALLOCATE_PTR (elem, unsigned int, NELEM); + + while (!EndOfFile) { + + int Nbytes = NBUFFER - Nstart; + bzero (&buffer[Nstart], Nbytes + 1); + // ^-- we have allocated one extra byte into which we never read so there will always be a NULL terminating the string + + int nread = fread (&buffer[Nstart], 1, Nbytes, f); + if (ferror (f)) { + perror ("error reading data file"); + return NULL; + } + + // we still need to parse the rest of the buffer, but there might not be an EOL on the last line + if (nread == 0) { + EndOfFile = TRUE; + } + + int bufferStatus = TRUE; + + char *c0 = buffer; // c0 always marks the start of a line + while (bufferStatus) { + char *c1 = strchr (c0, '\n'); // find the end of this current line (also valid for a Mac: \r\n) + if (!c1) { + c1 = strchr (c0, '\r'); // try \r for Windows files + } + if (!c1) { + Nstart = strlen (c0); + if (EndOfFile) { + // if we have reached EOF, we need to do one last pass in case there is a line without a return + c1 = c0 + Nstart; + bufferStatus = FALSE; + if (Nstart == 0) continue; // if we have reached EOF and c0 points at the last valid character, we are done + } else { + // if we have not reached EOF, we need to shift the buffer to the start of this line and read more data + memmove (buffer, c0, Nstart); + bufferStatus = FALSE; + continue; + } + } + *c1 = 0; // mark the end of the line + + // skip to the next line (but if EOF, do not overrun buffer) + if (*c0 == '#') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + if (*c0 == '!') { if (!EndOfFile) { c0 = c1 + 1; } continue; } + + // the times are written as MJD in the file: + double dvalue; + int readStatus = IsCSV ? dparse_csv (&dvalue, COLUMN_NUMBER, c0) : dparse (&dvalue, COLUMN_NUMBER, c0); + if (!readStatus) continue; // skip invalid entries + + // convert the (double) mjd value to (unsigned int) time + elem[Nelem] = ohana_mjd_to_sec (dvalue); + + Nelem ++; + if (Nelem == NELEM) { + NELEM += 1000; + REALLOCATE (elem, unsigned int, NELEM); + } + + if (!EndOfFile) { + c0 = c1 + 1; + } + } + } + FREE (buffer); + + *nelem = Nelem; + return elem; +} + +/* + time_t tvalue; + int readStatus = IsCSV ? tparse_csv (&tvalue, COLUMN_NUMBER, c0) : tparse (&tvalue, COLUMN_NUMBER, c0); + if (!readStatus) continue; // skip invalid entries + + dvalue = TimeValue (tvalue, TimeReference, TimeFormat); + elem[Nelem] = (int) dvalue; +*/ Index: /branches/eam_branches/relphot.20210521/src/RepairWarpMeasures.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/RepairWarpMeasures.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/RepairWarpMeasures.c (revision 41602) @@ -0,0 +1,181 @@ +# include "relphot.h" + +FILE *logfile = NULL; + +void RepairWarpMeasuresOpenLogfile () { + + char name[DVO_MAX_PATH]; + snprintf (name, DVO_MAX_PATH, "%s/repair.warp.log", HOSTDIR); + logfile = fopen (name, "w"); + if (!logfile) fprintf (stderr, "failed to open repair warp logfile %s\n", name); +} + +void RepairWarpMeasuresCloseLogfile () { + if (!logfile) return; + fclose (logfile); +} + +// XXX need to load_images first and generate some lookup tables +int RepairWarpMeasures (Catalog *catalog) { + + off_t Nimage; + Image *image = getimages (&Nimage, NULL); + + myAssert (!catalog->Nmeasure || catalog->measure, "programming error"); + + int NfixChipID = 0, NfixStackID = 0, NfixWarpID = 0, NfixWarpImageID = 0, NmissWarp = 0, NmissStack = 0, NbadWarp = 0; + + int onePercent = catalog->Nmeasure / 100; + + for (off_t j = 0; j < catalog->Nmeasure; j++) { + if (j % onePercent == 0) fprintf (stderr, "."); + + Measure *measure = &catalog->measure[j]; + + // get the associated average value: + int averef = measure->averef; + Average *average = &catalog->average[averef]; + + // double-check for consistency + myAssert (average->objID == measure->objID, "objID mismatch: %f %f : %d %d\n", average->R, average->D, average->objID, measure->objID); + + // repair extID for non-warps: + if (isGPC1chip(measure->photcode)) { + double mjd = ohana_sec_to_mjd (measure->t); + int ccdnum = measure->photcode % 100; + uint64_t extID = CreatePSPSDetectionID (mjd, ccdnum, measure->detID); + if (extID != measure->extID) { + measure->extID = extID; + NfixChipID ++; + } + continue; + } + + // repair extID for non-warps: + if (isGPC1stack(measure->photcode)) { + int im = getImageByID (measure->imageID); + if (im < 0) { + // if we don't have a matching image, we are probably assigned to an image on the wrong side of the sky + // reconstruct the imageID from the R,D,photcode & X,Y consistency + im = GetStackSeq (image, average->R, average->D, measure->photcode, measure->Xccd, measure->Yccd); + if (im < 0) { + // still failed, this is bad + if (NmissStack < 10) fprintf (stderr, "missing stack exposure: %f %f : %d %d\n", measure->R, measure->D, measure->imageID, measure->photcode); + NmissStack ++; + continue; + } + // the old imageID was wrong. replace with the new one: + if (logfile) fprintf (logfile, "fix stack %10.6f %10.6f %5d %6.1f %6.1f 0x%08x %d -> %d\n", average->R, average->D, measure->photcode, measure->Xccd, measure->Yccd, measure->detID, measure->imageID, image[im].imageID); + measure->imageID = image[im].imageID; + } else { + // we have a matching image, but check that it is correct by re-projecting the pixel position + double Xtst, Ytst; + RD_to_XY (&Xtst, &Ytst, average->R, average->D, &image[im].coords); + + // find the pixel offset + double dX = (Xtst - measure->Xccd); + double dY = (Ytst - measure->Yccd); + + // if dPos is small, we have the right image + double dPos = hypot(dX,dY); + if (dPos > 10.0) { + // if dPos is large, we don't the right imageID, reconstruct the imageID from + // the R,D,photcode & X,Y consistency + im = GetStackSeq (image, average->R, average->D, measure->photcode, measure->Xccd, measure->Yccd); + if (im < 0) { + // still failed, this is bad + if (NmissStack < 10) fprintf (stderr, "missing stack exposure: %f %f : %d %d\n", measure->R, measure->D, measure->imageID, measure->photcode); + NmissStack ++; + continue; + } + // the old imageID was wrong. replace with the new one: + if (logfile) fprintf (logfile, "fix stack %10.6f %10.6f %5d %6.1f %6.1f 0x%08x %d -> %d\n", average->R, average->D, measure->photcode, measure->Xccd, measure->Yccd, measure->detID, measure->imageID, image[im].imageID); + measure->imageID = image[im].imageID; + } + } + uint64_t extID = CreatePSPSStackDetectionID (35, image[im].externID, measure->detID); + if (extID != measure->extID) { + measure->extID = extID; + NfixStackID ++; + } + continue; + } + + // we are only going to repair warp detections + if (!isGPC1warp(measure->photcode)) continue; + + // warp coordinates to confirm warp + double X = measure->Xccd; + double Y = measure->Yccd; + + // check if this detection is far from its correct location + double Rwrp = measure->R; + double Dwrp = measure->D; + + double Rave = average->R; + double Dave = average->D; + + float csdec = cos(Dave * RAD_DEG); + + // find the ra,dec displacement in arcsec: + double dR = 3600.0*(Rwrp - Rave)*csdec; + double dD = 3600.0*(Dwrp - Dave); + + // skip detections which are within a small distance of the expected location + // NOTE: this actually works surprisingly well near the pole + double dPos = hypot(dR,dD); + if (dPos < 2.0) { + int im = getImageByID (measure->imageID); + uint64_t extID = CreatePSPSStackDetectionID (34, image[im].externID, measure->detID); + if (extID != measure->extID) { + measure->extID = extID; + NfixWarpID ++; + } + continue; + } + + // find the corrected warp ID: + int warpSeq = GetWarpSeq (image, measure->t, measure->photcode, Rave, Dave, X, Y); + if (warpSeq < 0) { + if (NmissWarp < 10) fprintf (stderr, "missing warp exposure: %f %f : %d %d\n", Rave, Dave, measure->t, measure->photcode); + NmissWarp ++; + continue; + } + + // assert on coords being TAN? + Coords *imcoords = &image[warpSeq].coords; + + double Rnew, Dnew; + XY_to_RD (&Rnew, &Dnew, X, Y, imcoords); + + Rnew = ohana_normalize_angle_to_midpoint(Rnew, 180.0); + Dnew = ohana_normalize_angle_to_midpoint(Dnew, 0.0); + + // find the ra,dec displacement in arcsec: + double dRnew = 3600.0*(Rnew - Rave)*csdec; + double dDnew = 3600.0*(Dnew - Dave); + + // detections should now be repaired; fail if not + dPos = hypot(dRnew,dDnew); + if (dPos > 5.0) { + fprintf (stderr, "measurement still far from average location: %f %f vs %f %f : %d %d\n", Rave, Dave, Rnew, Dnew, measure->t, measure->photcode); + NbadWarp ++; + } + + measure->imageID = image[warpSeq].imageID; + measure->R = Rnew; + measure->D = Dnew; + + uint64_t extID = CreatePSPSStackDetectionID (34, image[warpSeq].externID, measure->detID); + if (extID != measure->extID) { + measure->extID = extID; + NfixWarpID ++; + } + NfixWarpImageID ++; + } + + fprintf (stderr, "\n"); + fprintf (stderr, "repaired %s : warp image ID %d : ext ID chip %d stack %d warp %d : missed warp %d stack %d : bad warp: %d\n", catalog->filename, NfixWarpImageID, NfixChipID, NfixStackID, NfixWarpID, NmissWarp, NmissStack, NbadWarp); + + return (TRUE); +} Index: /branches/eam_branches/relphot.20210521/src/SetSignals.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/SetSignals.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/SetSignals.c (revision 41602) @@ -0,0 +1,123 @@ +# include "relphot.h" + +static int Protect = FALSE; +static int Trapped = FALSE; + +void TrapSignal (int sig) { + fprintf (stderr, "trapped signal %d\n", sig); + if (sig == 11) { + fprintf (stderr, "seg fault\n"); + exit (1); + } + if (Protect) { + Trapped = TRUE; + fprintf (stderr, "blocking until protected sections are clear\n"); + return; + } + Shutdown ("%s","halted by signal (trapped)"); +} + +void SetProtect (int mode) { + Protect = mode; + if (Trapped && !Protect) Shutdown ("%s","halted by signal (protect)"); +} + +int SetSignals () { + + int i; + + /* disable almost all signal interrupts */ + for (i = 0; i < 36; i++) { + switch (i) { + /* can't redirect these signals */ + case SIGKILL: /* kill -9: cannot be caught or ignored */ + case SIGSTOP: /* SIGSTOP: cannot be caught or ignored */ + + /* ignore these signals */ + case SIGCHLD: /* child halted: ignore */ + case SIGCONT: /* continue - maintain this action */ + case SIGTSTP: /* stop signal sent from tty - why ignore? */ + case SIGURG: /* socket signal, ignore this */ +# ifdef SIGPWR + case SIGPWR: /* power failure - why ignore this? (Sys V) */ +# endif +# ifdef SIGWINCH + case SIGWINCH: /* window resized (4.3BSD) */ +# endif + break; + + default: + signal (i, TrapSignal); + } + } + return (TRUE); +} +/* + + Signal Value Action Comment + ------------------------------------------------------------------------- + SIGHUP 1 A Hangup detected on controlling terminal + or death of controlling process + SIGINT 2 A Interrupt from keyboard + SIGQUIT 3 A Quit from keyboard + SIGILL 4 A Illegal Instruction + SIGABRT 6 C Abort signal from abort(3) + SIGFPE 8 C Floating point exception + SIGKILL 9 AEF Kill signal + SIGSEGV 11 C Invalid memory reference + SIGPIPE 13 A Broken pipe: write to pipe with no readers + SIGALRM 14 A Timer signal from alarm(2) + SIGTERM 15 A Termination signal + SIGUSR1 30,10,16 A User-defined signal 1 + SIGUSR2 31,12,17 A User-defined signal 2 + SIGCHLD 20,17,18 B Child stopped or terminated + SIGCONT 19,18,25 Continue if stopped + SIGSTOP 17,19,23 DEF Stop process + SIGTSTP 18,20,24 D Stop typed at tty + SIGTTIN 21,21,26 D tty input for background process + SIGTTOU 22,22,27 D tty output for background process + + Next various other signals. + + Signal Value Action Comment + --------------------------------------------------------------------- + SIGTRAP 5 CG Trace/breakpoint trap + SIGIOT 6 CG IOT trap. A synonym for SIGABRT + SIGEMT 7,-,7 G + SIGBUS 10,7,10 AG Bus error + SIGSYS 12,-,12 G Bad argument to routine (SVID) + SIGSTKFLT -,16,- AG Stack fault on coprocessor + SIGURG 16,23,21 BG Urgent condition on socket (4.2 BSD) + SIGIO 23,29,22 AG I/O now possible (4.2 BSD) + SIGPOLL AG A synonym for SIGIO (System V) + SIGCLD -,-,18 G A synonym for SIGCHLD + SIGXCPU 24,24,30 AG CPU time limit exceeded (4.2 BSD) + SIGXFSZ 25,25,31 AG File size limit exceeded (4.2 BSD) + SIGVTALRM 26,26,28 AG Virtual alarm clock (4.2 BSD) + SIGPROF 27,27,29 AG Profile alarm clock + SIGPWR 29,30,19 AG Power failure (System V) + SIGINFO 29,-,- G A synonym for SIGPWR + SIGLOST -,-,- AG File lock lost + SIGWINCH 28,28,20 BG Window resize signal (4.3 BSD, Sun) + SIGUNUSED -,31,- AG Unused signal + (Here - denotes that a signal is absent; there where three values are given, the first one is usually valid for alpha and + sparc, the middle one for i386 and ppc, the last one for mips. Signal 29 is SIGINFO / SIGPWR on an alpha but SIGLOST on a + sparc.) + + The letters in the "Action" column have the following meanings: + + A Default action is to terminate the process. + + B Default action is to ignore the signal. + + C Default action is to dump core. + + D Default action is to stop the process. + + E Signal cannot be caught. + + F Signal cannot be ignored. + + G Not a POSIX.1 conformant signal. + +*/ Index: /branches/eam_branches/relphot.20210521/src/Shutdown.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/Shutdown.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/Shutdown.c (revision 41602) @@ -0,0 +1,26 @@ +# include "relphot.h" + +static FITS_DB *db = NULL; + +void set_db (FITS_DB *in) { + db = in; +} + +/* clean up open / locked ImageCat before shutting down */ +int Shutdown (char *format, ...) { + va_list argp; + char *formatplus; + + ALLOCATE (formatplus, char, strlen(format)); + strcpy (formatplus, format); + strcat (formatplus, "\n"); + + va_start (argp, format); + vfprintf (stderr, formatplus, argp); + free (formatplus); + va_end (argp); + + fprintf (stderr, "ERROR: relphot halted\n"); + exit (1); +} + Index: /branches/eam_branches/relphot.20210521/src/StackImageMaps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/StackImageMaps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/StackImageMaps.c (revision 41602) @@ -0,0 +1,165 @@ +# include "relphot.h" + +// generate an index of stack image ra,dec centers sorted indexes to the image IDs + +# define D_NSTACKS 1000 +# define NGROUPS 5 + +typedef struct { + unsigned short photcode; + double *Rcenter; + double *Dcenter; + int *imageID; + int *imageSeq; + int Nstacks; + int NSTACKS; +} StackGroupType; + +// one group for each photcode +StackGroupType *stackgroup = NULL; + +int MakeStackIndex (void) { + + off_t Nimage; + Image *image = getimages (&Nimage, NULL); + + ALLOCATE (stackgroup, StackGroupType, NGROUPS); + + for (int i = 0; i < NGROUPS; i++) { + stackgroup[i].photcode = 11000 + i*100; + + stackgroup[i].Nstacks = 0; + stackgroup[i].NSTACKS = D_NSTACKS; + ALLOCATE (stackgroup[i].Rcenter, double, stackgroup[i].NSTACKS); + ALLOCATE (stackgroup[i].Dcenter, double, stackgroup[i].NSTACKS); + ALLOCATE (stackgroup[i].imageID, int, stackgroup[i].NSTACKS); + ALLOCATE (stackgroup[i].imageSeq, int, stackgroup[i].NSTACKS); + } + + // first generate a list of obstime values + for (off_t i = 0; i < Nimage; i++) { + if (!isGPC1stack(image[i].photcode)) continue; + + int Ng = -1; + switch (image[i].photcode) { + case 11000: Ng = 0; break; + case 11100: Ng = 1; break; + case 11200: Ng = 2; break; + case 11300: Ng = 3; break; + case 11400: Ng = 4; break; + default: myAbort("impossible photcode"); + } + myAssert (Ng < NGROUPS, "oops"); + + int N = stackgroup[Ng].Nstacks; + XY_to_RD (&stackgroup[Ng].Rcenter[N], &stackgroup[Ng].Dcenter[N], 0.5*image[i].NX, 0.5*image[i].NY, &image[i].coords); + stackgroup[Ng].Rcenter[N] = ohana_normalize_angle (stackgroup[Ng].Rcenter[N]); + + stackgroup[Ng].imageSeq[N] = i; + stackgroup[Ng].imageID[N] = image[i].imageID; + + stackgroup[Ng].Nstacks ++; + + if (stackgroup[Ng].Nstacks >= stackgroup[Ng].NSTACKS) { + stackgroup[Ng].NSTACKS += D_NSTACKS; + REALLOCATE (stackgroup[Ng].Rcenter, double, stackgroup[Ng].NSTACKS); + REALLOCATE (stackgroup[Ng].Dcenter, double, stackgroup[Ng].NSTACKS); + REALLOCATE (stackgroup[Ng].imageID, int, stackgroup[Ng].NSTACKS); + REALLOCATE (stackgroup[Ng].imageSeq, int, stackgroup[Ng].NSTACKS); + } + } + + // sort the list: + for (int i = 0; i < NGROUPS; i++) { + sort_by_ra (stackgroup[i].Rcenter, stackgroup[i].Dcenter, stackgroup[i].imageID, stackgroup[i].imageSeq, stackgroup[i].Nstacks); + } + + return TRUE; +} + +// find the stack which yields the given Rstk, Dstk for the given X,Y (and photcode) +int GetStackSeq (Image *image, double Rstk, double Dstk, unsigned short photcode, float X, float Y) { + + if (!stackgroup) return -1; + + // we have the stack centers; find all stacks within 0.3 degrees of this point + Rstk = ohana_normalize_angle (Rstk); + double dD = 0.3; + double dR = dD / cos(RAD_DEG*Dstk); + double Rmin = Rstk - dR; + double Rmax = Rstk + dR; + double Dmin = Dstk - dD; + double Dmax = Dstk + dD; + + int Ng = (photcode / 100) % 10; + int N = ohana_bisection_double (stackgroup[Ng].Rcenter, stackgroup[Ng].Nstacks, Rmin); + + double dPosMin = NAN; + int nPosMin = -1; + + for (; N < stackgroup[Ng].Nstacks; N++) { + if (stackgroup[Ng].Dcenter[N] < Dmin) continue; + if (stackgroup[Ng].Dcenter[N] > Dmax) continue; + if (stackgroup[Ng].Rcenter[N] > Rmax) break; + + int im = stackgroup[Ng].imageSeq[N]; + + // project to image pixels + double Xtst, Ytst; + RD_to_XY (&Xtst, &Ytst, Rstk, Dstk, &image[im].coords); + + // find the pixel offset + double dX = (Xtst - X); + double dY = (Ytst - Y); + + // if dPos is small, we have the right image + double dPos = hypot(dX,dY); + if (dPos < 10.0) return im; // 10 pixels == 2.0 arcsec + + // check for multiple possible matches?? + + // record the min pos so we can see how things fail + if (isnan(dPosMin)) { + dPosMin = dPos; + nPosMin = im; + } else { + if (dPos < dPosMin) { + dPosMin = dPos; + nPosMin = im; + } + } + } + + // we did not find the right image, return what we found (-1 or closest match) + return nPosMin; +} + +void FreeStackGroups (void) { + + if (!stackgroup) return; + + for (int i = 0; i < NGROUPS; i++) { + free (stackgroup[i].Rcenter); + free (stackgroup[i].Dcenter); + free (stackgroup[i].imageID); + free (stackgroup[i].imageSeq); + } + free (stackgroup); +} + +void sort_by_ra (double *R, double *D, int *I, int *S, int N) { + +# define SWAPFUNC(A,B){ double dtmp; int itmp; \ + dtmp = R[A]; R[A] = R[B]; R[B] = dtmp; \ + dtmp = D[A]; D[A] = D[B]; D[B] = dtmp; \ + itmp = I[A]; I[A] = I[B]; I[B] = itmp; \ + itmp = S[A]; S[A] = S[B]; S[B] = itmp; \ + } +# define COMPARE(A,B)(R[A] < R[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} Index: /branches/eam_branches/relphot.20210521/src/StarOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/StarOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/StarOps.c (revision 41602) @@ -0,0 +1,1122 @@ +# include "relphot.h" + +enum {THREAD_RUN, THREAD_DONE}; + +typedef struct { + int entry; + int state; + Catalog *catalog; + int Ncatalog; + FlatCorrectionTable *flatcorr; + SetMrelInfo summary; +} ThreadInfo; + +// ** internal functions: +void *setMrel_worker (void *data); +int setMrel_threaded (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr); +int setMrel_catalog (Catalog *catalog, int Nc, int pass, FlatCorrectionTable *flatcorr, SetMrelInfo *results, int Nsecfilt); +int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure); + +// we want to allocate the StatDataSet arrays only once (or once per thread). this +// function finds the largest value of Nmeasure so we can allocate that max size when +// needed +static int Nmax; +void initMrel (Catalog *catalog, int Ncatalog) { + + off_t i, j; + + Nmax = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + Nmax = MAX (Nmax, catalog[i].averageT[j].Nmeasure); + } + } +} + +// +float getMrel (Catalog *catalog, off_t meas, int cat, dvoMagClassType class, dvoMagSourceType source) { + + int ave = catalog[cat].measureT[meas].averef; + int photcode = catalog[cat].measureT[meas].photcode; + + int ecode = GetPhotcodeEquivCodebyCode (photcode); + int Nsec = GetPhotcodeNsec(ecode); + int Nsecfilt = GetPhotcodeNsecfilt (); + + int entry = Nsecfilt*ave+Nsec; + + SecFilt *secfilt = &catalog[cat].secfilt[entry]; + + // is this star OK? + if (secfilt->flags & STAR_BAD) return (NAN); + + switch (class) { + case MAG_CLASS_PSF: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MpsfChp; + case MAG_SRC_WRP: + return secfilt->MpsfWrp; + case MAG_SRC_STK: + return secfilt->MpsfStk; + default: + return NAN; + } + break; + case MAG_CLASS_KRON: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MkronChp; + case MAG_SRC_WRP: + return secfilt->MkronWrp; + case MAG_SRC_STK: + return secfilt->MkronStk; + default: + return NAN; + } + break; + case MAG_CLASS_APER: + switch (source) { + case MAG_SRC_CHP: + return secfilt->MapChp; + case MAG_SRC_WRP: + return secfilt->MapWrp; + case MAG_SRC_STK: + return secfilt->MapStk; + default: + return NAN; + } + break; + case MAG_CLASS_NONE: + case MAG_CLASS_DEV: // DeVaucouleur Model (only for galphot) + case MAG_CLASS_EXP: // Exponential Model (only for galphot) + case MAG_CLASS_SER: // Sersic Model (only for galphot) + return NAN; + break; + } + return NAN; // should not be able to reach here +} + +void SetMrelInfoReset (SetMrelInfo *results) { + results->Nfew = 0; + results->Ncode = 0; + results->Nsys = 0; + results->Nbad = 0; + results->Ncal = 0; + results->Nmos = 0; + results->Ngrp = 0; + results->Ngrid = 0; +} + +static int NsynthNames = 6; +static float synthMaxMags[] = {13.5, 13.5, 13.5, 13.0, 12.0, 13.5}; +static char *synthNames[] = { "g", "r", "i", "z", "y", "w"}; + +void SetMrelInfoResetObject (SetMrelInfo *results) { + + int i; + for (i = 0; i < results->Nsecfilt; i++) { + results->havePS1[i] = FALSE; + results->haveSYN[i] = FALSE; + results->needSYN[i] = FALSE; + results->measSYN[i] = -1; + + results->psfQfMax[i] = NAN; + results->psfQfPerfMax[i] = NAN; + + results->Nmeas[i] = 0; + results->NmeasGood[i] = 0; + results->Next[i] = 0; + results->NexpPS1[i] = 0; + results->havePS1[i] = 0; + results->haveUbercal[i] = 0; + + results->Nstargal = 0; + results->Nphotflags = 0; + + results->tessID[i] = 0; + results->projID[i] = 0; + results->skycellID[i] = 0; + + results->minUbercalDist[i] = 1000; + + results->psfData[i].Nlist = 0; + results->aperData[i].Nlist = 0; + results->kronData[i].Nlist = 0; + } +} + +void SetMrelInfoInit (SetMrelInfo *results, int allocLists) { + + int i; + SetMrelInfoReset (results); + if (allocLists) { + results->Nsecfilt = GetPhotcodeNsecfilt (); + + results->psfData = StatDataSetAlloc (results->Nsecfilt, Nmax); + results->aperData = StatDataSetAlloc (results->Nsecfilt, Nmax); + results->kronData = StatDataSetAlloc (results->Nsecfilt, Nmax); + + ALLOCATE (results->psfqf_list, double, Nmax); + ALLOCATE (results->psfqfperf_list, double, Nmax); + ALLOCATE (results->stargal_list, double, Nmax); + ALLOCATE (results->photflag_list, uint32_t, Nmax); + + ALLOCATE (results->havePS1, int, results->Nsecfilt); + ALLOCATE (results->haveSYN, int, results->Nsecfilt); + ALLOCATE (results->needSYN, int, results->Nsecfilt); + ALLOCATE (results->measSYN, int, results->Nsecfilt); + ALLOCATE (results->minSYN, float, results->Nsecfilt); + + ALLOCATE (results->psfQfMax, float, results->Nsecfilt); + ALLOCATE (results->psfQfPerfMax, float, results->Nsecfilt); + + ALLOCATE (results->Nmeas, int, results->Nsecfilt); + ALLOCATE (results->NmeasGood, int, results->Nsecfilt); + ALLOCATE (results->Next, int, results->Nsecfilt); + ALLOCATE (results->NexpPS1, int, results->Nsecfilt); + ALLOCATE (results->haveUbercal, int, results->Nsecfilt); + + ALLOCATE (results->tessID, int, results->Nsecfilt); + ALLOCATE (results->projID, int, results->Nsecfilt); + ALLOCATE (results->skycellID, int, results->Nsecfilt); + + ALLOCATE (results->minUbercalDist, float, results->Nsecfilt); + + SetMrelInfoResetObject (results); + + for (i = 0; i < NsynthNames; i++) { + int photcode = GetPhotcodeCodebyName (synthNames[i]); + if (!photcode) continue; + int Nsec = GetPhotcodeNsec (photcode); + if (Nsec < 0) continue; + results->minSYN[Nsec] = synthMaxMags[i]; + } + } else { + results->Nsecfilt = 0; + results->psfData = NULL; + results->aperData = NULL; + results->kronData = NULL; + + results->psfqf_list = NULL; + results->psfqfperf_list = NULL; + results->stargal_list = NULL; + results->photflag_list = NULL; + + results->havePS1 = NULL; + results->haveSYN = NULL; + results->needSYN = NULL; + results->measSYN = NULL; + results->minSYN = NULL; + + results->psfQfMax = NULL; + results->psfQfPerfMax = NULL; + + results->Nmeas = NULL; + results->NmeasGood = NULL; + results->Next = NULL; + results->NexpPS1 = NULL; + results->haveUbercal = NULL; + + results->tessID = NULL; + results->projID = NULL; + results->skycellID = NULL; + + results->minUbercalDist = NULL; + } +} + +void SetMrelInfoFree (SetMrelInfo *results) { + + StatDataSetFree (results->psfData, results->Nsecfilt); + StatDataSetFree (results->aperData, results->Nsecfilt); + StatDataSetFree (results->kronData, results->Nsecfilt); + + FREE (results->psfqf_list); + FREE (results->psfqfperf_list); + FREE (results->stargal_list); + FREE (results->photflag_list); + + FREE (results->havePS1); + FREE (results->haveSYN); + FREE (results->needSYN); + FREE (results->measSYN); + FREE (results->minSYN ); + + FREE (results->psfQfMax); + FREE (results->psfQfPerfMax); + + FREE (results->Nmeas); + FREE (results->NmeasGood); + FREE (results->Next); + FREE (results->NexpPS1); + FREE (results->haveUbercal); + + FREE (results->tessID); + FREE (results->projID); + FREE (results->skycellID); + + FREE (results->minUbercalDist); +} + +void SetMrelInfoAccum (SetMrelInfo *summary, SetMrelInfo *results) { + summary->Nfew += results->Nfew ; + summary->Ncode += results->Ncode ; + summary->Nsys += results->Nsys ; + summary->Nbad += results->Nbad ; + summary->Ncal += results->Ncal ; + summary->Nmos += results->Nmos ; + summary->Ngrp += results->Ngrp ; + summary->Ngrid += results->Ngrid; +} + +// mutex to lock setMrel_worker operations +static pthread_mutex_t setMrel_mutex = PTHREAD_MUTEX_INITIALIZER; +static int nextCatalog = 0; + +// we have an array of catalogs (catalog, Ncatalog). we need to hand out catalogs one at a time to +// the worker threads as they need +off_t getNextCatalogForThread (int Ncatalog) { + + pthread_mutex_lock (&setMrel_mutex); + if (nextCatalog >= Ncatalog) { + pthread_mutex_unlock (&setMrel_mutex); + return (-1); + } + int thisCatalog = nextCatalog; + nextCatalog ++; + + pthread_mutex_unlock (&setMrel_mutex); + return (thisCatalog); +} + +// setMrel and setMrelOutput are extremely similar, but have slightly different implications: +// * setMrel uses the internal Tiny structures only +// * setMrelOutput skips stars for which there are too few good measurements +// * setMrelOutput is meant to be called repeatedly, relaxing the criteria for 'good' on each pass +// * setMrelOutput updates 2MASS average flags +// * setMrelOutput updates average EXT flags (PS1 and 2MASS) + +int setMrel (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + int i; + + if (NTHREADS) { + int status = setMrel_threaded (catalog, Ncatalog, flatcorr); + return status; + } + + int Nsecfilt = GetPhotcodeNsecfilt (); + + SetMrelInfo summary, results; + SetMrelInfoInit (&summary, FALSE); + SetMrelInfoInit (&results, TRUE); + + for (i = 0; i < Ncatalog; i++) { + setMrel_catalog_alt (catalog, i, FALSE, flatcorr, &results, Nsecfilt); + SetMrelInfoAccum (&summary, &results); + } + if (VERBOSE2) fprintf (stderr, "%d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrp: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrp, summary.Ngrid, summary.Nsys); + + SetMrelInfoFree (&results); + SetMrelInfoFree (&summary); + + return (TRUE); +} + +// setMrelOutput is NOT threaded because we read/write catalogs one at a time (for now) +int setMrelOutput (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + int i; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + SetMrelInfo summary, results; + SetMrelInfoInit (&summary, FALSE); + SetMrelInfoInit (&results, TRUE); // allocates results->list,dlist,wlist + + for (i = 0; i < Ncatalog; i++) { + setMrel_catalog_alt (catalog, i, TRUE, flatcorr, &results, Nsecfilt); + SetMrelInfoAccum (&summary, &results); + } + if (VERBOSE2) fprintf (stderr, "%d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrp: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrp, summary.Ngrid, summary.Nsys); + + SetMrelInfoFree (&results); + SetMrelInfoFree (&summary); + return (TRUE); +} + +int setMrel_threaded (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + int i; + + SetMrelInfo summary; + SetMrelInfoInit (&summary, FALSE); + + pthread_attr_t attr; + pthread_attr_init (&attr); + pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED); + + pthread_t *threads; + ALLOCATE (threads, pthread_t, NTHREADS); + + ThreadInfo *threadinfo; + ALLOCATE (threadinfo, ThreadInfo, NTHREADS); + + // each time this function is called, we cycle through the available catalogs. + // make sure we start at 0 + nextCatalog = 0;; + + // launch N worker threads + for (i = 0; i < NTHREADS; i++) { + threadinfo[i].entry = i; + threadinfo[i].state = THREAD_RUN; + threadinfo[i].catalog = catalog; + threadinfo[i].Ncatalog = Ncatalog; + threadinfo[i].flatcorr = flatcorr; + SetMrelInfoInit (&threadinfo[i].summary, FALSE); + pthread_create (&threads[i], NULL, setMrel_worker, &threadinfo[i]); + } + pthread_attr_destroy (&attr); + + // wait until all threads have finished + while (1) { + int allDone = TRUE; + for (i = 0; i < NTHREADS; i++) { + if (threadinfo[i].state == THREAD_RUN) allDone = FALSE; + } + if (allDone) { + break; + } + usleep (500000); + } + + // all threads are done, free the threads array and grab the info + free (threads); + + // report stats & summary from the threads + for (i = 0; i < NTHREADS; i++) { + if (VERBOSE2) fprintf (stderr, "setMrel thread %d : %d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrp: %d, Ngrid: %d, Nsys: %d)\n", + i, + threadinfo[i].summary.Ncode, + threadinfo[i].summary.Nfew, + threadinfo[i].summary.Nbad, + threadinfo[i].summary.Ncal, + threadinfo[i].summary.Nmos, + threadinfo[i].summary.Ngrp, + threadinfo[i].summary.Ngrid, + threadinfo[i].summary.Nsys); + SetMrelInfoAccum (&summary, &threadinfo[i].summary); + SetMrelInfoFree (&threadinfo[i].summary); + } + if (VERBOSE2) fprintf (stderr, "total : %d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrp: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrp, summary.Ngrid, summary.Nsys); + free (threadinfo); + + SetMrelInfoFree (&summary); + return TRUE; +} + +void *setMrel_worker (void *data) { + + ThreadInfo *threadinfo = data; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + SetMrelInfo results; + SetMrelInfoInit (&results, TRUE); // allocate list, dlist arrays here + + while (1) { + off_t i = getNextCatalogForThread(threadinfo->Ncatalog); + if (i == -1) { + threadinfo->state = THREAD_DONE; + SetMrelInfoFree (&results); + return NULL; + } + + Catalog *catalog = threadinfo->catalog; + FlatCorrectionTable *flatcorr = threadinfo->flatcorr; + + setMrel_catalog_alt (catalog, i, FALSE, flatcorr, &results, Nsecfilt); + SetMrelInfoAccum (&threadinfo->summary, &results); + } + + SetMrelInfoFree (&results); + return NULL; +} + +# define SKIP_THIS_MEAS(REASON) { \ + catalog[Nc].measureT[m].dbFlags |= ID_MEAS_SKIP_PHOTOM; \ + if (catalog[Nc].measure) { \ + catalog[Nc].measure [m].dbFlags |= ID_MEAS_SKIP_PHOTOM; \ + } \ + results->REASON ++; \ + continue; } + + +int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure) { + + off_t k; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + off_t m = average[0].measureOffset; + + for (k = 0; k < average[0].Nmeasure; k++, m++) { + fprintf (stderr, "meas: %08x\n", measure[m].dbFlags); + } + + int Ns; + for (Ns = 0; Ns < Nsecfilt; Ns++) { + fprintf (stderr, "secf: %08x\n", secfilt[Ns].flags); + } + return 1; +} + +/* set measure.Mcal for all measures except ID_MEAS_NOCAL and ID_IMAGE_PHOTOM_NOCAL */ +int setMcalOutput (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + int i; + off_t j, k, m; + + MEAS_BAD = ID_MEAS_NOCAL; + IMAGE_BAD = ID_IMAGE_PHOTOM_NOCAL; + + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + m = catalog[i].averageT[j].measureOffset; + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue; + float McalPSF = getMcal (m, i, MAG_CLASS_PSF); + float McalAPER = getMcal (m, i, MAG_CLASS_APER); + if (isnan(McalPSF)) continue; + float Mmos = getMmos (m, i); + if (isnan(Mmos)) continue; + float Mgrp = getMgrp (m, i, catalog[i].measureT[m].airmass, NULL); + if (isnan(Mgrp)) continue; + float Mgrid = getMgrid (m, i); + if (isnan(Mgrid)) continue; + + // Note that this operation is setting measure->McalAPER to image->McalAPER only + // for the stacks. Elsewhere (setMrelCatalog) we are using image->McalPSF for all + // types of measurements EXCEPT stacks. + + // we need to use McalAPER for stacks (and only stacks) + int useStackAper = !USE_MCAL_PSF_FOR_STACK_APER && isGPC1stack(catalog[i].measureT[m].photcode); + + // note that measurements for which the image is not selected will not be modified + // (that is a good thing! -- we keep a prior calibration) + + // set the output calibration + // float Moff = Mmos + Mgrp + Mgrid; + float Moff = Mmos + Mgrid; // Mgrp has already been applied to the image.Mcal + catalog[i].measure[m].McalPSF = McalPSF + Moff; + catalog[i].measure[m].McalAPER = useStackAper ? McalAPER + Moff : McalPSF + Moff; + + if (catalog[i].measureT[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { + myAssert (isfinite(catalog[i].measure[m].McalPSF), "oops, broke an ubercal mag"); + } + } + } + } + return (TRUE); +} + +int markObjects (Catalog *catalog, int Ncatalog) { + + int i, n; + off_t j; + + // How strongly do I own this object? + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + int nOwn = 0; + int m = catalog[i].averageT[j].measureOffset; + for (n = 0; n < catalog[i].averageT[j].Nmeasure; n++) { + if (!catalog[i].measureT[m+n].myDet) continue; + nOwn ++; + } + catalog[i].averageT[j].nOwn = nOwn; + } + } + return TRUE; +} + +int dumpObjects (char *filename, Catalog *catalog, int Ncatalog) { + + int i, n; + off_t j; + + FILE *ftest = fopen (filename, "w"); + + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + int m = catalog[i].averageT[j].measureOffset; + for (n = 0; n < catalog[i].averageT[j].Nmeasure; n++) { + fprintf (ftest, "%08x %08x %10.6f %10.6f %3d %1d\n", catalog[i].averageT[j].catID, catalog[i].averageT[j].objID, catalog[i].averageT[j].R, catalog[i].averageT[j].D, catalog[i].averageT[j].Nmeasure, catalog[i].measureT[m+n].myDet); + } + } + } + fclose (ftest); + return TRUE; +} + +int dumpMags (FILE *fout, Catalog *catalog, int Ncatalog) { + + int i, n; + off_t j; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + for (i = 0; i < Ncatalog; i++) { + for (j = 0; (j < catalog[i].Naverage) && (j < 2); j++) { + fprintf (fout, "%08x %08x %10.6f %10.6f : \n", catalog[i].averageT[j].catID, catalog[i].averageT[j].objID, catalog[i].averageT[j].R, catalog[i].averageT[j].D); + for (n = 0; n < 5; n++) { + fprintf (fout, "secf %5d | %6.3f %6.3f | %6.3f %6.3f | %6.3f %6.3f\n", n, catalog[i].secfilt[j*Nsecfilt + n].MpsfChp, catalog[i].secfilt[j*Nsecfilt + n].MkronChp, catalog[i].secfilt[j*Nsecfilt + n].MpsfStk, catalog[i].secfilt[j*Nsecfilt + n].MkronStk, catalog[i].secfilt[j*Nsecfilt + n].MpsfWrp, catalog[i].secfilt[j*Nsecfilt + n].MkronWrp); + } + int m = catalog[i].averageT[j].measureOffset; + for (n = 0; n < catalog[i].averageT[j].Nmeasure; n++) { + fprintf (fout, "meas %5d %5d | %6.3f %6.3f | %6.3f %6.3f\n", m+n, catalog[i].measureT[m+n].photcode, catalog[i].measureT[m+n].M, catalog[i].measureT[m+n].Mkron, catalog[i].measureT[m+n].McalPSF, catalog[i].measureT[m+n].McalAPER); + } + } + } + return TRUE; +} + +void clean_stars (Catalog *catalog, int Ncatalog) { + + int i, j, Ndel, Nave, Ntot, mark, Ns, Nscat, Nchi, Nnan; + float dM; + double MaxScatter, MaxChisq; + double *xlist, *slist, *dlist; + + StatType stats; + liststats_setmode (&stats, "MEAN"); + + if (VERBOSE) fprintf (stderr, "marking poor stars\n"); + + /* find Mchisq median -> ChiSq lim must be > median */ + for (i = Ntot = 0; i < Ncatalog; i++) { + Ntot += catalog[i].Naverage; + } + ALLOCATE (xlist, double, Ntot); + ALLOCATE (slist, double, Ntot); + ALLOCATE (dlist, double, Ntot); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + // eliminate bad stars using the stats for a single secfilt at a time + for (Ns = 0; Ns < Nphotcodes; Ns ++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + for (i = Ntot = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + if ( catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD ) continue; + float Mchisq = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; + if (isnan(Mchisq)) continue; + xlist[Ntot] = Mchisq; + slist[Ntot] = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; + dlist[Ntot] = 1; + Ntot ++; + } + } + + liststats (xlist, dlist, NULL, Ntot, &stats); + MaxChisq = MAX (STAR_CHISQ, 2*stats.median); + + liststats (slist, dlist, NULL, Ntot, &stats); + MaxScatter = MAX (STAR_SCATTER, 2*stats.median); + fprintf (stderr, "Max Scatter: %f, Max Chisq: %f\n", MaxScatter, MaxChisq); + + Ndel = Nave = Nscat = Nnan = Nchi = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; + float Mchisq = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; + mark = (dM > MaxScatter) || (isnan(Mchisq)) || (Mchisq > MaxChisq); + if (mark) { + catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_OBJ_POOR; + Ndel ++; + if (dM > MaxScatter) { Nscat ++; } + if (isnan(Mchisq)) { Nnan ++; } + if (Mchisq > MaxChisq) { Nchi ++; } + } else { + catalog[i].secfilt[Nsecfilt*j+Nsec].flags &= ~ID_OBJ_POOR; + } + Nave ++; + } + } + fprintf (stderr, "%d stars marked variable (%d scat, %d nan, %d chi), %d total\n", Ndel, Nscat, Nnan, Nchi, Nave); + } + free (xlist); + free (slist); + free (dlist); +} + +// clean_measures examines the stats for a single star. It measures the INNER 50% mean +// and sigma, it then re-measures the mean and sigma using all stars with NSIGMA_CLIP (3) +// sigma of the INNER 50% mean. it then flags any measurements which are more than +// NSIGMA_REJECT (5) sigma of the mean + +// this function only operations on the PSF magnitudes + +// XXX EAM clean_measures is no longer used -- we rely on IRLS to down-weight outliers + +# define NSIGMA_CLIP 3.0 +# define NSIGMA_REJECT 5.0 +void clean_measures (Catalog *catalog, int Ncatalog, int final, FlatCorrectionTable *flatcorr) { + + off_t j, k, m, Nmax, Ndel, Nave; + int i, N, image_bad, TOOFEW; + off_t *ilist; + double *tlist, *list, *dlist; + float Msys, Mcal, Mmos, Mgrp, Mgrid; + int Ncal, Nmos, Ngrp, Ngrid, Nfew; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + if (VERBOSE) fprintf (stderr, "marking poor measures\n"); + /* Nmeasure is now different, need to reallocate */ + Nmax = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + Nmax = MAX (Nmax, catalog[i].averageT[j].Nmeasure); + } + } + ALLOCATE (list, double, MAX (1, Nmax)); + ALLOCATE (dlist, double, MAX (1, Nmax)); + ALLOCATE (ilist, off_t, Nmax); + ALLOCATE (tlist, double, Nmax); + + /* it makes no sense to mark 3-sigma outliers with <5 measurements */ + TOOFEW = MAX (5, STAR_TOOFEW); + + // stats structures for inner and full stats + StatType instats, stats; + liststats_setmode (&instats, "INNER_MEAN"); + liststats_setmode (&stats, "MEAN"); + + Ndel = Nave = 0; + Ncal = Nmos = Ngrp = Ngrid = Nfew = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + int Ns; + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + /* on final processing, skip stars already measured */ + if (final && catalog[i].found_t[Nsecfilt*j + Nsec]) continue; + + /* skip bad stars to prevent them from becoming good (on inner sample) */ + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + + /* accumulate list of valid measurements */ + m = catalog[i].averageT[j].measureOffset; + N = 0; + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + // skip measurements that do not match the current photcode + int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); + if (ecode != thisCode) { continue; } + + // SKIP gpc1 stack data + if (isGPC1stack(catalog[i].measureT[m].photcode)) continue; + + // SKIP gpc1 forced-warp data + if (isGPC1warp(catalog[i].measureT[m].photcode)) continue; + + // NOTE: we do not skip MEAS_BAD because this measurement is just an internal assessment of the outliers + Mcal = getMcal (m, i, MAG_CLASS_PSF); + if (isnan(Mcal)) { Ncal ++; continue; } + Mmos = getMmos (m, i); + if (isnan(Mmos)) { Nmos ++; continue; } + Mgrp = getMgrp (m, i, catalog[i].measureT[m].airmass, NULL); + if (isnan(Mgrp)) { Ngrp ++; continue; } + Mgrid = getMgrid (m, i); + if (isnan(Mgrid)) { Ngrid ++; continue; } + + Msys = PhotSysTiny (&catalog[i].measureT[m], &catalog[i].averageT[j], &catalog[i].secfilt[j*Nsecfilt], MAG_CLASS_PSF); + list[N] = Msys - Mcal - Mmos - Mgrp - Mgrid; + dlist[N] = MAX (catalog[i].measureT[m].dM, MIN_ERROR); + N++; + } + if (N <= TOOFEW) { + Nfew ++; + continue; + } + + /* 3-sigma clip based on stats of inner 50% */ + + // calculated mean of inner 50% + liststats (list, dlist, NULL, N, &instats); + instats.sigma = MAX (MIN_ERROR, instats.sigma); /* if measurements agree too well, sigma -> 0.0 */ + + // ignore entries > 3sigma from inner mean + for (k = m = 0; k < N; k++) { + if (fabs (list[k] - instats.median) < NSIGMA_CLIP*instats.sigma) { + list[m] = list[k]; + m++; + } + } + // recalculate the mean & sigma of the accepted measurements + liststats (list, dlist, NULL, m, &stats); + stats.sigma = MAX (MIN_ERROR, stats.sigma); + + /* apply to list of all relevant measurements, including IMAGE_POOR & IMAGE_FEW */ + image_bad = IMAGE_BAD; + IMAGE_BAD = ID_IMAGE_PHOTOM_NOCAL; + m = catalog[i].averageT[j].measureOffset; + N = 0; + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + // skip measurements that do not match the current photcode + int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); + if (ecode != thisCode) { continue; } + + // SKIP gpc1 stack data + if (isGPC1stack(catalog[i].measureT[m].photcode)) continue; + + // SKIP gpc1 forced-warp data + if (isGPC1warp(catalog[i].measureT[m].photcode)) continue; + + // NOTE: we do not skip MEAS_BAD because this measurement is just an internal assessment of the outliers + Mcal = getMcal (m, i, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + Mmos = getMmos (m, i); + if (isnan(Mmos)) continue; + Mgrid = getMgrid (m, i); + if (isnan(Mgrid)) continue; + + Msys = PhotSysTiny (&catalog[i].measureT[m], &catalog[i].averageT[j], &catalog[i].secfilt[j*Nsecfilt], MAG_CLASS_PSF); + list[N] = Msys - Mcal - Mmos - Mgrid; + dlist[N] = MAX (catalog[i].measureT[m].dM, MIN_ERROR); + ilist[N] = m; + N++; + Nave ++; + } + if (N < TOOFEW) continue; + + /* mark bad measures (> 5 sigma deviant) */ + for (k = 0; k < N; k++) { + // treat the scatter of the star as a systematic term. this is a bit of an + // over-estimage (a perfect Gauss distribution with perfect errors would have + // mySigma = sqrt(2) too large) + float mySigma = hypot (stats.sigma, dlist[k]); + if (fabs (list[k] - stats.median) > NSIGMA_REJECT*mySigma) { + catalog[i].measureT[ilist[k]].dbFlags |= ID_MEAS_POOR_PHOTOM; + if (final) { + // for the final pass, we have a duplicate set of values in measure and measureT + catalog[i].measure[ilist[k]].dbFlags |= ID_MEAS_POOR_PHOTOM; + } + Ndel ++; + } else { + catalog[i].measureT[ilist[k]].dbFlags &= ~ID_MEAS_POOR_PHOTOM; + if (final) { + // for the final pass, we have a duplicate set of values in measure and measureT + catalog[i].measure[ilist[k]].dbFlags &= ~ID_MEAS_POOR_PHOTOM; + } + } + } + IMAGE_BAD = image_bad; + } + } + } + if (VERBOSE) fprintf (stderr, OFF_T_FMT" measures marked poor, "OFF_T_FMT" total\n", Ndel, Nave); + + free (list); + free (dlist); + free (ilist); + free (tlist); +} + +StatType statsStarN (Catalog *catalog, int Ncatalog, int Nsec, int seccode, FlatCorrectionTable *flatcorr) { + + off_t j, k, m, Ntot; + int i, n, N; + double *list, *dlist; + float Mcal, Mmos, Mgrid; + StatType stats; + // int N1, N2, N3, N4, N0; + // N1 = N2 = N3 = N4 = N0 = 0; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Ntot = 0; + for (i = 0; i < Ncatalog; i++) { + Ntot += catalog[i].Naverage; + } + + ALLOCATE (list, double, Ntot); + ALLOCATE (dlist, double, Ntot); + + n = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + /* calculate the average value for a single star */ + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) { continue; } + m = catalog[i].averageT[j].measureOffset; + + N = 0; + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); + if (ecode != seccode) { continue;} + Mcal = getMcal (m, i, MAG_CLASS_PSF); + if (isnan(Mcal)) { continue;} + Mmos = getMmos (m, i); + if (isnan(Mmos)) { continue; } + Mgrid = getMgrid (m, i); + if (isnan(Mgrid)) { continue;} + N++; + } + + list[n] = N; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +// stats for a single secfilt at a time +StatType statsStarX (Catalog *catalog, int Ncatalog, int Nsec) { + + off_t j, Ntot; + int i, n; + double *list, *dlist; + StatType stats; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Ntot = 0; + for (i = 0; i < Ncatalog; i++) { + Ntot += catalog[i].Naverage; + } + + ALLOCATE (list, double, Ntot); + ALLOCATE (dlist, double, Ntot); + + n = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + /* calculate the average value for a single star */ + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + + float Mchisq = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; + if (isnan(Mchisq)) continue; + list[n] = Mchisq; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsStarS (Catalog *catalog, int Ncatalog, int Nsec) { + + int i, n; + off_t j, Ntot; + double *list, *dlist; + float dM; + StatType stats; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Ntot = 0; + for (i = 0; i < Ncatalog; i++) { + Ntot += catalog[i].Naverage; + } + + ALLOCATE (list, double, Ntot); + ALLOCATE (dlist, double, Ntot); + + n = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + /* calculate the average value for a single star */ + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + + dM = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; + if (isnan(dM)) continue; + list[n] = dM; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +void plot_stars (Catalog *catalog, int Ncatalog) { + + int i, bin; + off_t j; + float dMrel; + double *xlist, *Mlist; + Graphdata graphdata; + +# define NBIN 200 + ALLOCATE (xlist, double, NBIN); + ALLOCATE (Mlist, double, NBIN); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + int Ns; + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + for (i = 0; i < NBIN; i++) xlist[i] = 0.00025*i; + bzero (Mlist, NBIN*sizeof(double)); + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + dMrel = catalog[i].secfilt[Nsecfilt*j+Nsec].dMpsfChp; + bin = dMrel / 0.00025; + bin = MAX (0, MIN (NBIN-1, bin)); + Mlist[bin] += 1.0; + } + } + + plot_defaults (&graphdata); + graphdata.style = 1; + plot_list (&graphdata, xlist, Mlist, NBIN, "dMrel hist", "%s.dMhist.png", OUTROOT); + } + free (xlist); + free (Mlist); +} + +void plot_chisq (Catalog *catalog, int Ncatalog) { + + off_t j, Ntotal; + int i, N, value; + double *xlist, *ylist; + Graphdata graphdata; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Ntotal = 0; + for (i = 0; i < Ncatalog; i++) Ntotal += catalog[i].Naverage; + + ALLOCATE (xlist, double, Ntotal); + ALLOCATE (ylist, double, Ntotal); + + int Ns; + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + N = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + xlist[N] = catalog[i].secfilt[Nsecfilt*j+Nsec].MpsfChp; + value = catalog[i].secfilt[Nsecfilt*j+Nsec].Mchisq; + if (isnan((double)(value))) continue; + ylist[N] = value; + N++; + } + } + + plot_defaults (&graphdata); + graphdata.ymin = -3.0; + plot_list (&graphdata, xlist, ylist, N, "chisq", "%s.chisq.png", OUTROOT); + } + + free (xlist); + free (ylist); +} + +void plot_star_coords (Catalog *catalog, int Ncatalog) { + + int i; + off_t j, N; + double *xlist, *ylist; + // double Xmin, Ymin, Xmax, Ymax; + Graphdata graphdata; + + N = 0; + for (i = 0; i < Ncatalog; i++) { + N += catalog[i].Naverage; + } + ALLOCATE (xlist, double, N); + ALLOCATE (ylist, double, N); + + N = 0; + // Xmin = Ymin = +360.0; + // Xmax = Ymax = -360.0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + xlist[N] = catalog[i].averageT[j].R; + ylist[N] = catalog[i].averageT[j].D; + N++; + } + } + plot_defaults (&graphdata); + plot_list (&graphdata, xlist, ylist, N, "coords", "%s.coords.png", OUTROOT); + + free (xlist); + free (ylist); +} + +void dump_catalog (Catalog *catalog, off_t c, int Npass) { + + return; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + FILE *fout = NULL; + char filename[64]; + snprintf (filename, 64, "tcat.%02d.dat", Npass); + fout = fopen (filename, "w"); + + for (off_t n = 0; n < catalog[c].Naverage; n++) { + + off_t m = catalog[c].averageT[n].measureOffset; + for (off_t k = 0; k < catalog[c].averageT[n].Nmeasure; k++, m++) { + + float Mcal = getMcal (m, c, MAG_CLASS_PSF); // image zero point + float Mmos = getMmos (m, c); // mosaic zero point + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); // average magnitude + // float Mgrid = getMgrid (m, c); // camera offset (deprecated?) + // float Mflat = getMflat (m, c, flatcorr, catalog); // flat-field correction + + off_t n = catalog[c].measureT[m].averef; + + // nominal magnitude for this measurement (instrumental + Ko(airmass - 1) + Co + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + // for historical reasons, Mflat is defined with the wrong sign + float delta = MsysPSF - MrelPSF - Mcal - Mmos - Mgrp; + + fprintf (fout, "%f %f : %f %f : %f %f %f : %f %f\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysPSF, MrelPSF, Mcal, Mmos, Mgrp, catalog[c].measureT[m].airmass, delta); + } + } + fclose (fout); +} Index: /branches/eam_branches/relphot.20210521/src/StatDataSetOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/StatDataSetOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/StatDataSetOps.c (revision 41602) @@ -0,0 +1,57 @@ +# include "relphot.h" + +StatDataSet *StatDataSetAlloc (int Nsecfilt, int Nmax) { + + int i; + + StatDataSet *dataset = NULL; + ALLOCATE (dataset, StatDataSet, Nsecfilt); + for (i = 0; i < Nsecfilt; i++) { + ALLOCATE (dataset[i].flxlist, double, Nmax); + ALLOCATE (dataset[i].wgtlist, double, Nmax); + ALLOCATE (dataset[i].errlist, double, Nmax); + ALLOCATE (dataset[i].ranking, int, Nmax); + ALLOCATE (dataset[i].measSeq, int, Nmax); + ALLOCATE (dataset[i].msklist, int, Nmax); + + ALLOCATE (dataset[i].values, double, Nmax); + ALLOCATE (dataset[i].wtvals, double, Nmax); + ALLOCATE (dataset[i].wtlist, double, Nmax); + ALLOCATE (dataset[i].ykeep, double, Nmax); + ALLOCATE (dataset[i].dykeep, double, Nmax); + ALLOCATE (dataset[i].wtkeep, double, Nmax); + ALLOCATE (dataset[i].ysample, double, Nmax); + ALLOCATE (dataset[i].dysample, double, Nmax); + ALLOCATE (dataset[i].wtsample, double, Nmax); + ALLOCATE (dataset[i].bvalue, double, NBOOTSTRAP); + } + + return dataset; +} + +void StatDataSetFree (StatDataSet *dataset, int Nsecfilt) { + + int i; + + for (i = 0; i < Nsecfilt; i++) { + FREE (dataset[i].flxlist); + FREE (dataset[i].wgtlist); + FREE (dataset[i].errlist); + FREE (dataset[i].ranking); + FREE (dataset[i].measSeq); + FREE (dataset[i].msklist); + + FREE (dataset[i].values); + FREE (dataset[i].wtvals); + FREE (dataset[i].wtlist); + FREE (dataset[i].ykeep); + FREE (dataset[i].dykeep); + FREE (dataset[i].wtkeep); + FREE (dataset[i].ysample); + FREE (dataset[i].dysample); + FREE (dataset[i].wtsample); + FREE (dataset[i].bvalue); + } + FREE (dataset); +} + Index: /branches/eam_branches/relphot.20210521/src/TGroupOps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/TGroupOps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/TGroupOps.c (revision 41602) @@ -0,0 +1,1594 @@ +# include "relphot.h" + +void assignMosaicsToTGroups (void); +int save_test_night_measures (FILE *fout, TGroup *myTGroup, Catalog *catalog, FlatCorrectionTable *flatcorr); + +// tgroupTimes carries the times of the tgroups (initially, just the photometric nights) +static int NtgroupTimes = 0; +static int NTGROUPTIMES = 0; +static TGTimes **tgroupTimes = NULL; + +// TGroup pointers for each image +static TGroup **ImageToTGroup = NULL; + +// elsewhere, we have loaded a set of catalogs with measures (catalog[cat].measure[meas]) + +// TGroup pointers for each measure,catalog set +static TGroup ***MeasureToTGroup = NULL; + +void sort_times (unsigned int *T, int N); + +void sort_tgtimes (TGTimes **T, int N) { + +# define SWAPFUNC(A,B){ TGTimes *tmp; \ + tmp = T[A]; T[A] = T[B]; T[B] = tmp; \ + } +# define COMPARE(A,B)(T[A][0].start < T[B][0].start) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +// load the photometric nights from a file : these will be used to define the tgroups +// this is called in args.c after TGROUP_ZEROPT is set +void loadTGroups (char *filename) { + + if (!TGROUP_ZEROPT) return; + + // read start times from a file (currently a list of MJD values) + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "tgroup file %s not found\n", filename); + exit (4); + } + // ReadTGroupFile returns a list of the time values (int) + unsigned int *tgroupTimesRaw = ReadTGroupFile (f, &NtgroupTimes); + if (!tgroupTimesRaw) { + fprintf (stderr, "error reading tgroup file %s\n", filename); + exit (4); + } + fprintf (stderr, "loaded %d tgroups from file %s\n", NtgroupTimes, filename); + + // sort the times for quick bisection + sort_times (tgroupTimesRaw, NtgroupTimes); + + // we first define the collection of times (TGTimes) then for each time, we define a tgroup for + // each photcode being considered below, if there are no images in this tgroup with a + // given photcode, it is left empty (no images) + + NTGROUPTIMES = NtgroupTimes; // if we have loaded a list of times, use that as the starting alloc number + ALLOCATE (tgroupTimes, TGTimes *, NTGROUPTIMES); + + // the start of an MJD day is 14:00 HST, so the end time can be the start time + 23h 59m 59s (86399s) + // XXX NOTE that this is HST-centric + + for (int i = 0; i < NtgroupTimes; i++) { + ALLOCATE (tgroupTimes[i], TGTimes, 1); + tgroupTimes[i][0].start = tgroupTimesRaw[i]; + tgroupTimes[i][0].stop = tgroupTimesRaw[i] + 86399; + + // One TGroup per *active* photcode. Check if photcode is active with: + // GetActivePhotcodeIndex (photcode) (returns -1 if not active) + + ALLOCATE (tgroupTimes[i][0].byCode, TGroup, Nphotcodes); + tgroupTimes[i][0].nCode = Nphotcodes; + TGroup *tgroup = tgroupTimes[i][0].byCode; + + for (int j = 0; j < Nphotcodes; j++) { + tgroup[j].parent = (void *) tgroupTimes[i]; + tgroup[j].McalPSF = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].McalAPER = 0.0; // note : tgroup stores only offsets relative to the original image values + tgroup[j].dKlam = 0.0; + tgroup[j].dMcal = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].dMsys = 0.0; + tgroup[j].dMmin = NAN; + tgroup[j].dMmax = NAN; + tgroup[j].McalChiSq = 0.0; // NAN or 0.0? + + tgroup[j].stdev = NAN; + tgroup[j].nFitPhotom = 0; + tgroup[j].nValPhotom = 0; + + tgroup[j].flags = ID_IMAGE_PHOTOM_UBERCAL; // we start by treating the suspected photometric nights as photometric + tgroup[j].photcode = photcodes[j][0].code; + + tgroup[j].NIMAGE = 1000; + tgroup[j].Nimage = 0; + ALLOCATE (tgroup[j].image, off_t, tgroup[j].NIMAGE); + tgroup[j].image[0] = -1; + + tgroup[j].NMOSAIC = 100; + tgroup[j].Nmosaic = 0; + ALLOCATE (tgroup[j].mosaic, Mosaic *, tgroup[j].NMOSAIC); + + tgroup[j].Nmeasure = 0; + tgroup[j].NMEASURE = 1000; + ALLOCATE (tgroup[j].measure, off_t, tgroup[j].NMEASURE); + ALLOCATE (tgroup[j].catalog, off_t, tgroup[j].NMEASURE); + } + } + + free (tgroupTimesRaw); + return; +} + +// create a new TGroup based on the supplied time and photcode +void extendTGroups (unsigned int tzero) { + + if (!TGROUP_ZEROPT) return; + + // index of the new entry + int Ntgt = NtgroupTimes; + NtgroupTimes ++; + + if (NtgroupTimes >= NTGROUPTIMES) { + NTGROUPTIMES += 100; + REALLOCATE (tgroupTimes, TGTimes *, NTGROUPTIMES); + } + + double mjd = ohana_sec_to_mjd (tzero); + double mjdInt = (int) mjd; + time_t start = ohana_mjd_to_sec (mjdInt); + + ALLOCATE (tgroupTimes[Ntgt], TGTimes, 1); + tgroupTimes[Ntgt][0].start = start; + tgroupTimes[Ntgt][0].stop = start + 86399; + + ALLOCATE (tgroupTimes[Ntgt][0].byCode, TGroup, Nphotcodes); + tgroupTimes[Ntgt][0].nCode = Nphotcodes; + TGroup *tgroup = tgroupTimes[Ntgt][0].byCode; + + for (int j = 0; j < Nphotcodes; j++) { + tgroup[j].parent = (void *) tgroupTimes[Ntgt]; + tgroup[j].McalPSF = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].McalAPER = 0.0; // note : tgroup stores only offsets relative to the original image values + tgroup[j].dKlam = 0.0; + tgroup[j].dMcal = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].dMsys = 0.0; + tgroup[j].dMmin = NAN; + tgroup[j].dMmax = NAN; + tgroup[j].McalChiSq = 0.0; // NAN or 0.0? + + tgroup[j].stdev = NAN; + tgroup[j].nFitPhotom = 0; + tgroup[j].nValPhotom = 0; + + tgroup[j].flags = 0; // tgroups NOT in the original file are not suspected as photometric + tgroup[j].photcode = photcodes[j][0].code; + + tgroup[j].NIMAGE = 1000; + tgroup[j].Nimage = 0; // the new TGroup is not suspected to be photometric + ALLOCATE (tgroup[j].image, off_t, tgroup[j].NIMAGE); + tgroup[j].image[0] = -1; + + tgroup[j].NMOSAIC = 100; + tgroup[j].Nmosaic = 0; + ALLOCATE (tgroup[j].mosaic, Mosaic *, tgroup[j].NMOSAIC); + + tgroup[j].Nmeasure = 0; + tgroup[j].NMEASURE = 1000; + ALLOCATE (tgroup[j].measure, off_t, tgroup[j].NMEASURE); + ALLOCATE (tgroup[j].catalog, off_t, tgroup[j].NMEASURE); + } + + // sort the times for quick bisection + sort_tgtimes (tgroupTimes, NtgroupTimes); + return; +} + +// Assign the images to the tgroups. The initial tgroups table contains only suspected +// photometric nights. As we find images which are not in the current tgroups list, add +// new non-photometric entries. +void initTGroups (Image *subset, off_t Nsubset) { + + if (!TGROUP_ZEROPT) return; + + /* a 'tgroup' in relphot is a virtual concept: there is no + * entry in the image table that represents this tgroup. Instead, it is an + * internal construct that defines a group of related images + */ + + if (!tgroupTimes) { + NTGROUPTIMES = 100; + ALLOCATE (tgroupTimes, TGTimes *, NTGROUPTIMES); + } + + ALLOCATE (ImageToTGroup, TGroup *, Nsubset); // tgroup to which image belongs + + // assign each image to a tgroup + int Nsimple = 0; + for (off_t i = 0; i < Nsubset; i++) { + ImageToTGroup[i] = NULL; + + // ignore mosaic images (photcode == 0) + if (!subset[i].photcode) { + continue; + } + + // skip inactive photcodes (do not assign to a TGroup) + if (GetActivePhotcodeIndex (subset[i].photcode) < 0) continue; + + TGroup *myGroup = findTGroup(subset[i].tzero, subset[i].photcode); + if (!myGroup) { + // generate a new TGroup which is not photometric + extendTGroups (subset[i].tzero); + myGroup = findTGroup(subset[i].tzero, subset[i].photcode); + myAssert (myGroup, "oops, we just extended to include this"); + } + + // add pointer to tgroup for this image + ImageToTGroup[i] = myGroup; + + // note the array 'subset' is registered in ImageOps.c:images by initImages + // and can be retrieve with getimages + + // add image to tgroup image list + myGroup[0].image[myGroup[0].Nimage] = i; // reference to entry in 'subset' + myGroup[0].Nimage ++; + if (myGroup[0].Nimage == myGroup[0].NIMAGE) { + myGroup[0].NIMAGE += 1000; + REALLOCATE (myGroup[0].image, off_t, myGroup[0].NIMAGE); + } + } + + // assign the mosaics to each tgroup -- we do this by image + assignMosaicsToTGroups (); + + initTGroupsMcal (); + + fprintf (stderr, "matched %d images to %d tgroups, %d simple chips not matched to tgroups\n", (int) (Nsubset - Nsimple), (int) NtgroupTimes, (int) Nsimple); + return; +} + +void freeTGroups (void) { + + if (!TGROUP_ZEROPT) return; + + free (ImageToTGroup); + + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + free (tgroup[j].image); + free (tgroup[j].mosaic); + free (tgroup[j].measure); + free (tgroup[j].catalog); + } + free (tgroup); + free (tgroupTimes[i]); + } + free (tgroupTimes); + + // free (ImageToGroup); why not? + return; +} + +void initTGroupsMcal (void) { + + off_t Nimages; + off_t *LineNumber; + Image *images = getimages (&Nimages, &LineNumber); + + if (!TGROUP_ZEROPT) return; + + // we use liststats to calculate the median Mcal for the input images + // in each tgroup as a starting point. + StatType stats; + liststats_setmode (&stats, "MEDIAN"); + + // init the tgroup array values + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + tgroup[j].McalPSF = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].McalAPER = 0.0; // note : tgroup stores only offsets relative to the original image values + tgroup[j].dMcal = 0.0; // note : at the end, tgroup.Mcal is added back to the input images + tgroup[j].dMsys = 0.0; + tgroup[j].McalChiSq = 0.0;// NAN or 0.0? + + tgroup[j].stdev = NAN; + tgroup[j].nFitPhotom = 0; + tgroup[j].nValPhotom = 0; + + if (tgroup[j].Nimage == 0) continue; // no images, ignore the tgroup + + // calculate the median of the Mcal values for the images in this tgroup: + + ALLOCATE_PTR (McalPSF, double, tgroup[j].Nimage); + ALLOCATE_PTR (McalAPER, double, tgroup[j].Nimage); + for (int im = 0; im < tgroup[j].Nimage; im++) { + int Nsub = tgroup[j].image[im]; + McalPSF[im] = images[Nsub].McalPSF; + McalAPER[im] = images[Nsub].McalAPER; + + // set these back to 0.0 to have no future effect + images[Nsub].McalPSF = 0.0; + images[Nsub].McalAPER = 0.0; + } + + liststats (McalPSF, NULL, NULL, tgroup[j].Nimage, &stats); + tgroup[j].McalPSF = stats.median; + FREE (McalPSF); + + liststats (McalAPER, NULL, NULL, tgroup[j].Nimage, &stats); + tgroup[j].McalAPER = stats.median; + FREE (McalAPER); + } + } + return; +} + +void assignMosaicsToTGroups (void) { + + if (!TGROUP_ZEROPT) return; + + // init the tgroup array values + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + + if (tgroup[j].Nimage == 0) continue; // no images, ignore the tgroup + + // find the mosaics by image (if MOSAIC_ZEROPT is active) + for (int im = 0; MOSAIC_ZEROPT && (im < tgroup[j].Nimage); im++) { + int imageIdx = tgroup[j].image[im]; + Mosaic *mosaic = getMosaicForImage (imageIdx); + + if (!mosaic) continue; // this is a simple chip, not part of a mosaic + if (mosaic->inTGroup) continue; + + // mosaic to tgroup and extend as needed + tgroup[j].mosaic[tgroup[j].Nmosaic] = mosaic; + mosaic->inTGroup = TRUE; + + // advance array and extend if needed + tgroup[j].Nmosaic ++; + if (tgroup[j].Nmosaic >= tgroup[j].NMOSAIC) { + tgroup[j].NMOSAIC += 100; + REALLOCATE (tgroup[j].mosaic, Mosaic *, tgroup[j].NMOSAIC); + } + } + } + } + return; +} + +TGroup *getTGroupForImage (off_t im) { + + if (im < 0) return NULL; + if (!ImageToTGroup) return NULL; + + // test if im > Nimages / Nsubset? + TGroup *myGroup = ImageToTGroup[im]; + return (myGroup); +} + +// use bisection to find the overlapping tgroup (returns exact match) +// tgroupTimes is a sorted, unique list of times +// assume a 1day range for now +TGroup *findTGroup (unsigned int start, int photcode) { + + if (!NtgroupTimes) return NULL; // if none are yet defined, do not even try + + // find index of matching photcode from our list of active secondary photcodes + int Ns = GetActivePhotcodeIndex (photcode); + if (Ns < 0) return NULL; + + off_t Nlo, Nhi, N; + + // find the last tgroup before start + Nlo = 0; // first valid tgroupTimes value + Nhi = NtgroupTimes - 1; // last valid tgroupTimes value + + // if start is not in this range, return NULL + if (start < tgroupTimes[Nlo][0].start) return NULL; + if (start > tgroupTimes[Nhi][0].start + 86399) return NULL; // 1-day range for tgroup for now + + while (Nhi - Nlo > 4) { + N = 0.5*(Nlo + Nhi); + if (tgroupTimes[N][0].start < start) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N, NtgroupTimes - 1); + } + } + // we now have : tgroupTimes[Nlo] < start <= tgroupTimes[Nhi] + + // find a matched tgroup starting from Nlo, or return NULL + // tgroupTimes is a lower bound, the upper bound is tgroupTimes + 86399 (1d) + for (N = Nlo; N <= Nhi; N++) { + if (start < tgroupTimes[N][0].start) continue; + if (start > tgroupTimes[N][0].start + 86399) continue; // XXX since they are sorted, probably do not need this check + // N is the tgroupTimes entry we want. + // double-check that tgroupTimes[N][0].byCode[Ns].photcode = ecode? + return &tgroupTimes[N][0].byCode[Ns]; + } + return (NULL); +} + +void setMcalFromTGroups () { + + off_t Nimage; + Image *image; + + if (!TGROUP_ZEROPT) return; + + image = getimages (&Nimage, NULL); + + fprintf (stderr, "*** return Mcal from tgroup.Mcal to image.Mcal ***\n"); + + // NOTE: nights which are poor should have had Mcal set to 0, making this a NOP for those nights + + // copy the tgroup results to the images. set the tgroup Mcal to 0.0 since we have moved its + // impact to the images + for (off_t k = 0; k < NtgroupTimes; k++) { + TGroup *tgroup = tgroupTimes[k][0].byCode; + for (off_t i = 0; i < tgroupTimes[k][0].nCode; i++) { + for (off_t j = 0; j < tgroup[i].Nimage; j++) { + off_t im = tgroup[i].image[j]; + double Mgrp = (TGROUP_FIT_AIRMASS) ? tgroup[i].McalPSF + tgroup[i].dKlam*(image[im].secz - 1.0) : tgroup[i].McalPSF; + if (tgroup[i].flags & ID_IMAGE_TGROUP_PHOTCAL) { + image[im].McalPSF = Mgrp; + image[im].McalAPER = Mgrp; + image[im].dMcal = tgroup[i].dMcal; + image[im].McalChiSq = tgroup[i].McalChiSq; + image[im].dMagSys = tgroup[i].dMsys; + image[im].nFitPhotom = tgroup[i].nFitPhotom; + } + image[im].flags |= (tgroup[i].flags & ID_IMAGE_NIGHT_POOR); // probably not necessary : it is set in markBadTGroup + image[im].flags |= (tgroup[i].flags & ID_IMAGE_MOSAIC_POOR); // probably not necessary : it is set in markBadTGroup + image[im].flags |= (tgroup[i].flags & ID_IMAGE_PHOTOM_FEW); + image[im].flags |= (tgroup[i].flags & ID_IMAGE_PHOTOM_POOR); + image[im].flags |= (tgroup[i].flags & ID_IMAGE_PHOTOM_UBERCAL); + image[im].flags |= (tgroup[i].flags & ID_IMAGE_TGROUP_PHOTCAL); + image[im].flags |= (tgroup[i].flags & ID_IMAGE_MOSAIC_PHOTCAL); + image[im].flags |= (tgroup[i].flags & ID_IMAGE_IMAGE_PHOTCAL); + + // fprintf (stderr, "TG to IMAGE: %f +/- %f -> %f (%d)\n", tgroup[i].McalPSF, tgroup[i].dMcal, image[im].McalPSF, tgroup[i].nFitPhotom); + } + tgroup[i].McalPSF = 0.0; + tgroup[i].McalAPER = 0.0; + tgroup[i].dKlam = 0.0; + } + } +} + +void markBadTGroup (TGroup *tgroup) { + + off_t Nimage; + Image *image; + + // this tgroup has been identified as poor. + tgroup->flags |= ID_IMAGE_NIGHT_POOR; + + // all associated mosaics should be marked as coming from a bad night + // these will be fitted independently. Make the starting solution consistent by + // setting the mosaic McalPSF for the current best fit from the tgroup + + for (off_t j = 0; j < tgroup->Nmosaic; j++) { + Mosaic *mos = tgroup->mosaic[j]; + mos->flags |= ID_IMAGE_NIGHT_POOR; + mos->McalPSF = (TGROUP_FIT_AIRMASS) ? tgroup->McalPSF + tgroup->dKlam*(mos->secz - 1.0) : tgroup->McalPSF; + mos->McalAPER = (TGROUP_FIT_AIRMASS) ? tgroup->McalAPER + tgroup->dKlam*(mos->secz - 1.0) : tgroup->McalAPER; + } + + // all associated images should be marked as coming from a bad night + // these will be fitted independently + image = getimages (&Nimage, NULL); + for (off_t j = 0; j < tgroup->Nimage; j++) { + off_t im = tgroup->image[j]; + image[im].flags |= ID_IMAGE_NIGHT_POOR; + } + + // reset the tgroup to have no impact on the solution + tgroup->McalPSF = 0.0; + tgroup->McalAPER = 0.0; + tgroup->dKlam = 0.0; +} + +// for now, we do not allow a night to be redeemed, so this +// operation should have no impact. +void markGoodTGroup (TGroup *tgroup) { + + off_t Nimage; + Image *image; + + // this tgroup has been identified as good. + tgroup->flags &= ~ID_IMAGE_NIGHT_POOR; + + // all associated mosaics should be marked as NOT coming from a bad night + for (off_t j = 0; j < tgroup->Nmosaic; j++) { + tgroup->mosaic[j]->flags &= ~ID_IMAGE_NIGHT_POOR; + } + + // all associated images should be marked as NOT coming from a bad night + image = getimages (&Nimage, NULL); + for (off_t j = 0; j < tgroup->Nimage; j++) { + off_t im = tgroup->image[j]; + image[im].flags &= ~ID_IMAGE_NIGHT_POOR; + } +} + +void initTGroupBins (Catalog *catalog, int Ncatalog) { + + off_t i, j; + + /* measure -> tgroup */ + if (!TGROUP_ZEROPT) return; + + ALLOCATE (MeasureToTGroup, TGroup **, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + ALLOCATE (MeasureToTGroup[i], TGroup *, MAX (catalog[i].Nmeasure, 1)); + for (j = 0; j < catalog[i].Nmeasure; j++) MeasureToTGroup[i][j] = NULL; + } +} + +void freeTGroupBins (int Ncatalog) { + + off_t i; + + /* measure -> tgroup */ + if (!TGROUP_ZEROPT) return; + + for (i = 0; i < Ncatalog; i++) { + free (MeasureToTGroup[i]); + } + free (MeasureToTGroup); +} + +int findTGroups (Catalog *catalog, int Ncatalog) { + + if (!TGROUP_ZEROPT) return (FALSE); + // if we are calibrating by tgroup, redefine myDet == on one of my tgroups + + int Nmatch = 0; + for (int i = 0; i < Ncatalog; i++) { + for (off_t j = 0; j < catalog[i].Nmeasure; j++) { + catalog[i].measureT[j].myDet = FALSE; // a detetion is not mine until proven otherwise + + if (TimeSelect) { + if (catalog[i].measureT[j].t < TSTART) continue; + if (catalog[i].measureT[j].t > TSTOP) continue; + } + + // check if this measure has a relevant photcode + int Ns = GetActivePhotcodeIndex (catalog[i].measureT[j].photcode); + if (Ns < 0) continue; + + matchTGroups (catalog, j, i); + Nmatch ++; + } + } + fprintf (stderr, "Matched %d detections to tgroups\n", Nmatch); + + return (TRUE); +} + +void matchTGroups (Catalog *catalog, off_t meas, int cat) { + + MeasureTiny *measure = &catalog[cat].measureT[meas]; + + off_t ID = measure[0].imageID; // ID is a unique ID in DVO + off_t idx = getImageByID (ID); // idx is the sequence number in the subset list + if (idx == -1) { + if (VERBOSE2) fprintf (stderr, "missed measurement "OFF_T_FMT", %d\n", meas, cat); + return; + } + + TGroup *myGroup = ImageToTGroup[idx]; + if (!myGroup) return; + + // this measurement is on one of my tgroups, mark it as mine. + catalog[cat].measureT[meas].myDet = TRUE; + MeasureToTGroup[cat][meas] = myGroup; + + myGroup->catalog[myGroup->Nmeasure] = cat; + myGroup->measure[myGroup->Nmeasure] = meas; + myGroup->Nmeasure ++; + + if (myGroup->Nmeasure == myGroup->NMEASURE) { + myGroup->NMEASURE += 1000; + REALLOCATE (myGroup->catalog, off_t, myGroup->NMEASURE); + REALLOCATE (myGroup->measure, off_t, myGroup->NMEASURE); + } + return; +} + +// dZpt (if supplied) returns the error on Mgrp +float getMgrp (off_t meas, int cat, float airmass, float *dZpt) { + + if (dZpt) { *dZpt = 0; } + + if (!TGROUP_ZEROPT) return 0.0; + + // unassigned measurements belong to simple chips + TGroup *myGroup = MeasureToTGroup[cat][meas]; + if (!myGroup) return 0.0; + + float value = myGroup->McalPSF + myGroup->dKlam*(airmass - 1.0); + if (dZpt) { *dZpt = myGroup->dMcal; } + return value; +} + +int getTGroupFlags (off_t meas, int cat) { + + if (!TGROUP_ZEROPT) return (0); + + // unassigned measurements belong to simple chips + TGroup *myGroup = MeasureToTGroup[cat][meas]; + if (!myGroup) return (0); + + return (myGroup->flags); +} + +typedef struct { + int Nfew; + int Nbad; + int Ncal; + int Nmos; + int Ngrid; + int Nrel; + int Nsys; + int Nskip; + off_t Nmax; + FitDataSet psfStars; + FitDataSet kronStars; + FitDataSet brightStars; +} SetMgrpInfo; + +enum {THREAD_RUN, THREAD_DONE}; + +typedef struct { + int entry; + int state; + Catalog *catalog; + Image *image; + FlatCorrectionTable *flatcorr; + SetMgrpInfo info; +} ThreadInfo; + +int setMgrp_tgroup (TGroup *tgroup, off_t Nmos, Image *image, Catalog *catalog, SetMgrpInfo *info, FlatCorrectionTable *flatcorr); +void *setMgrp_worker (void *data); +int setMgrp_threaded (Catalog *catalog, FlatCorrectionTable *flatcorr); + +void SetMgrpInfoInit (SetMgrpInfo *info, off_t Nmax, int allocLists) { + info->Nfew = 0; + info->Nbad = 0; + info->Ncal = 0; + info->Nmos = 0; + info->Nrel = 0; + info->Ngrid = 0; + info->Nskip = 0; + info->Nsys = 0; + + info->Nmax = Nmax; + + if (allocLists) { + int fitOrder = TGROUP_FIT_AIRMASS ? 1 : 0; + + FitDataSetAlloc (&info->psfStars, Nmax, fitOrder, 0); + FitDataSetAlloc (&info->kronStars, Nmax, fitOrder, 0); + FitDataSetAlloc (&info->brightStars, Nmax, fitOrder, 0); + + // until the analysis has converged a bit, do not use the IRLS analysis + if (UseStandardOLS(ZPT_TGROUP)) { + info->brightStars.MaxIterations = 0; + info->kronStars.MaxIterations = 0; + info->psfStars.MaxIterations = 0; + } + FitDataSetAddPriors (&info->psfStars); // why only psfStars? + if (TGROUP_FIT_AIRMASS) { + info->psfStars.bPriorValue[1] = 0.0; // note that we are fitting relative to the nominal slope + info->psfStars.bPriorSigma[1] = 0.04; // XXX this prior sigma needs to be user-configured + } + } +} + +void SetMgrpInfoFree (SetMgrpInfo *info) { + FitDataSetFree (&info->brightStars); + FitDataSetFree (&info->kronStars); + FitDataSetFree (&info->psfStars); +} + +void SetMgrpInfoAccum (SetMgrpInfo *summary, SetMgrpInfo *results) { + summary->Nfew += results->Nfew ; + summary->Nsys += results->Nsys ; + summary->Nbad += results->Nbad ; + summary->Ncal += results->Ncal ; + summary->Nmos += results->Nmos ; + summary->Nrel += results->Nrel ; + summary->Ngrid += results->Ngrid; + summary->Nskip += results->Nskip; +} + +static int npass_output = 0; + +// mutex to lock setMgrp_worker operations +static pthread_mutex_t setMgrp_mutex = PTHREAD_MUTEX_INITIALIZER; +static int nextTGroup = 0; + +// we have an array of (tgroupTimes, NtgroupTimes). we need to hand out tgroupTimes one at a time to +// the worker threads as they need +off_t getNextTGroupForThread () { + + pthread_mutex_lock (&setMgrp_mutex); + if (nextTGroup >= NtgroupTimes) { + pthread_mutex_unlock (&setMgrp_mutex); + return (-1); + } + int thisTGroup = nextTGroup; + nextTGroup ++; + + pthread_mutex_unlock (&setMgrp_mutex); + return (thisTGroup); +} + +int setMgrp (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + off_t N; + Image *image; + + if (!TGROUP_ZEROPT) return (FALSE); + if (TGROUP_ZPT_MODE == TGROUP_ZPT_MODE_NONE) return (FALSE); + + // plots cannot be done in a threaded context (the plot commands collide) + // so do not run setMgrp in threaded mode if PLOTSTUFF is set + if (NTHREADS && !PLOTSTUFF) { + int status = setMgrp_threaded (catalog, flatcorr); + return status; + } + + image = getimages (&N, NULL); + + off_t Nmax = 0; + for (off_t i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + Nmax = MAX (Nmax, tgroup[j].Nmeasure); + } + } + + SetMgrpInfo info; + SetMgrpInfoInit (&info, Nmax, TRUE); + + for (off_t i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + setMgrp_tgroup (&tgroup[j], i, image, catalog, &info, flatcorr); + // fprintf (stderr, "TGROUP time %.0f photcode %d Mcal: %f, dK: %f, dMcal: %f, sigma: %f, chisq: %f, %d of %d, limiting negative clouds to %f\n", + // ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, tgroup[j].McalPSF, tgroup[j].dKlam, tgroup[j].dMcal, tgroup[j].stdev, tgroup[j].McalChiSq, tgroup[j].nFitPhotom, tgroup[j].nValPhotom, CLOUD_TOLERANCE); + } + } + SetMgrpInfoFree (&info); + + if (PLOTSTUFF) { + fprintf (stdout, "press return\n"); + if (fscanf (stdin, "%*c") != 1) fprintf (stderr, "\n"); + } + + npass_output ++; + + fprintf (stderr, "%d tgroups marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", info.Nfew, info.Nbad, info.Ncal, info.Ngrid, info.Nrel, info.Nsys); + + return (TRUE); +} + +# define NIGHT_TOOFEW 100 +# define NIGHT_GOOD_FRACTION 0.1 + +// 'tgroup' is a pointer to the current tgroup of interest (entry Ngrp) +int setMgrp_tgroup (TGroup *myTGroup, off_t Ngrp, Image *image, Catalog *catalog, SetMgrpInfo *info, FlatCorrectionTable *flatcorr) { + + off_t j; + + // calculate the statistics for both good and bad nights, but only set Mcal for the good nights + // we only skip the statistics for nights with too few measurements or exposures + + FitDataSet *psfStars = &info->psfStars; + FitDataSet *kronStars = &info->kronStars; + FitDataSet *brightStars = &info->brightStars; + + assert (Ngrp >= 0); + assert (Ngrp < NtgroupTimes); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + TGTimes *tgroup = (TGTimes *) myTGroup->parent; + double mjdStart = ohana_sec_to_mjd (tgroup->start); + + // unset this flag at start (set below if zeropoint is calculated) + myTGroup->flags &= ~ID_IMAGE_TGROUP_PHOTCAL; + + // in clean_tgroups (run on each iteration), we identify good and bad tgroups. We mark + // bad tgroups and do not save a zero point for them. Instead we will fit those + // exposures or images separately + int useMgrp = TRUE; + if (TGROUP_ZPT_MODE == TGROUP_ZPT_MODE_GOOD_NIGHT) { + if (myTGroup->flags & ID_IMAGE_NIGHT_POOR) useMgrp = FALSE; + } + + // for testing, supply the MJD of a night or nights here to dump the full list of measurements + int testImage = FALSE; + // testImage |= (abs(mjdStart - 56586) < 0.1); + // testImage |= (abs(mjdStart - 57974) < 0.1); + // testImage |= (abs(mjdStart - 57975) < 0.1); + // testImage |= (abs(mjdStart - 57977) < 0.1); + // testImage |= (abs(mjdStart - 57981) < 0.1); + // testImage |= (abs(mjdStart - 58710) < 0.1); + + FILE *fout = NULL; + if (testImage) { + char filename[64]; + snprintf (filename, 64, "test.%05d.%02d.dat", (int) Ngrp, npass_output); + fout = fopen (filename, "w"); + fprintf (fout, "# tgroup %s (%.0f)\n", ohana_sec_to_date(tgroup->start), mjdStart); + save_test_night_measures (fout, myTGroup, catalog, flatcorr); + fclose (fout); + } + + // number of stars to measure the bright-end scatter + int Nbright = 0; + + int N = 0; + for (j = 0; j < myTGroup->Nmeasure; j++) { + + off_t m = myTGroup->measure[j]; + off_t c = myTGroup->catalog[j]; + + if (catalog[c].measureT[m].dbFlags & MEAS_BAD) { + info->Nbad ++; + continue; + } + float Mcal = getMcal (m, c, MAG_CLASS_PSF); + if (isnan(Mcal)) { + info->Ncal++; + continue; + } + float Mmos = getMmos (m, c); + if (isnan(Mmos)) { + info->Nmos++; + continue; + } + float Mgrid = getMgrid (m, c); + if (isnan(Mgrid)) { + info->Ngrid ++; + continue; + } + + // Mrel* is the average magnitude for this star. For PS1 stacks, we have too much + // PSF variability. We need to calibrate the PSF magnitudes separately from the + // Aperture-like magnitues. (We have an option to use the kron magnitudes or the + // other apertures here). I basically need to do this analysis separately for each + // magnitude type + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); + if (isnan(MrelPSF)) { + info->Nrel ++; + continue; + } + float MrelKron = getMrel (catalog, m, c, MAG_CLASS_KRON, MAG_SRC_CHP); + + // image.Mcal is not supposed to include the flat-field correction, so we need to + // apply that offset as well here for this image (in other words, each detection is + // being compared to the model, excluding the zero point, Mcal. The model includes + // the flat-correction. NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat) + + float Mflat = getMflat (m, c, flatcorr, catalog); + + off_t n = catalog[c].measureT[m].averef; + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + if (isnan(MsysPSF)) { + info->Nsys++; + continue; + } + float MsysKron = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_KRON); + + // here we are only checking for equiv photcodes (not active photcodes) + PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode); + if (!code) goto skip; + if (code->equiv < 1) goto skip; + int Nsec = GetPhotcodeNsec (code->equiv); + if (Nsec == -1) goto skip; + + skip: + assert (N < info->Nmax); + assert (N >= 0); + assert (Nbright < info->Nmax); + assert (Nbright >= 0); + + float Moff = Mcal + Mmos + Mgrid - Mflat; + + psfStars->alldata-> yVector[N] = MsysPSF - MrelPSF - Moff; + psfStars->alldata->dyVector[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR); + + kronStars->alldata-> yVector[N] = MsysKron - MrelKron - Moff; + kronStars->alldata->dyVector[N] = psfStars->alldata->dyVector[N]; + + if (TGROUP_FIT_AIRMASS) { + psfStars->alldata-> xVector[N] = (catalog[c].measureT[m].airmass - 1.0); + kronStars->alldata-> xVector[N] = psfStars->alldata-> xVector[N]; + } + + if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) { + brightStars->alldata-> yVector[Nbright] = psfStars->alldata-> yVector[N]; + brightStars->alldata->dyVector[Nbright] = psfStars->alldata->dyVector[N]; + if (TGROUP_FIT_AIRMASS) { + brightStars->alldata-> xVector[Nbright] = psfStars->alldata-> xVector[N]; + } + Nbright ++; + } + N++; + } + + /* skip nights with too few good measurements or too many bad measurements */ + int mark = (N < NIGHT_TOOFEW) || (N < NIGHT_GOOD_FRACTION*myTGroup->Nmeasure); + if (mark) { + myTGroup->flags |= ID_IMAGE_PHOTOM_FEW; + myTGroup->McalPSF = 0.0; + myTGroup->McalAPER = 0.0; + myTGroup->dMcal = NAN; + myTGroup->stdev = NAN; + myTGroup->dMmin = NAN; + myTGroup->dMmax = NAN; + myTGroup->McalChiSq = NAN; + myTGroup->nFitPhotom = 0; + myTGroup->nValPhotom = N; + myTGroup->dKlam = (TGROUP_FIT_AIRMASS) ? psfStars->bSaveArray[1][0] : 0; + info->Nfew ++; + if (testImage) { + fprintf (stderr, "NOTE: *** marked test image poor : %d %d %d***\n", (int) N, (int) NIGHT_TOOFEW, (int) (NIGHT_GOOD_FRACTION*myTGroup->Nmeasure)); + } + return TRUE; // skip nights with too few good measurements + } else { + myTGroup->flags &= ~ID_IMAGE_PHOTOM_FEW; + } + + /* we have a non-trivial fraction of measurements which are extreme outliers, and + measurements with very small errors. To avoid over-weighting measurements with very + small errors, let's add something in quadarature. + */ + + StatType stats = FitDataSetSoften (psfStars, N); + double altSigma = (stats.Upper80 - stats.Lower20) / 1.6; + + // do anything special with identified good nights? + // (myTGroup->flags & ID_IMAGE_PHOTOM_UBERCAL) + + fit1d_irls (psfStars, N); + fit1d_irls (kronStars, N); + fit1d_irls (brightStars, Nbright); + + if (useMgrp) { + myTGroup->McalPSF = psfStars->bSaveArray[0][0]; + myTGroup->McalAPER = kronStars->bSaveArray[0][0]; // XXX this does not make sense: I need to apply the airmass slope calculated above first + myTGroup->flags |= ID_IMAGE_TGROUP_PHOTCAL; // set this flag (unset by default) + } else { + myTGroup->McalPSF = 0.0; + myTGroup->McalAPER = 0.0; + } + + // record statistics on the fit regardless if it is used or not + myTGroup->dMcal = psfStars->bSigma[0]; + myTGroup->stdev = psfStars->sigma; + myTGroup->dMmin = psfStars->min; + myTGroup->dMmax = psfStars->max; + myTGroup->McalChiSq = psfStars->chisq; + myTGroup->nFitPhotom = psfStars->Nmeas; + myTGroup->nValPhotom = N; + myTGroup->dKlam = (TGROUP_FIT_AIRMASS) ? psfStars->bSaveArray[1][0] : 0; + + // bright end scatter (systematic error) + myTGroup->dMsys = brightStars->sigma; + + TGTimes *mygroup = (TGTimes *) myTGroup->parent; + fprintf (stderr, "TGroup Stats %f,%d : %d %d %6.3f %6.3f %6.3f %f %d\n", ohana_sec_to_mjd(mygroup->start), myTGroup->photcode, (int) N, (int) myTGroup->Nmeasure, psfStars->bSaveArray[0][0], stats.median, altSigma, myTGroup->McalChiSq, useMgrp); + + if (testImage) { + TGTimes *parent = (TGTimes *) myTGroup->parent; + fprintf (stderr, "test night %.0f aper: %f %f %d ... ", ohana_sec_to_mjd(parent->start), myTGroup->McalPSF, myTGroup->dMcal, myTGroup->nFitPhotom); + fprintf (stderr, "%f %f : %f\n", myTGroup->McalPSF, myTGroup->dMsys, myTGroup->McalChiSq); + } + if (PLOTSTUFF) { + fprintf (stderr, "Mgrp: %6.3f +/- %6.3f %5d of %5d | %.0f\n", myTGroup->McalPSF, myTGroup->dMcal, myTGroup->nFitPhotom, N, ohana_sec_to_mjd(tgroup->start)); + plot_setMcal (psfStars->alldata->yVector, N); + } + + return TRUE; +} + +int save_test_night_measures (FILE *fout, TGroup *myTGroup, Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int Nsecfilt = GetPhotcodeNsecfilt (); + + for (int j = 0; j < myTGroup->Nmeasure; j++) { + + off_t m = myTGroup->measure[j]; + off_t c = myTGroup->catalog[j]; + + // XXX these two should both be 0.0 if we are fitting tgroups + float Mcal = getMcal (m, c, MAG_CLASS_PSF); // image zero point + float Mmos = getMmos (m, c); // mosaic zero point + + float Mgrid = getMgrid (m, c); // camera offset (deprecated?) + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); // average magnitude + float Mflat = getMflat (m, c, flatcorr, catalog); // flat-field correction + + off_t n = catalog[c].measureT[m].averef; + + // magnitude for this measurement + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + // for historical reasons, Mflat is defined with the wrong sign + float delta = MsysPSF - MrelPSF - Mcal - Mmos - Mgrid + Mflat - myTGroup->McalPSF; + + int isBad = (catalog[c].measureT[m].dbFlags & MEAS_BAD); + + double mjdMeas = ohana_sec_to_mjd (catalog[c].measureT[m].t); + + fprintf (fout, "%f %f : %f %f %f %f %f : %f %f %d : %f\n", + catalog[c].averageT[n].R, catalog[c].averageT[n].D, + MsysPSF, MrelPSF, Mcal, Mgrid, Mflat, catalog[c].measureT[m].airmass, delta, isBad, mjdMeas); + + } + return TRUE; +} + +int setMgrp_threaded (Catalog *catalog, FlatCorrectionTable *flatcorr) { + + int i; + off_t N; + + Image *image = getimages (&N, NULL); + + off_t Nmax = 0; + for (off_t i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + Nmax = MAX (Nmax, tgroup[j].Nmeasure); + } + } + + SetMgrpInfo summary; + SetMgrpInfoInit (&summary, Nmax, FALSE); + + pthread_attr_t attr; + pthread_attr_init (&attr); + pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED); + + pthread_t *threads; + ALLOCATE (threads, pthread_t, NTHREADS); + + ThreadInfo *threadinfo; + ALLOCATE (threadinfo, ThreadInfo, NTHREADS); + + // each time this function is called, we cycle through the available tgroups + // make sure we start at 0 + nextTGroup = 0;; + + // launch N worker threads + for (i = 0; i < NTHREADS; i++) { + threadinfo[i].entry = i; + threadinfo[i].state = THREAD_RUN; + threadinfo[i].catalog = catalog; + threadinfo[i].image = image; + threadinfo[i].flatcorr = flatcorr; + + // we do NOT allocate the arrays here, we only supply basic info used in the threads + // to allocate the arrays and set the MaxIterations + SetMgrpInfoInit (&threadinfo[i].info, Nmax, FALSE); + pthread_create (&threads[i], NULL, setMgrp_worker, &threadinfo[i]); + } + pthread_attr_destroy (&attr); + + // wait until all threads have finished + while (1) { + int allDone = TRUE; + for (i = 0; i < NTHREADS; i++) { + if (threadinfo[i].state == THREAD_RUN) allDone = FALSE; + } + if (allDone) { + break; + } + usleep (500000); + } + + // all threads are done, free the threads array and grab the info + free (threads); + + // report stats & summary from the threads + for (i = 0; i < NTHREADS; i++) { + fprintf (stderr, "setMgrp thread %d : %d tgroups marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n", + i, + threadinfo[i].info.Nfew, + threadinfo[i].info.Nbad, + threadinfo[i].info.Ncal, + threadinfo[i].info.Nmos, + threadinfo[i].info.Ngrid, + threadinfo[i].info.Nrel, + threadinfo[i].info.Nsys, + threadinfo[i].info.Nskip); + SetMgrpInfoAccum (&summary, &threadinfo[i].info); + } + fprintf (stderr, "total : %d tgroups marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n", + summary.Nfew, + summary.Nbad, + summary.Ncal, + summary.Nmos, + summary.Ngrid, + summary.Nrel, + summary.Nsys, + summary.Nskip); + free (threadinfo); + + npass_output ++; + + return TRUE; +} + +void *setMgrp_worker (void *data) { + + ThreadInfo *threadinfo = data; + + SetMgrpInfo results; + SetMgrpInfoInit (&results, threadinfo->info.Nmax, TRUE); // allocate list, dlist arrays here + + while (1) { + + off_t i = getNextTGroupForThread(); + if (i == -1) { + threadinfo->state = THREAD_DONE; + return NULL; + } + + Catalog *catalog = threadinfo->catalog; + FlatCorrectionTable *flatcorr = threadinfo->flatcorr; + Image *image = threadinfo->image; + + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + setMgrp_tgroup (&tgroup[j], i, image, catalog, &results, flatcorr); + SetMgrpInfoAccum (&threadinfo->info, &results); + } + } + + SetMgrpInfoFree (&results); + return NULL; +} + +StatType statsTGroupM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!TGROUP_ZEROPT) return (stats); + + // XXX do this by time & photcode? + ALLOCATE (list, double, NtgroupTimes*Nphotcodes); + ALLOCATE (dlist, double, NtgroupTimes*Nphotcodes); + + int NIGHT_BAD = ID_IMAGE_NIGHT_POOR | ID_IMAGE_PHOTOM_FEW; + + int n = 0; + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + if (tgroup[j].flags & NIGHT_BAD) continue; + list[n] = tgroup[j].McalPSF; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsTGroupdM (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!TGROUP_ZEROPT) return (stats); + + ALLOCATE (list, double, NtgroupTimes*Nphotcodes); + ALLOCATE (dlist, double, NtgroupTimes*Nphotcodes); + + int n = 0; + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + if (tgroup[j].flags & IMAGE_BAD) continue; + list[n] = tgroup[j].dMcal; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +StatType statsTGroupX (Catalog *catalog) { + OHANA_UNUSED_PARAM(catalog); + + double *list, *dlist; + StatType stats; + + bzero (&stats, sizeof (StatType)); + if (!TGROUP_ZEROPT) return (stats); + + ALLOCATE (list, double, NtgroupTimes*Nphotcodes); + ALLOCATE (dlist, double, NtgroupTimes*Nphotcodes); + + int n = 0; + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + if (tgroup[j].flags & IMAGE_BAD) continue; + list[n] = tgroup[j].McalChiSq; + dlist[n] = 1; + n++; + } + } + + liststats_setmode (&stats, "MEAN"); + + liststats (list, dlist, NULL, n, &stats); + free (list); + free (dlist); + return (stats); +} + +/* new rules for bad tgroups (bad nights) + * too few exposures (Nmosaic < X) + * dMsys or stdev > X + * X%-ile of dMsys or stdev + * McalChiSq > X + * X%-ile of McalChiSq + + * I can define rules for these but they would be arbitrary + * I think I should do an analysis of the observed distribution + + */ + +static float MinMaxChiSq = NAN; +static float MinMaxScatter = NAN; + +void clean_tgroups () { + + double *mlist, *slist; + + if (!TGROUP_ZEROPT) return; + + if (VERBOSE) fprintf (stderr, "marking poor tgroups\n"); + + ALLOCATE (mlist, double, NtgroupTimes*Nphotcodes); + ALLOCATE (slist, double, NtgroupTimes*Nphotcodes); + + // measure the good/bad statistics using nights which are actually calibrated + // But they could be applied to all nights below (allowing bad nights to become good) + // However, for now, we will NOT allow nights to be redeemed (bad -> good) + int N = 0; + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + if (!(tgroup[j].flags & ID_IMAGE_TGROUP_PHOTCAL)) continue; + mlist[N] = tgroup[j].McalChiSq; + slist[N] = tgroup[j].stdev; // stdev of all measurements + N++; + } + } + + StatType stats; + liststats_setmode (&stats, "MEAN"); + + liststats (mlist, NULL, NULL, N, &stats); + float ChiSqUpper90 = stats.Upper90; + if (isnan (MinMaxChiSq)) MinMaxChiSq = 2.0*stats.median; + float MaxChiSq = MAX (MinMaxChiSq, ChiSqUpper90); + + liststats (slist, NULL, NULL, N, &stats); + float ScatterUpper90 = stats.Upper90; + if (isnan (MinMaxScatter)) MinMaxScatter = 2.0*stats.median; + float MaxScatter = MAX (MinMaxScatter, ScatterUpper90); + + fprintf (stderr, "TGROUPS: Max ChiSq: %f, Max Scatter: %f | ChiSquare Upper 90: %f, Scatter Upper 90: %f\n", MaxChiSq, MaxScatter, ChiSqUpper90, ScatterUpper90); + + int Ntotal = 0, Npoor = 0, Nmark = 0, Nscatter = 0, Nchisq = 0, NfewNights = 0, NfewExp = 0; + for (int i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + + Ntotal ++; + + int mark = FALSE; + + // we do not allow bad nights to be redeemed + if (tgroup[j].flags & ID_IMAGE_NIGHT_POOR) { + fprintf (stderr, "TGroup Already Poor, not redeemed %f,%d : %6.3f %6.3f %6.3f\n", ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, tgroup[j].McalPSF, tgroup[j].stdev, tgroup[j].McalChiSq); + Npoor ++; + continue; + } + + // too few measurements (set in setMgrp_tgroup) + if (tgroup[j].flags & ID_IMAGE_PHOTOM_FEW) { + mark = TRUE; + NfewNights ++; + } + // too few exposures (configure) + if (tgroup[j].Nmosaic < 4) { + mark = TRUE; + NfewExp ++; + } + // scatter too large + if (tgroup[j].stdev > MaxScatter) { + mark = TRUE; + Nscatter ++; + } + // chisq too large + if (tgroup[j].McalChiSq > MaxChiSq) { + mark = TRUE; + Nchisq ++; + } + if (mark) { + Nmark ++; + markBadTGroup(&tgroup[j]); // mark the tgroup & associated images & mosaics with a bad night + fprintf (stderr, "TGroup Poor %f,%d : %6.3f %6.3f %6.3f\n", ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, tgroup[j].McalPSF, tgroup[j].stdev, tgroup[j].McalChiSq); + } else { + markGoodTGroup(&tgroup[j]); // mark the tgroup & associated images & mosaics with a good night + } + } + } + + fprintf (stderr, "%d + %d of %d tgroups marked poor (%d scatter, %d few nights, %d few exposures, %d chisq)\n", Nmark, Npoor, Ntotal, Nscatter, NfewNights, NfewExp, Nchisq); + + free (mlist); + free (slist); +} + +void plot_tgroup_fields (Catalog *catalog) { + + off_t m, c, Nimage; + double *xlist, *ylist; + char string[64]; + Graphdata graphdata; + + if (!TGROUP_ZEROPT) return; + + // Image *image = getimages (&Nimage, NULL); returned value ignored + getimages (&Nimage, NULL); + + off_t N = 0; + for (off_t i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + N = MAX (N, tgroup[j].Nmeasure); + } + } + + ALLOCATE (xlist, double, N); + ALLOCATE (ylist, double, N); + + for (off_t k = 0; k < NtgroupTimes; k++) { + N = 0; + TGroup *tgroup = tgroupTimes[k][0].byCode; + for (off_t i = 0; i < tgroupTimes[i][0].nCode; i++) { + for (off_t j = 0; j < tgroup[i].Nmeasure; j++) { + + m = tgroup[i].measure[j]; + c = tgroup[i].catalog[j]; + + if (catalog[c].measureT[m].dbFlags & (ID_MEAS_AREA | ID_MEAS_NOCAL)) continue; + + // ave = catalog[c].measureT[m].averef; + xlist[N] = catalog[c].measureT[m].R; + ylist[N] = catalog[c].measureT[m].D; + N++; + } + } + sprintf (string, "TGroup "OFF_T_FMT, k); + plot_defaults (&graphdata); + plot_list (&graphdata, xlist, ylist, N, string, NULL); + } + + free (ylist); + free (xlist); +} + +void plot_tgroups () { + + off_t i, bin; + double *xlist, *Mlist, *dlist; + Graphdata graphdata; + + if (!TGROUP_ZEROPT) return; + + ALLOCATE (xlist, double, NtgroupTimes*Nphotcodes); + ALLOCATE (dlist, double, NtgroupTimes*Nphotcodes); + ALLOCATE (Mlist, double, NtgroupTimes*Nphotcodes); + + int Npts = 0; + for (i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + Mlist[Npts] = tgroup[j].McalPSF; + dlist[Npts] = tgroup[j].dMcal; + xlist[Npts] = tgroupTimes[i][0].start; + Npts ++; + } + } + + plot_defaults (&graphdata); + graphdata.xmin = 0.95; + graphdata.xmax = 2.50; + graphdata.ymin = PlotdMmin; + graphdata.ymax = PlotdMmax; + plot_list (&graphdata, xlist, Mlist, Npts, "airmass vs Mcal", "%s.airmass.png", OUTROOT); + plot_defaults (&graphdata); + graphdata.size = 1.5; + graphdata.ptype = 7; + plot_list (&graphdata, Mlist, dlist, Npts, "Mcal vs dMcal", "%s.MdM.png", OUTROOT); + +# define NBIN 200 + REALLOCATE (xlist, double, NBIN); + REALLOCATE (Mlist, double, NBIN); + + /**** dMcal histgram ****/ + for (i = 0; i < NBIN; i++) xlist[i] = 0.00005*i; + bzero (Mlist, NBIN*sizeof(double)); + + for (i = 0; i < NtgroupTimes; i++) { + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (int j = 0; j < tgroupTimes[i][0].nCode; j++) { + bin = tgroup[j].dMcal / 0.00005; + bin = MAX (0, MIN (NBIN - 1, bin)); + Mlist[bin] += 1.0; + } + } + plot_defaults (&graphdata); + graphdata.style = 1; + plot_list (&graphdata, xlist, Mlist, NBIN, "dMcal hist", "%s.dMcalhist.png", OUTROOT); + + free (dlist); + free (xlist); + free (Mlist); +} + +// temporary test output +void dump_tgroups (Catalog *catalog, int Npass) { + + return; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + int Ngrp = 0; + for (off_t i = 0; i < NtgroupTimes; i++) { + + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + + FILE *fout = NULL; + char filename[64]; + snprintf (filename, 64, "tgrp.%05d.%02d.dat", Ngrp, Npass); + fout = fopen (filename, "w"); + Ngrp ++; + + TGroup *myTGroup = &tgroup[j]; + + for (int k = 0; k < myTGroup->Nmeasure; k++) { + + off_t m = myTGroup->measure[k]; + off_t c = myTGroup->catalog[k]; + + float Mcal = getMcal (m, c, MAG_CLASS_PSF); // image zero point + float Mmos = getMmos (m, c); // mosaic zero point + float Mgrp = getMgrp (m, c, catalog[c].measureT[m].airmass, NULL); + + float MrelPSF = getMrel (catalog, m, c, MAG_CLASS_PSF, MAG_SRC_CHP); // average magnitude + // float Mgrid = getMgrid (m, c); // camera offset (deprecated?) + // float Mflat = getMflat (m, c, flatcorr, catalog); // flat-field correction + + off_t n = catalog[c].measureT[m].averef; + + // nominal magnitude for this measurement (instrumental + Ko(airmass - 1) + Co + float MsysPSF = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt], MAG_CLASS_PSF); + + // for historical reasons, Mflat is defined with the wrong sign + float delta = MsysPSF - MrelPSF - Mcal - Mmos - Mgrp; + + fprintf (fout, "%f %f : %f %f : %f %f %f : %f %f\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, MsysPSF, MrelPSF, Mcal, Mmos, Mgrp, catalog[c].measureT[m].airmass, delta); + } + fclose (fout); + } + } +} + +void dump_tgroup_imstats (int Npass) { + + FILE *fout = NULL; + char filename[64]; + + snprintf (filename, 64, "imstats.grp.%02d.dat", Npass); + fout = fopen (filename, "w"); + + for (off_t i = 0; i < NtgroupTimes; i++) { + + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + fprintf (fout, "%.0f %5d | %6ld %6d %6d %4ld %4ld | 0x%08x %6.3f %6.3f %6.3f %6.3f %6.3f %6.3f %6.3f %6.3f\n", + ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, + tgroup[j].Nmeasure, tgroup[j].nFitPhotom, tgroup[j].nValPhotom, tgroup[j].Nimage, tgroup[j].Nmosaic, + tgroup[j].flags, tgroup[j].McalPSF, tgroup[j].dKlam, tgroup[j].dMcal, tgroup[j].dMsys, tgroup[j].stdev, tgroup[j].dMmin, tgroup[j].dMmax, tgroup[j].McalChiSq); + } + } + fclose (fout); + + // ************ mosaic stats + + snprintf (filename, 64, "imstats.mos.%02d.dat", Npass); + fout = fopen (filename, "w"); + + for (off_t i = 0; i < NtgroupTimes; i++) { + + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + + for (off_t k = 0; k < tgroup->Nmosaic; k++) { + Mosaic *mosaic = tgroup[j].mosaic[k]; + + fprintf (fout, "%4ld %3ld %3ld %.0f %5d %5d %5d | %12.5f %5.3f 0x%08x | %6.3f %6.3f %6.3f %6.3f\n", + i, j, k, ohana_sec_to_mjd(tgroupTimes[i][0].start), mosaic->photcode, mosaic->nFitPhotom, mosaic->nValPhotom, + ohana_sec_to_mjd(mosaic->start), mosaic->secz, mosaic->flags, + mosaic->McalPSF, mosaic->dMcal, mosaic->dMsys, mosaic->McalChiSq); + } + } + } + fclose (fout); + + // ************ image stats + + snprintf (filename, 64, "imstats.chp.%02d.dat", Npass); + fout = fopen (filename, "w"); + + off_t Nimage = 0; + Image *image = getimages (&Nimage, NULL); + for (off_t i = 0; i < NtgroupTimes; i++) { + + TGroup *tgroup = tgroupTimes[i][0].byCode; + for (off_t j = 0; j < tgroupTimes[i][0].nCode; j++) { + + for (off_t k = 0; k < tgroup->Nimage; k++) { + off_t im = tgroup[j].image[k]; + + fprintf (fout, "%4ld %3ld %3ld %.0f %5d | %12.5f %5.3f 0x%08x | %6.3f %6.3f %6.3f %6.3f %6.3f\n", + i, j, k, ohana_sec_to_mjd(tgroupTimes[i][0].start), image[im].photcode, + ohana_sec_to_mjd(image[im].tzero), image[im].secz, image[im].flags, + image[im].McalPSF, image[im].McalAPER, image[im].dMcal, image[im].dMagSys, image[im].McalChiSq); + } + } + } + + fclose (fout); +} + +/* + for testing, put lines like these in relphot_images.c within the iteration loop + dump_tgroups (catalog, i + 40); + dump_catalog (catalog, 0, i + 40); // for a test, just dump a specific catalog + dump_tgroup_imstats (i + 40); +*/ Index: /branches/eam_branches/relphot.20210521/src/WarpImageMaps.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/WarpImageMaps.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/WarpImageMaps.c (revision 41602) @@ -0,0 +1,279 @@ +# include "relphot.h" + +// we have a list of warp images, each with obstime & photcode. I need to make a list of the obstime groups + +# define D_NOBSTIMES 10000 + +typedef struct { + int obstime; + unsigned short photcode; + int Nwarps; + int NWARPS; + off_t *warps; + double *Rmin; + double *Rmax; + double *Dmin; + double *Dmax; + int *onBoundary; +} WarpGroup; + +WarpGroup *warpgroup = NULL; +int Nwarpgroup = 0; +int NWARPGROUP = 0; + +myIndexType *warpObstimeIndex = NULL; + +int FindWarpGroups (void) { + + off_t Nimage; + Image *image = getimages (&Nimage, NULL); + + int *obstimes = NULL; + int Nobstimes = 0; + int NOBSTIMES = D_NOBSTIMES; + ALLOCATE (obstimes, int, NOBSTIMES); + + // first generate a list of obstime values + for (off_t i = 0; i < Nimage; i++) { + if (!isGPC1warp(image[i].photcode)) continue; + + obstimes[Nobstimes] = image[i].tzero; + Nobstimes ++; + + CHECK_REALLOCATE (obstimes, int, NOBSTIMES, Nobstimes, D_NOBSTIMES); + } + if (!Nobstimes) { + free (obstimes); + return FALSE; + } + + // sort the list: + isort (obstimes, Nobstimes); + + // find the unique obstimes + int *uniqtimes = NULL; + int *uniqcount = NULL; + int Nuniqtimes = 0; + ALLOCATE (uniqtimes, int, Nobstimes); + ALLOCATE (uniqcount, int, Nobstimes); + + // generate a uniq set of obstimes + for (int i = 0; i < Nobstimes; Nuniqtimes ++) { + uniqtimes[Nuniqtimes] = obstimes[i]; + int Ndup = 0; + int lastValue = uniqtimes[Nuniqtimes]; + while ((i < Nobstimes) && (obstimes[i] == lastValue)) { + i++; + Ndup ++; + uniqcount[Nuniqtimes] = Ndup; + } + } + REALLOCATE (uniqtimes, int, Nuniqtimes); + REALLOCATE (uniqcount, int, Nuniqtimes); + + // now I need to assign all warp images to a warpgroup + // I have 2 options to go from warp obstime to warpgroup: + // a) use a bisection lookup [may be slow] + // b) use an index on obstime [require ~5*3.14*1e7*4 bytes ~ 600MB for index] + + // generate an index on obstime + warpObstimeIndex = myIndexInit (); + warpObstimeIndex->minID = obstimes[0]; + warpObstimeIndex->maxID = obstimes[Nobstimes-1]; + + myIndexSetRange (warpObstimeIndex); + + // generate the warp groups (create the index as we go) + Nwarpgroup = Nuniqtimes; + ALLOCATE (warpgroup, WarpGroup, Nwarpgroup); + + for (int i = 0; i < Nuniqtimes; i++) { + warpgroup[i].obstime = uniqtimes[i]; + warpgroup[i].photcode = 0; // not yet set + warpgroup[i].Nwarps = 0; + warpgroup[i].NWARPS = 100; + ALLOCATE (warpgroup[i].warps, off_t, warpgroup[i].NWARPS); + myIndexSetEntry (warpObstimeIndex, uniqtimes[i], i); + } + + // assign all warps to one of the warpgroups + for (off_t i = 0; i < Nimage; i++) { + if (!isGPC1warp(image[i].photcode)) continue; + + int seq = myIndexGetEntry (warpObstimeIndex, image[i].tzero); + myAssert (warpgroup[seq].obstime == image[i].tzero, "oops"); + + if (!warpgroup[seq].photcode) { + warpgroup[seq].photcode = image[i].photcode; + } else { + myAssert (warpgroup[seq].photcode == image[i].photcode, "oops"); + } + + int N = warpgroup[seq].Nwarps; + warpgroup[seq].warps[N] = i; + warpgroup[seq].Nwarps ++; + + CHECK_REALLOCATE (warpgroup[seq].warps, off_t, warpgroup[seq].NWARPS, warpgroup[seq].Nwarps, 100); + } + + // we now have the warps assigned to groups. now we need to generate the grid data to find the warp assignments + for (int i = 0; i < Nwarpgroup; i++) { + + myAssert (warpgroup[i].Nwarps == uniqcount[i], "failure"); + ALLOCATE (warpgroup[i].Rmin, double, warpgroup[i].Nwarps); + ALLOCATE (warpgroup[i].Rmax, double, warpgroup[i].Nwarps); + ALLOCATE (warpgroup[i].Dmin, double, warpgroup[i].Nwarps); + ALLOCATE (warpgroup[i].Dmax, double, warpgroup[i].Nwarps); + ALLOCATE (warpgroup[i].onBoundary, int, warpgroup[i].Nwarps); + + for (int j = 0; j < warpgroup[i].Nwarps; j++) { + + off_t N = warpgroup[i].warps[j]; + + int Nx = image[N].NX; + int Ny = image[N].NY; + + warpgroup[i].onBoundary[j] = FALSE; + + double R, D; + XY_to_RD (&R, &D, 0.0, 0.0, &image[N].coords); + R = ohana_normalize_angle_to_midpoint(R, 180.0); + warpgroup[i].Rmin[j] = R; + warpgroup[i].Rmax[j] = R; + warpgroup[i].Dmin[j] = D; + warpgroup[i].Dmax[j] = D; + + // XXX need to worry about 0,360 boundary + XY_to_RD (&R, &D, Nx, 0.0, &image[N].coords); + R = ohana_normalize_angle_to_midpoint(R, 180.0); + warpgroup[i].Rmin[j] = MIN (warpgroup[i].Rmin[j], R); + warpgroup[i].Rmax[j] = MAX (warpgroup[i].Rmax[j], R); + warpgroup[i].Dmin[j] = MIN (warpgroup[i].Dmin[j], D); + warpgroup[i].Dmax[j] = MAX (warpgroup[i].Dmax[j], D); + + // XXX need to worry about 0,360 boundary + XY_to_RD (&R, &D, 0.0, Ny, &image[N].coords); + R = ohana_normalize_angle_to_midpoint(R, 180.0); + warpgroup[i].Rmin[j] = MIN (warpgroup[i].Rmin[j], R); + warpgroup[i].Rmax[j] = MAX (warpgroup[i].Rmax[j], R); + warpgroup[i].Dmin[j] = MIN (warpgroup[i].Dmin[j], D); + warpgroup[i].Dmax[j] = MAX (warpgroup[i].Dmax[j], D); + + // XXX need to worry about 0,360 boundary + XY_to_RD (&R, &D, Nx, Ny, &image[N].coords); + R = ohana_normalize_angle_to_midpoint(R, 180.0); + warpgroup[i].Rmin[j] = MIN (warpgroup[i].Rmin[j], R); + warpgroup[i].Rmax[j] = MAX (warpgroup[i].Rmax[j], R); + warpgroup[i].Dmin[j] = MIN (warpgroup[i].Dmin[j], D); + warpgroup[i].Dmax[j] = MAX (warpgroup[i].Dmax[j], D); + + // bump Rmin,Rmax,Dmin,Dmax 10 arcsec worth of padding + double dR = 10.0/3600.0 / cos (RAD_DEG*warpgroup[i].Dmin[j]); + double dD = 10.0/3600.0; + + // at north pole, force test of all nearby skycells + if (warpgroup[i].Dmax[j] > 89.8) { + warpgroup[i].Rmin[j] = 0.0; + warpgroup[i].Rmax[j] = 360.0; + warpgroup[i].Dmax[j] = 90.0; + warpgroup[i].Dmin[j] -= dD; + continue; + } + + if (warpgroup[i].Rmax[j] - warpgroup[i].Rmin[j] > 270.0) { + // Rmin and Rmax are in the range 0 - 360. For images at the 0,360 boundary, + // "Rmax" is the lower edge, and "Rmin" is the upper edge. we need to flip them + // and then break the 0-360 range: + double tmp = warpgroup[i].Rmin[j]; + warpgroup[i].Rmin[j] = warpgroup[i].Rmax[j] - 360.0; + warpgroup[i].Rmax[j] = tmp; + warpgroup[i].onBoundary[j] = TRUE; + } + + warpgroup[i].Rmin[j] -= dR; + warpgroup[i].Rmax[j] += dR; + warpgroup[i].Dmin[j] -= dD; + warpgroup[i].Dmax[j] += dD; + } + } + + free (uniqtimes); + free (uniqcount); + free (obstimes); + return TRUE; +} + +int GetWarpSeq (Image *image, int obstime, unsigned short photcode, double Rave, double Dave, float X, float Y) { + + if (!warpObstimeIndex) return -1; + + int seq = myIndexGetEntry (warpObstimeIndex, obstime); + myAssert (seq > -1, "ooops"); + myAssert (warpgroup[seq].photcode == photcode, "oops"); + + double dPosMin = NAN; + int nPosMin = -1; + + // we now have the warp group, but which is the correct warp? + for (int i = 0; i < warpgroup[seq].Nwarps; i++) { + if (warpgroup[seq].onBoundary[i]) { + int inRange1 = (Rave >= warpgroup[seq].Rmin[i]) && (Rave <= warpgroup[seq].Rmax[i]); + int inRange2 = (Rave >= warpgroup[seq].Rmin[i] + 360.0) && (Rave <= warpgroup[seq].Rmax[i] + 360.0); + if (!inRange1 && !inRange2) continue; + } else { + if (Rave < warpgroup[seq].Rmin[i]) continue; + if (Rave > warpgroup[seq].Rmax[i]) continue; + } + if (Dave < warpgroup[seq].Dmin[i]) continue; + if (Dave > warpgroup[seq].Dmax[i]) continue; + + // this is a possible image: check the coordinates: + int N = warpgroup[seq].warps[i]; + + // project Rave,Dave to image pixels + double Xtst, Ytst; + RD_to_XY (&Xtst, &Ytst, Rave, Dave, &image[N].coords); + + // find the pixel offset + double dX = (Xtst - X); + double dY = (Ytst - Y); + + // skip detections which are within a small distance from the expected location + double dPos = hypot(dX,dY); + if (dPos < 20.0) return N; // 20.0 pixels = 5.0 arcsec + + // check for multiple possible matches?? + + // record the min pos so we can see how things fail + if (isnan(dPosMin)) { + dPosMin = dPos; + nPosMin = i; + } else { + if (dPos < dPosMin) { + dPosMin = dPos; + nPosMin = i; + } + } + } + + // if we did not find any matched, give up + if (nPosMin < 0) return -1; + + // return closest image + return warpgroup[seq].warps[nPosMin]; +} + +void FreeWarpGroups (void) { + + for (int i = 0; i < Nwarpgroup; i++) { + free (warpgroup[i].warps); + free (warpgroup[i].Rmin); + free (warpgroup[i].Rmax); + free (warpgroup[i].Dmin); + free (warpgroup[i].Dmax); + free (warpgroup[i].onBoundary); + } + + free (warpgroup); + myIndexFree (warpObstimeIndex); +} Index: /branches/eam_branches/relphot.20210521/src/ZeroPointModes.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/ZeroPointModes.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/ZeroPointModes.c (revision 41602) @@ -0,0 +1,202 @@ +# include "relphot.h" + +static int CurrentLoop = -1; // track the number of iterations + +void SetZptIteration (int current) { + // XXX add 'manual' option here: ask for desired zpt iteration + if (MANUAL_ITERATION) { + int newValue; + fprintf (stdout, "Set Iteration (current = %d): ", CurrentLoop); + if (!fscanf (stdin, "%d", &newValue)) return; + fprintf (stdout, "Using iteration %d\n", newValue); + CurrentLoop = newValue; + return; + } + + // automatic iteration: + CurrentLoop = current; +} + +int GetZptIteration (void) { + return CurrentLoop; +} + +void SetZeroPointModes (void) { + + if (TGROUP_ZEROPT && MOSAIC_ZEROPT) { + if (CurrentLoop < 6) { + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_ALL; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + } + if ((CurrentLoop >= 6) && (CurrentLoop < 12)) { + // after iterating a few times on the TGroup zero points: + // * identify the photometric nights + // * fit the mosaics from the non-photometric nights + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_GOOD_NIGHT; // stop fitting the bad nights + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_BAD_NIGHT; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + clean_tgroups(); // do this on every pass to update the status of nights + } + if ((CurrentLoop >= 12) && (CurrentLoop < 18)) { + // after iterating a few times on the TGroup & Mosaic zero points: + // * identify the good / bad Mosaics + // * fit the chips from the bad Mosaics + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_GOOD_NIGHT; // stop fitting the bad nights + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_BAD_NIGHT_GOOD_MOSAIC; // only fit good mosaics on bad nights + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_BAD_NIGHT_BAD_MOSAIC; // only fit bad mosaics on bad nights + clean_tgroups(); // do this on every pass + clean_mosaics(); // do this on every pass + } + if ((CurrentLoop >= 18) && (CurrentLoop <= 999)) { + // after iterating a few times on the TGroup & Mosaic zero points: + // * identify the good / bad Mosaics + // * fit the chips from the bad Mosaics + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_GOOD_NIGHT; // stop fitting the bad nights + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_BAD_NIGHT_GOOD_MOSAIC; // only fit good mosaics on bad nights + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_BAD_NIGHT_BAD_MOSAIC; // only fit bad mosaics on bad nights + clean_tgroups(); // do this on every pass + clean_mosaics(); // do this on every pass + clean_images(); // do this on every pass + } + } + + if (TGROUP_ZEROPT && !MOSAIC_ZEROPT) { + if (CurrentLoop <= 4) { + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_ALL; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + } + if ((CurrentLoop > 4) && (CurrentLoop <= 999)) { + // after iterating a few times on the TGroup zero points: + // * identify the photometric nights + // * fit the images from the non-photometric nights + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_GOOD_NIGHT; // stop fitting the bad nights + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_BAD_NIGHT; + clean_tgroups(); // do this on every pass or just sometimes? + } + } + + if (!TGROUP_ZEROPT && MOSAIC_ZEROPT) { + if (CurrentLoop <= 4) { + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_ALL; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + } + if ((CurrentLoop > 4) && (CurrentLoop <= 999)) { + // after iterating a few times on the Mosaic zero points: + // * identify the good / bad mosaics + // * fit the images from the bad mosaics + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_NONE; // stop fitting the bad nights + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_GOOD_MOSAIC; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_BAD_MOSAIC; + clean_mosaics(); // do this on every pass or just sometimes? + } + } + + if (!TGROUP_ZEROPT && !MOSAIC_ZEROPT) { + // if we are not fitting TGROUP or MOSAIC, just fit all images + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_ALL; + } +} + +// mode = TGROUP, MOSAIC, IMAGE +// should we use IRLS or OLS? depends on CurrentLoop and which stage we are in. +int UseStandardOLS (ZptFitModeType mode) { + + if (TGROUP_ZEROPT && MOSAIC_ZEROPT) { + switch (mode) { + case ZPT_STARS: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_TGROUP: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_MOSAIC: + if (CurrentLoop < 9) return TRUE; + return FALSE; + case ZPT_IMAGES: + default: + if (CurrentLoop < 15) return TRUE; + return FALSE; + } + } + + if (TGROUP_ZEROPT && !MOSAIC_ZEROPT) { + switch (mode) { + case ZPT_STARS: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_TGROUP: + case ZPT_MOSAIC: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_IMAGES: + default: + if (CurrentLoop < 7) return TRUE; + return FALSE; + } + } + + if (!TGROUP_ZEROPT && MOSAIC_ZEROPT) { + switch (mode) { + case ZPT_STARS: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_TGROUP: + case ZPT_MOSAIC: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_IMAGES: + default: + if (CurrentLoop < 7) return TRUE; + return FALSE; + } + } + + if (!TGROUP_ZEROPT && !MOSAIC_ZEROPT) { + switch (mode) { + case ZPT_STARS: + if (CurrentLoop < 3) return TRUE; + return FALSE; + case ZPT_TGROUP: + case ZPT_MOSAIC: + case ZPT_IMAGES: + default: + if (CurrentLoop < 3) return TRUE; + return FALSE; + } + } + return FALSE; +} + +/* TGROUP & MOSAIC: + 0 - TGROUP ONLY & OLS + 1 - TGROUP ONLY & OLS + 2 - TGROUP ONLY & OLS + + 3 - TGROUP ONLY & IRLS + 4 - TGROUP ONLY & IRLS + 5 - TGROUP ONLY & IRLS + + 6 - TGROUP + MOSAIC & OLS MOSAIC + 7 - TGROUP + MOSAIC & OLS MOSAIC + 8 - TGROUP + MOSAIC & OLS MOSAIC + + 9 - TGROUP + MOSAIC & IRLS MOSAIC + 10 - TGROUP + MOSAIC & IRLS MOSAIC + 11 - TGROUP + MOSAIC & IRLS MOSAIC + + 12 - TGROUP + MOSAIC + IMAGE & OLS IMAGE + 13 - TGROUP + MOSAIC + IMAGE & OLS IMAGE + 14 - TGROUP + MOSAIC + IMAGE & OLS IMAGE + + 15 - TGROUP + MOSAIC + IMAGE & IRLS IMAGE + 16 - TGROUP + MOSAIC + IMAGE & IRLS IMAGE + 17 - TGROUP + MOSAIC + IMAGE & IRLS IMAGE + +*/ + Index: /branches/eam_branches/relphot.20210521/src/args.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/args.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/args.c (revision 41602) @@ -0,0 +1,940 @@ +# include "relphot.h" + +RelphotMode args (int argc, char **argv) { + + int N; + double trange; + + /* define time */ + TimeSelect = FALSE; + if ((N = get_argument (argc, argv, "-time"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &TSTART)) { + fprintf (stderr, "ERROR: syntax error\n"); + return (MODE_ERROR); + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_dtime (argv[N], &trange)) { + if (!ohana_str_to_time (argv[N], &TSTOP)) { + fprintf (stderr, "ERROR: syntax error\n"); + return (MODE_ERROR); + } + } else { + if (trange < 0) { + trange = fabs (trange); + TSTOP = TSTART; + TSTART -= trange; + } else { + TSTOP = TSTART + trange; + } + } + remove_argument (N, &argc, argv); + TimeSelect = TRUE; + } + + /* specify portion of the sky */ + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + /* specify region file by name (eg n0000/0000.00) */ + UserCatalog = NULL; + if ((N = get_argument (argc, argv, "-catalog"))) { + remove_argument (N, &argc, argv); + UserCatalog = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // If we are looking at the whole sky (or whole relevant sky), use the full image table + // -- this save substantial memory. this could be automatic if the skyregion covers + // more than 2pi. + USE_ALL_IMAGES = FALSE; + if ((N = get_argument (argc, argv, "-use-all-images"))) { + remove_argument (N, &argc, argv); + USE_ALL_IMAGES = TRUE; + } + + USE_BASIC_CHECK = FALSE; + if ((N = get_argument (argc, argv, "-basic-image-search"))) { + remove_argument (N, &argc, argv); + USE_BASIC_CHECK = TRUE; + } + + IS_DIFF_DB = FALSE; + if ((N = get_argument (argc, argv, "-is-diff-db"))) { + remove_argument (N, &argc, argv); + IS_DIFF_DB = TRUE; + } + + USE_FULL_OVERLAP = TRUE; + if ((N = get_argument (argc, argv, "-sloppy-image-overlap"))) { + remove_argument (N, &argc, argv); + USE_FULL_OVERLAP = FALSE; + } + + VERBOSE = VERBOSE2 = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-vv"))) { + VERBOSE2 = VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + VERBOSE_IMAGE = FALSE; + if ((N = get_argument (argc, argv, "-vim"))) { + VERBOSE_IMAGE = TRUE; + remove_argument (N, &argc, argv); + } + + NTHREADS = 0; + if ((N = get_argument (argc, argv, "-threads"))) { + remove_argument (N, &argc, argv); + NTHREADS = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + // XXX for the moment, make this selection manual. it needs to be automatic + // based on the state of the SkyTable + HOST_ID = 0; + PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_PARALLEL_GROUPS = 0; + if ((N = get_argument (argc, argv, "-skip-parallel-groups"))) { + remove_argument (N, &argc, argv); + SKIP_PARALLEL_GROUPS = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + + // elements needed for parallel regions / parallel images + MANUAL_UNIQUER = NULL; + if ((N = get_argument (argc, argv, "-manual-uniquer"))) { + remove_argument (N, &argc, argv); + MANUAL_UNIQUER = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + PLOTSTUFF = FALSE; + if ((N = get_argument (argc, argv, "-plot"))) { + PLOTSTUFF = TRUE; + remove_argument (N, &argc, argv); + } + + PLOTDELAY = 500000; + if ((N = get_argument (argc, argv, "-plotdelay"))) { + remove_argument (N, &argc, argv); + PLOTDELAY = 1e6*atof(argv[N]); + PLOTSTUFF = TRUE; // always turn on plotting if i request a plot delay + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-outroot"))) { + remove_argument (N, &argc, argv); + OUTROOT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } else { + OUTROOT = strcreate ("relphot"); + } + + strcpy (STATMODE, "WT_MEAN"); + if ((N = get_argument (argc, argv, "-statmode"))) { + remove_argument (N, &argc, argv); + strcpy (STATMODE, argv[N]); + remove_argument (N, &argc, argv); + } + + NLOOP = 8; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + NLOOP = atof (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-nloop"))) { + remove_argument (N, &argc, argv); + NLOOP = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + MANUAL_ITERATION = FALSE; + if ((N = get_argument (argc, argv, "-manual-iteration"))) { + remove_argument (N, &argc, argv); + MANUAL_ITERATION = TRUE; + } + + NGRID = 8; + if ((N = get_argument (argc, argv, "-ngrid"))) { + remove_argument (N, &argc, argv); + NGRID = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + TEST_IMAGE1 = -1; + if ((N = get_argument (argc, argv, "-test-image1"))) { + char *endptr; + remove_argument (N, &argc, argv); + TEST_IMAGE1 = strtol(argv[N], &endptr, 0); + if (*endptr) relphot_usage (); + remove_argument (N, &argc, argv); + } + TEST_IMAGE2 = -1; + if ((N = get_argument (argc, argv, "-test-image2"))) { + char *endptr; + remove_argument (N, &argc, argv); + TEST_IMAGE2 = strtol(argv[N], &endptr, 0); + if (*endptr) relphot_usage (); + remove_argument (N, &argc, argv); + } + + RESET = FALSE; + if ((N = get_argument (argc, argv, "-reset"))) { + remove_argument (N, &argc, argv); + RESET = TRUE; + } + + RESET_ZEROPTS = FALSE; + if ((N = get_argument (argc, argv, "-reset-zpts"))) { + remove_argument (N, &argc, argv); + RESET_ZEROPTS = TRUE; + } + + REPAIR_WARPS = FALSE; + if ((N = get_argument (argc, argv, "-repair-warps"))) { + remove_argument (N, &argc, argv); + REPAIR_WARPS = TRUE; + } + + PRESERVE_PS1 = FALSE; + if ((N = get_argument (argc, argv, "-preserve-ps1"))) { + remove_argument (N, &argc, argv); + PRESERVE_PS1 = TRUE; + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + UPDATE_CATFORMAT = NULL; + if ((N = get_argument (argc, argv, "-update-catformat"))) { + remove_argument (N, &argc, argv); + UPDATE_CATFORMAT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + SAVE_IMAGE_UPDATES = TRUE; + if ((N = get_argument (argc, argv, "-skip-image-updates"))) { + remove_argument (N, &argc, argv); + SAVE_IMAGE_UPDATES = FALSE; + } + + MaxDensityUse = FALSE; + if ((N = get_argument (argc, argv, "-max-density"))) { + remove_argument (N, &argc, argv); + MaxDensityValue = atof(argv[N]); + remove_argument (N, &argc, argv); + MaxDensityUse = TRUE; + } + + CLOUD_TOLERANCE = 0.02; + if ((N = get_argument (argc, argv, "-cloud-limit"))) { + remove_argument (N, &argc, argv); + CLOUD_TOLERANCE = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + // XXX should we load a tree from CATDIR by default? + // NOTE: a given catdir needs an appropriate boundary tree + BOUNDARY_TREE = NULL; + if ((N = get_argument (argc, argv, "-boundary-tree"))) { + remove_argument (N, &argc, argv); + BOUNDARY_TREE = strcreate(argv[N]); + load_tess (BOUNDARY_TREE); + remove_argument (N, &argc, argv); + } + + SHOW_PARAMS = TRUE; + if ((N = get_argument (argc, argv, "-params"))) { + remove_argument (N, &argc, argv); + SHOW_PARAMS = FALSE; + } + + PlotMmin = 10.0; PlotMmax = 20.0; PlotdMmin = -1.0; PlotdMmax = 1.0; + if ((N = get_argument (argc, argv, "-plrange"))) { + remove_argument (N, &argc, argv); + PlotMmin = atof (argv[N]); + remove_argument (N, &argc, argv); + PlotMmax = atof (argv[N]); + remove_argument (N, &argc, argv); + PlotdMmin = atof (argv[N]); + remove_argument (N, &argc, argv); + PlotdMmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + // global restriction on fitting image zero points for mosaic cameras + FREEZE_IMAGES = FALSE; + if ((N = get_argument (argc, argv, "-imfreeze"))) { + remove_argument (N, &argc, argv); + FREEZE_IMAGES = TRUE; + } + // global restriction on fitting mosaic zero points + FREEZE_MOSAICS = FALSE; + if ((N = get_argument (argc, argv, "-mosfreeze"))) { + remove_argument (N, &argc, argv); + FREEZE_MOSAICS = TRUE; + } + CALIBRATE_STACKS_AND_WARPS = FALSE; + if ((N = get_argument (argc, argv, "-only-stacks-and-warps"))) { + remove_argument (N, &argc, argv); + CALIBRATE_STACKS_AND_WARPS = TRUE; + } + USE_MCAL_PSF_FOR_STACK_APER = FALSE; + if ((N = get_argument (argc, argv, "-use-mcal-psf-for-stack-aper"))) { + remove_argument (N, &argc, argv); + USE_MCAL_PSF_FOR_STACK_APER = TRUE; + } + + // default behavior is to fit all chips independently + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_ALL; + + /* XXX this argument used to do two things: specify the camera name and tell the analysis to + calculate a common zero point for a single mosaic. I've moved the MOSAICNAME concept into the + config system, but need to use this argument to specify that the mosaic zeropoints should be + calculated. */ + MOSAIC_ZEROPT = FALSE; + if ((N = get_argument (argc, argv, "-mosaic"))) { + remove_argument (N, &argc, argv); + MOSAIC_ZEROPT = TRUE; + if (!strcasecmp (MOSAICNAME, "none")) { + fprintf (stderr, "mosaic astrometry selected by MOSAICNAME not defined\n"); + exit (2); + } + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_ALL; + } + + // calibrate the photometry by night / time period + TGROUP_ZEROPT = FALSE; + char *TGROUP_FILENAME = NULL; + if ((N = get_argument (argc, argv, "-tgroup")) || (N = get_argument (argc, argv, "-tgroups"))) { + TGROUP_ZEROPT = TRUE; + remove_argument (N, &argc, argv); + // if we fit for TGROUPS, start with the images and mosaics frozen + TGROUP_ZPT_MODE = TGROUP_ZPT_MODE_ALL; + IMAGE_ZPT_MODE = IMAGE_ZPT_MODE_NONE; + MOSAIC_ZPT_MODE = MOSAIC_ZPT_MODE_NONE; + } + if ((N = get_argument (argc, argv, "-tgroup-list"))) { + if (!TGROUP_ZEROPT) { + fprintf (stderr, "-tgroup-list (filename) requires -tgroup option\n"); + exit (2); + } + remove_argument (N, &argc, argv); + TGROUP_FILENAME = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-tgroup")) || (N = get_argument (argc, argv, "-tgroups"))) { + fprintf (stderr, "use only one of -tgroups and -tgroup (same meaning)\n"); + exit (2); + } + TGROUP_FIT_AIRMASS = FALSE; + if ((N = get_argument (argc, argv, "-tgroup-fit-airmass"))) { + if (!TGROUP_ZEROPT) { + fprintf (stderr, "-tgroup-fit-airmass requires -tgroup (filename) option\n"); + exit (2); + } + TGROUP_FIT_AIRMASS = TRUE; + remove_argument (N, &argc, argv); + } + + // USE_GRID is not valid for all cases, probably should be its own mode... + USE_GRID = FALSE; + if ((N = get_argument (argc, argv, "-grid"))) { + remove_argument (N, &argc, argv); + USE_GRID = TRUE; + } + + KEEP_UBERCAL = TRUE; + if ((N = get_argument (argc, argv, "-reset-ubercal"))) { + remove_argument (N, &argc, argv); + KEEP_UBERCAL = FALSE; + } + + MIN_ERROR = 0.001; + if ((N = get_argument (argc, argv, "-minerror"))) { + remove_argument (N, &argc, argv); + MIN_ERROR = atof (argv[N]); + remove_argument (N, &argc, argv); + /* require MIN_ERROR > 0 */ + } + + AreaSelect = FALSE; + if ((N = get_argument (argc, argv, "-area"))) { + remove_argument (N, &argc, argv); + AreaXmin = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaXmax = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaYmin = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaYmax = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaSelect = TRUE; + } + + ImagSelect = FALSE; + if ((N = get_argument (argc, argv, "-instmag"))) { + remove_argument (N, &argc, argv); + ImagMin = atof (argv[N]); + remove_argument (N, &argc, argv); + ImagMax = atof (argv[N]); + remove_argument (N, &argc, argv); + ImagSelect = TRUE; + } + + DophotSelect = FALSE; + if ((N = get_argument (argc, argv, "-dophot"))) { + remove_argument (N, &argc, argv); + DophotValue = atof (argv[N]); + remove_argument (N, &argc, argv); + DophotSelect = TRUE; + } + + // if -synthphot is chosen, an artificial w-band measurement is generated for each object + // with r and i photometry. this was used to apply the ubercal photometry to w-band images. + SyntheticPhotometry = FALSE; + if ((N = get_argument (argc, argv, "-synthphot"))) { + remove_argument (N, &argc, argv); + SyntheticPhotometry = TRUE; + init_synthetic_mags(); + } + + refPhotcode = NULL; + if ((N = get_argument (argc, argv, "-refcode"))) { + remove_argument (N, &argc, argv); + refPhotcode = GetPhotcodebyName (argv[N]); + if (!refPhotcode) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", argv[N]); + exit (1); + } + remove_argument (N, &argc, argv); + } + USE_REFERENCE_WEIGHT = FALSE; + if ((N = get_argument (argc, argv, "-ref-weight"))) { + remove_argument (N, &argc, argv); + USE_REFERENCE_WEIGHT = TRUE; + } + + STAGES = STAGE_CHIP | STAGE_WARP | STAGE_STACK; + if ((N = get_argument (argc, argv, "-skip-chip"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_CHIP; + } + if ((N = get_argument (argc, argv, "-skip-warp"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_WARP; + } + if ((N = get_argument (argc, argv, "-skip-stack"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_STACK; + } + if (!STAGES) { + fprintf (stderr, "ERROR: no valid stages selected\n"); + exit (3); + } + + REGION_FILE = NULL; + if ((N = get_argument (argc, argv, "-region-hosts"))) { + remove_argument (N, &argc, argv); + REGION_FILE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + REGION_HOST_ID = 0; + if ((N = get_argument (argc, argv, "-region-hostID"))) { + remove_argument (N, &argc, argv); + REGION_HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + RelphotMode mode = MODE_ERROR; + if ((N = get_argument (argc, argv, "-images"))) { + remove_argument (N, &argc, argv); + mode = UPDATE_IMAGES; + } + if ((N = get_argument (argc, argv, "-averages"))) { + remove_argument (N, &argc, argv); + mode = UPDATE_AVERAGES; + } + + // if -synthphot-zpts is supplied, the map of corrections needed for the synthetic photometry is supplied + // and used to tie down the synthetic magnitudes + // NOTE: SYNTH_ZERO_POINTS is used for the same file in both -synthphot-zpts and -synthphot_means + SYNTH_ZERO_POINTS = NULL; + if ((N = get_argument (argc, argv, "-synthphot-zpts"))) { + remove_argument (N, &argc, argv); + myAssert (N < argc, "missing argument to -synthphot-zpts"); + SYNTH_ZERO_POINTS = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-synthphot_means"))) { + mode = SYNTH_PHOT; + remove_argument (N, &argc, argv); + myAssert (N < argc, "missing argument to -synthphot_means"); + SYNTH_ZERO_POINTS = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-apply-offsets"))) { + remove_argument (N, &argc, argv); + mode = APPLY_OFFSETS; + } + IMAGE_TABLE = NULL; + if ((N = get_argument (argc, argv, "-parallel-images"))) { + remove_argument (N, &argc, argv); + mode = PARALLEL_IMAGES; + if (N >= argc) relphot_usage(); + IMAGE_TABLE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + if (!REGION_FILE) relphot_usage(); + } + + PARALLEL_REGIONS_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-regions"))) { + remove_argument (N, &argc, argv); + mode = PARALLEL_REGIONS; + if (!REGION_FILE) relphot_usage(); + if ((N = get_argument (argc, argv, "-parallel-regions-manual"))) { + remove_argument (N, &argc, argv); + PARALLEL_REGIONS_MANUAL = TRUE; + } + } + + // (photcodes, Nphotcodes) is the list of active average photcodes + PhotcodeList = NULL; + photcodes = NULL; + switch (mode) { + case SYNTH_PHOT: + case UPDATE_AVERAGES: + if (argc != 1) relphot_usage(); + break; + + case UPDATE_IMAGES: + case PARALLEL_IMAGES: + case PARALLEL_REGIONS: + case APPLY_OFFSETS: + PhotcodeList = strcreate (argv[1]); + photcodes = ParsePhotcodeList (PhotcodeList, &Nphotcodes, TRUE); // require SEC photcodes + remove_argument (1, &argc, argv); + break; + + default: + fprintf (stderr, "no valid mode selected\n"); + relphot_usage(); + break; + } + if (argc != 1) relphot_usage (); + + // if we supplied a file, we need to load it now + if (TGROUP_FILENAME) { + loadTGroups (TGROUP_FILENAME); + free (TGROUP_FILENAME); + } + + return mode; +} + +// for the given measurement photcode, find the sequence number of +// the equivalent active photcode (-1 if none match) +// XXX this could be optimized by making a lookup table of active equiv codes +int GetActivePhotcodeIndex (int photcode) { + + // first find the matching photcode (our list of active secondary photcodes + // does not necessarily match the list of all secondary photcodes) + int Ns = -1; + int ecode = GetPhotcodeEquivCodebyCode (photcode); + for (int i = 0; (Ns < 0) && (i < Nphotcodes); i++) { + if (photcodes[i][0].code != ecode) continue; + Ns = i; + } + return Ns; +} + +// XXX need to free sky +void relphot_free (SkyTable *sky, SkyList *skylist) { + FREE (UserCatalog); + FREE (OUTROOT); + FREE (UPDATE_CATFORMAT); + + FREE (BOUNDARY_TREE); + FREE (REGION_FILE); + FREE (SYNTH_ZERO_POINTS); + FREE (IMAGE_TABLE); + FREE (CATDIR); + + FREE (PhotcodeList); + FREE (photcodes); + + free_tess(); + SkyTableFree (sky); + SkyListFree(skylist); + FreePhotcodeTable(); + + freeTGroups(); + free_error(); + + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); +} + +int args_client (int argc, char **argv) { + + int N; + double trange; + + // by definition, the client is not parallel + PARALLEL = FALSE; + PARALLEL_MANUAL = FALSE; + PARALLEL_SERIAL = FALSE; + SKIP_PARALLEL_GROUPS = 0; + + HOST_ID = 0; + if ((N = get_argument (argc, argv, "-hostID"))) { + remove_argument (N, &argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOST_ID) relphot_client_usage(); + + HOSTDIR = NULL; + if ((N = get_argument (argc, argv, "-hostdir"))) { + remove_argument (N, &argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOSTDIR) relphot_client_usage(); + + IMAGES = NULL; // used in -update mode + BCATALOG = NULL; // used in -load mode + MODE = MODE_NONE; + if ((N = get_argument (argc, argv, "-load"))) { + MODE = MODE_LOAD; + remove_argument (N, &argc, argv); + BCATALOG = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-update-catalogs"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot mix modes (-load, -update-catalogs, -update-objects)\n"); + relphot_client_usage(); + } + MODE = MODE_UPDATE; + remove_argument (N, &argc, argv); + IMAGES = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-update-objects"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot mix modes (-load, -update-catalogs, -update-objects)\n"); + relphot_client_usage(); + } + MODE = MODE_UPDATE_OBJECTS; + remove_argument (N, &argc, argv); + } + SYNTH_ZERO_POINTS = NULL; + if ((N = get_argument (argc, argv, "-synthphot-zpts"))) { + remove_argument (N, &argc, argv); + myAssert (N < argc, "missing argument to -synthphot-zpts"); + SYNTH_ZERO_POINTS = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-synthphot_means"))) { + MODE = MODE_SYNTH_PHOT; + remove_argument (N, &argc, argv); + SYNTH_ZERO_POINTS = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!MODE) relphot_client_usage(); + + strcpy (STATMODE, "WT_MEAN"); + if ((N = get_argument (argc, argv, "-statmode"))) { + remove_argument (N, &argc, argv); + strcpy (STATMODE, argv[N]); + remove_argument (N, &argc, argv); + } + + BOUNDARY_TREE = NULL; + if ((N = get_argument (argc, argv, "-boundary-tree"))) { + remove_argument (N, &argc, argv); + BOUNDARY_TREE = strcreate(argv[N]); + load_tess (BOUNDARY_TREE); + remove_argument (N, &argc, argv); + } + + USE_ALL_IMAGES = FALSE; + if ((N = get_argument (argc, argv, "-use-all-images"))) { + remove_argument (N, &argc, argv); + USE_ALL_IMAGES = TRUE; + } + + USE_BASIC_CHECK = FALSE; + if ((N = get_argument (argc, argv, "-basic-image-search"))) { + remove_argument (N, &argc, argv); + USE_BASIC_CHECK = TRUE; + } + + /* specify portion of the sky */ + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + IS_DIFF_DB = FALSE; + if ((N = get_argument (argc, argv, "-is-diff-db"))) { + remove_argument (N, &argc, argv); + IS_DIFF_DB = TRUE; + } + + VERBOSE = VERBOSE2 = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-vv"))) { + VERBOSE2 = VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + TEST_IMAGE1 = -1; + TEST_IMAGE2 = -1; + + RESET = FALSE; + if ((N = get_argument (argc, argv, "-reset"))) { + remove_argument (N, &argc, argv); + RESET = TRUE; + } + + RESET_ZEROPTS = FALSE; + if ((N = get_argument (argc, argv, "-reset-zpts"))) { + remove_argument (N, &argc, argv); + RESET_ZEROPTS = TRUE; + } + + PRESERVE_PS1 = FALSE; + if ((N = get_argument (argc, argv, "-preserve-ps1"))) { + remove_argument (N, &argc, argv); + PRESERVE_PS1 = TRUE; + } + + REPAIR_WARPS = FALSE; + if ((N = get_argument (argc, argv, "-repair-warps"))) { + remove_argument (N, &argc, argv); + REPAIR_WARPS = TRUE; + } + + KEEP_UBERCAL = TRUE; + if ((N = get_argument (argc, argv, "-reset-ubercal"))) { + remove_argument (N, &argc, argv); + KEEP_UBERCAL = FALSE; + } + + USE_MCAL_PSF_FOR_STACK_APER = FALSE; + if ((N = get_argument (argc, argv, "-use-mcal-psf-for-stack-aper"))) { + remove_argument (N, &argc, argv); + USE_MCAL_PSF_FOR_STACK_APER = TRUE; + } + + /* define time */ + TimeSelect = FALSE; + if ((N = get_argument (argc, argv, "-time"))) { + remove_argument (N, &argc, argv); + if (!ohana_str_to_time (argv[N], &TSTART)) { + fprintf (stderr, "ERROR: syntax error\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); + if (!ohana_str_to_dtime (argv[N], &trange)) { + if (!ohana_str_to_time (argv[N], &TSTOP)) { + fprintf (stderr, "ERROR: syntax error\n"); + return (FALSE); + } + } else { + if (trange < 0) { + trange = fabs (trange); + TSTOP = TSTART; + TSTART -= trange; + } else { + TSTOP = TSTART + trange; + } + } + remove_argument (N, &argc, argv); + TimeSelect = TRUE; + } + + + MIN_ERROR = 0.001; + if ((N = get_argument (argc, argv, "-minerror"))) { + remove_argument (N, &argc, argv); + MIN_ERROR = atof (argv[N]); + remove_argument (N, &argc, argv); + /* require MIN_ERROR > 0 */ + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + UPDATE_CATFORMAT = NULL; + if ((N = get_argument (argc, argv, "-update-catformat"))) { + remove_argument (N, &argc, argv); + UPDATE_CATFORMAT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + AreaSelect = FALSE; + if ((N = get_argument (argc, argv, "-area"))) { + remove_argument (N, &argc, argv); + AreaXmin = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaXmax = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaYmin = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaYmax = atof (argv[N]); + remove_argument (N, &argc, argv); + AreaSelect = TRUE; + } + + SyntheticPhotometry = FALSE; + if ((N = get_argument (argc, argv, "-synthphot"))) { + remove_argument (N, &argc, argv); + SyntheticPhotometry = FALSE; + init_synthetic_mags(); + } + + ImagSelect = FALSE; + if ((N = get_argument (argc, argv, "-instmag"))) { + remove_argument (N, &argc, argv); + ImagMin = atof (argv[N]); + remove_argument (N, &argc, argv); + ImagMax = atof (argv[N]); + remove_argument (N, &argc, argv); + ImagSelect = TRUE; + } + + DophotSelect = FALSE; + if ((N = get_argument (argc, argv, "-dophot"))) { + remove_argument (N, &argc, argv); + DophotValue = atof (argv[N]); + remove_argument (N, &argc, argv); + DophotSelect = TRUE; + } + + MaxDensityUse = FALSE; + if ((N = get_argument (argc, argv, "-max-density"))) { + remove_argument (N, &argc, argv); + MaxDensityValue = atof(argv[N]); + remove_argument (N, &argc, argv); + MaxDensityUse = TRUE; + } + + STAGES = STAGE_CHIP | STAGE_WARP | STAGE_STACK; + if ((N = get_argument (argc, argv, "-skip-chip"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_CHIP; + } + if ((N = get_argument (argc, argv, "-skip-warp"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_WARP; + } + if ((N = get_argument (argc, argv, "-skip-stack"))) { + remove_argument (N, &argc, argv); + STAGES &= ~STAGE_STACK; + } + if (!STAGES) { + fprintf (stderr, "ERROR: no valid stages selected\n"); + exit (3); + } + + if ((MODE == MODE_SYNTH_PHOT) && (argc == 1)) return TRUE; + if ((MODE == MODE_UPDATE_OBJECTS) && (argc == 1)) return TRUE; + if (argc != 2) relphot_client_usage (); + + return TRUE; +} + +// XXX need to free sky +void relphot_client_free (SkyTable *sky, SkyList *skylist) { + FREE (HOSTDIR); + FREE (BCATALOG); + FREE (IMAGES); + + FREE (SYNTH_ZERO_POINTS); + FREE (BOUNDARY_TREE); + FREE (UPDATE_CATFORMAT); + + FREE (CATDIR); + FREE (PhotcodeList); + FREE (photcodes); + + free_tess(); + SkyTableFree (sky); + SkyListFree(skylist); + FreePhotcodeTable(); + + ohana_memcheck (VERBOSE); + ohana_memdump (VERBOSE); +} + + Index: /branches/eam_branches/relphot.20210521/src/assign_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/assign_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/assign_images.c (revision 41602) @@ -0,0 +1,228 @@ +# include "relphot.h" + +// This function generates a subset of the images based on selections. Input db has already +// been loaded with the raw fits table data +int assign_images (FITS_DB *db, RegionHostTable *regionHosts) { + + off_t Nimage; + + INITTIME; + + // convert database table to internal structure (binary to Image) + // 'image' points to the same memory as db->ftable->buffer + Image *image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + MARKTIME("convert image table to internal structure: %f sec\n", dtime); + + // *** NOTE : for the moment, regions must be in the range 0 - 360, -90 - +90 + + // generate the chip match here so we can define the mosaic centers (if needed) + BuildChipMatch (image, Nimage); + MARKTIME("build chip match for %d images: %f sec\n", (int) Nimage, dtime); + + char mapfile[DVO_MAX_PATH]; + snprintf (mapfile, DVO_MAX_PATH, "%s/AstroMap.fits", CATDIR); + AstromOffsetTable *table = AstromOffsetMapLoad (mapfile, 100000, VERBOSE); + + // assign images.coords.offsetMap -> table->map[i] + if (table) { + AstromOffsetTableMatchChips (image, Nimage, table); + } + put_astrom_table (table); + + if (MOSAIC_ZEROPT) { + makeMosaics (image, Nimage, FALSE); + + // center coords and Mcal, dMcal, Mchisq for the mosaics + setMosaicCenters (image, Nimage); + MARKTIME("set mosaic coordinates and Mcal values: %f sec\n", dtime); + } + + // register the image array with ImageOps.c for later getimageByID calls + initImages (image, NULL, Nimage); + + if (VERBOSE) fprintf (stderr, "finding images\n"); + + // for each regionHost, select images which are contained by the region + // even faster would be to use a tree to get to the real regions... + select_images_hostregion (regionHosts, image, Nimage); + + return TRUE; +} + +# define D_NIMAGE 1000 + +// assign images to the region hosts; at the end, each host will have its list of images +int select_images_hostregion (RegionHostTable *regionHosts, Image *image, off_t Nimage) { + + off_t i, j; + + // INITTIME; + + for (i = 0; i < regionHosts->Nhosts; i++) { + regionHosts->hosts[i].Nimage = 0; + regionHosts->hosts[i].NIMAGE = D_NIMAGE; + ALLOCATE (regionHosts->hosts[i].image, Image, regionHosts->hosts[i].NIMAGE); + } + + for (j = 0; j < Nimage; j++) { + + /* exclude images by photcode */ + int Ns = GetActivePhotcodeIndex (image[j].photcode); + if (Ns < 0) continue; + + /* exclude images by time */ + if (TimeSelect) { + if (image[j].tzero < TSTART) continue; + if (image[j].tzero > TSTOP) continue; + } + + // do not include DIS (PHU-level mosaics) in the output list + if (!strcmp(&image[j].coords.ctype[4], "-DIS")) continue; + + // Exclude images with crazy astrometry + // XXX NOTE : this is gpc1-specific + { + double dP1 = hypot(image[j].coords.pc1_1, image[j].coords.pc1_2); + double dP2 = hypot(image[j].coords.pc2_1, image[j].coords.pc2_2); + if (fabs(dP1 - 1.0) > 0.02) continue; + if (fabs(dP2 - 1.0) > 0.02) continue; + + double X00, Y00, X10, Y10, X01, Y01; + XY_to_LM (&X00, &Y00, 0.0, 0.0, &image[j].coords); + XY_to_LM (&X10, &Y10, image[j].NX, 0.0, &image[j].coords); + XY_to_LM (&X01, &Y01, 0.0, image[j].NY, &image[j].coords); + double dS0 = hypot ((X00 - X10), (Y00 - Y10)); + double dS1 = hypot ((X00 - X01), (Y00 - Y01)); + if (dS0 > 6000) continue; + if (dS1 > 6500) continue; + } + + // use a reference coordinate for each image to assign to hosts + // define image center - note the DIS images (mosaic phu) are special, but we have + // already excluded them above. we also save the image centers for reference + double Xc, Yc; + double Rc, Dc; + + Xc = 0.5*image[j].NX; + Yc = 0.5*image[j].NY; + + XY_to_RD (&Rc, &Dc, Xc, Yc, &image[j].coords); + Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0); + image[j].RAo = Rc; + image[j].DECo = Dc; + + // NOTE: if we are NOT using mosaic centers, we don't need to keep chips with centers + // on opposite sides of 0,360 together. in which case, it is OK for the range of Rc + // to be 0.0 to 360.0. also NOTE: RAo,DECo are only used for debugging reference. + + if (MOSAIC_ZEROPT) { + // use the coords of the associated mosaic to select (only for chips; stacks use their own center) + Mosaic *mosaic = getMosaicForImage (j); + if (mosaic) { + Rc = mosaic->coords.crval1; + Dc = mosaic->coords.crval2; + // NOTE : have defined mosaic Rc,Dc to choose the side of 0,360 on which most of the + // chips are located. but, for host assignment, we rationalize to 0.0 - 360.0 + Rc = ohana_normalize_angle_to_midpoint (Rc, 180.0); + } + } + + i = find_host_for_coords (regionHosts, Rc, Dc); + + if (i == -1) continue; + + RegionHostInfo *host = ®ionHosts->hosts[i]; + + // image bounds are defined for a range centered on the mosaic center thus, a mosaic + // with center 0.5 will have chips bounds ranging from ~ -1.5 to +2.5 or so, while a mosaic + // with center 359.5 will have chips bounds ranging from ~ 357.5 to 361.5 or so + double Rmin, Rmax, Dmin, Dmax; + calculate_image_bounds (&image[j], &Rmin, &Rmax, &Dmin, &Dmax, Rc); + + host->RminCat = MIN(Rmin, host->RminCat); + host->RmaxCat = MAX(Rmax, host->RmaxCat); + host->DminCat = MIN(Dmin, host->DminCat); + host->DmaxCat = MAX(Dmax, host->DmaxCat); + + // regionHosts needs to have the full outer boundary + // (so reload_catalogs covers the correct region) + regionHosts->Rmin = MIN(Rmin, regionHosts->Rmin); + regionHosts->Rmax = MAX(Rmax, regionHosts->Rmax); + regionHosts->Dmin = MIN(Dmin, regionHosts->Dmin); + regionHosts->Dmax = MAX(Dmax, regionHosts->Dmax); + + // this is a bit memory expensive : I am making a complete copy of the image table here + off_t Nsubset = regionHosts->hosts[i].Nimage; + regionHosts->hosts[i].image[Nsubset] = image[j]; + // regionHosts->hosts[i].line_number[Nsubset] = j; + + regionHosts->hosts[i].Nimage ++; + if (regionHosts->hosts[i].Nimage == regionHosts->hosts[i].NIMAGE) { + regionHosts->hosts[i].NIMAGE += D_NIMAGE; + REALLOCATE (regionHosts->hosts[i].image, Image, regionHosts->hosts[i].NIMAGE); + // REALLOCATE (regionHosts->hosts[i].line_number, off_t, regionHosts->hosts[i].NIMAGE); + } + } + + return TRUE; +} + +double Xf[] = {0.0, 1.0, 0.0, 1.0}; +double Yf[] = {0.0, 0.0, 1.0, 1.0}; + +int calculate_image_bounds (Image *image, double *rmin, double *rmax, double *dmin, double *dmax, double Rmid) { + + int n; + + double Rmin = 360.0; + double Rmax = 0.0; + double Dmin = +90.0; + double Dmax = -90.0; + + // define image corners + for (n = 0; n < 4; n++) { + double Xc, Yc, Rc, Dc; + Xc = Xf[n]*image->NX; + Yc = Yf[n]*image->NY; + XY_to_RD (&Rc, &Dc, Xc, Yc, &image->coords); + Rc = ohana_normalize_angle_to_midpoint (Rc, Rmid); + + Rmin = MIN (Rmin, Rc); + Rmax = MAX (Rmax, Rc); + Dmin = MIN (Dmin, Dc); + Dmax = MAX (Dmax, Dc); + } + + *rmin = Rmin; + *rmax = Rmax; + *dmin = Dmin; + *dmax = Dmax; + + return TRUE; +} + +// Rc is in range 0.0 - 360.0, hosts Rmin,Rmax in range 0.0 - 360.0 +int find_host_for_coords (RegionHostTable *regionHosts, double Rc, double Dc) { + + int i; + + if (isnan(Rc)) return -1; + if (isnan(Dc)) return -1; + + for (i = 0; i < regionHosts->Nhosts; i++) { + + RegionHostInfo *host = ®ionHosts->hosts[i]; + if (Rc < host->Rmin) continue; + if (Rc >= host->Rmax) continue; + if (Dc < host->Dmin) continue; + if (Dc >= host->Dmax) continue; + + return i; + } + return -1; +} + Index: /branches/eam_branches/relphot.20210521/src/bcatalog.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/bcatalog.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/bcatalog.c (revision 41602) @@ -0,0 +1,459 @@ +# include "relphot.h" + +int LimitDensityCatalog_ByNmeasure (Catalog *subcatalog, Catalog *oldcatalog); + +int bcatalog (Catalog *subcatalog, Catalog *catalog, int Ncat) { + + off_t i, j, offset; + int found, Ns, *Nvalid; + off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm; + float mag; + int Ncode, Ntime, Ndophot, Nmag, Nsigma, Nimag, Nfew, Ngalaxy, Npsfqf, Nnan, Nbad, Npoor; + + int Nsecfilt = GetPhotcodeNsecfilt (); + assert (Nsecfilt == catalog[0].Nsecfilt); + + /* we are moving only the subset of measurements from catalog[0] to subcatalog[0] */ + memset(subcatalog, 0, sizeof(*subcatalog)); + NAVERAGE = 50; + NMEASURE = 1000; + ALLOCATE (subcatalog[0].measureT, MeasureTiny, NMEASURE); + ALLOCATE (subcatalog[0].averageT, AverageTiny, NAVERAGE); + ALLOCATE (subcatalog[0].secfilt, SecFilt, NAVERAGE*Nsecfilt); + Nmeasure = Naverage = 0; + + int NmMax = 0; + int NmMin = catalog[0].Nmeasure; + + Ncode = Ntime = Ndophot = Nmag = Nsigma = Nimag = Nfew = Npsfqf = Ngalaxy = Nnan = Nbad = Npoor = 0; + + // copy the following fields to the subcatalog: + subcatalog[0].catID = catalog[0].catID; + subcatalog[0].filename = catalog[0].filename; + + // array to count measures with photcodes->equiv to each of the secfilts + ALLOCATE (Nvalid, int, Nsecfilt); + + // flags used by the photometry analysis (excluding UBERCAL) + unsigned int PHOTOM_FLAGS = + ID_MEAS_NOCAL | // detection ignored for this analysis (photcode, time range) + ID_MEAS_POOR_PHOTOM | // detection is photometry outlier + ID_MEAS_SKIP_PHOTOM | // detection was ignored for photometry measurement + ID_MEAS_AREA | // detetion was outside acceptable area of device + ID_MEAS_SYNTH_MAG | // magnitude is synthetic + ID_MEAS_STACK_PRIMARY | // this stack measurement is in the primary skycell + ID_MEAS_STACK_PHOT_SRC | // this measurement supplied the stack photometry + ID_MEAS_PHOTOM_PSF | // this measurement is used for the mean psf mag + ID_MEAS_PHOTOM_APER | // this measurement is used for the mean ap mag + ID_MEAS_PHOTOM_KRON ; // this measurement is used for the mean kron mag + + /* exclude stars not in range or with too few measurements */ + for (i = 0; i < catalog[0].Naverage; i++) { + // if (catalog[0].average[i].Nmeasure < 2) continue; + + /* start with all stars good */ + CopyAverageToTiny (&subcatalog[0].averageT[Naverage], &catalog[0].average[i]); + subcatalog[0].averageT[Naverage].measureOffset = Nmeasure; + + for (j = 0; j < Nsecfilt; j++) { + subcatalog[0].secfilt[Nsecfilt*Naverage+j] = catalog[0].secfilt[Nsecfilt*i+j]; + Nvalid[j] = 0; // reset the Nvalid array for this star + } + + // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images) + if (RESET) { + int Ns; + + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + dvo_secfilt_init (&subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec], SECFILT_RESET_ALL); + } + } + + Nm = 0; + int nEXT = 0; + int nPSF = 0; + for (j = 0; j < catalog[0].average[i].Nmeasure; j++) { + + offset = catalog[0].average[i].measureOffset + j; + + /* select measurements by photcode */ + PhotCode *code = GetPhotcodebyCode (catalog[0].measure[offset].photcode); + if (!code) continue; + found = FALSE; + int Nsec = -1; // Nsec equivalent for the measurement + for (Ns = 0; !found && (Ns < Nphotcodes); Ns++) { + if (code->equiv != photcodes[Ns][0].code) continue; + found = TRUE; + Nsec = GetPhotcodeNsec(code->equiv); + } + if (!found) { + Ncode ++; + continue; + } + + /* select measurements by time */ + if (TimeSelect) { + if (catalog[0].measure[offset].t < TSTART) { Ntime ++; continue; } + if (catalog[0].measure[offset].t > TSTOP) { Ntime ++; continue; } + } + + /* select measurements by quality */ + if (DophotSelect && ((catalog[0].measure[offset].photFlags >> 16) != DophotValue)) { Ndophot ++; continue; } + + // skip garbage measurements + if (isnan(catalog[0].measure[offset].psfQF) || (catalog[0].measure[offset].psfQF < 0.95)) { Npsfqf ++; continue; } + if (isnan(catalog[0].measure[offset].psfQFperf) || (catalog[0].measure[offset].psfQFperf < 0.95)) { Npsfqf ++; continue; } + if (isnan(catalog[0].measure[offset].M)) { Nnan ++; continue; } + if (isnan(catalog[0].measure[offset].Mkron)) { Nnan ++; continue; } + + // require 0x01 in photFlags (fitted with a PSF) + if ((catalog[0].measure[offset].photFlags & 0x01) == 0) { Nbad ++; continue; } + + // very loose cut on PSF - Kron + float Mkp = catalog[0].measure[offset].M - catalog[0].measure[offset].Mkron; + if (fabs(Mkp) > 1.0) { Nbad ++; continue; } + + if (catalog[0].measure[offset].photFlags & code->photomBadMask) { + Nbad++; + continue; + } + if (catalog[0].measure[offset].photFlags & code->photomPoorMask) { Npoor++; continue;} + + // check for galaxies (XXX skip for now) + if (FALSE && !isnan(catalog[0].measure[offset].Map)) { + if (catalog[0].measure[offset].M - catalog[0].measure[offset].Map > 0.15) { + nEXT ++; + } else { + nPSF ++; + } + } + + /* select measurements by mag limit */ + mag = PhotCat (&catalog[0].measure[offset], MAG_CLASS_PSF); + if (mag > MAG_LIM) { Nmag ++; continue; } + + /* select measurements by measurement error */ + if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) { Nsigma ++; continue; } + + /* select measurements by mag limit */ + if (ImagSelect) { + mag = PhotInst (&catalog[0].measure[offset], MAG_CLASS_PSF); + if (mag < ImagMin) { Nimag ++; continue; } + if (mag > ImagMax) { Nimag ++; continue; } + } + + // count this measurement as valid for this secfilt entry + if (Nsec > -1) { + assert (Nsec < Nsecfilt); + Nvalid[Nsec] ++; + } + + CopyMeasureToTiny (&subcatalog[0].measureT[Nmeasure], &catalog[0].measure[offset]); + subcatalog[0].measureT[Nmeasure].dbFlags &= ~ID_MEAS_SKIP_PHOTOM; + subcatalog[0].measureT[Nmeasure].averef = Naverage; + if (RESET) { + // only reset Mcal for measures with a matching image + // do not reset Mcal for ubercal images unless explicitly requested + if (subcatalog[0].measureT[Nmeasure].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { + if (!KEEP_UBERCAL) { + subcatalog[0].measureT[Nmeasure].McalPSF = 0.0; + subcatalog[0].measureT[Nmeasure].McalAPER = 0.0; + subcatalog[0].measureT[Nmeasure].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL; + } + } else { + if (getImageEntry (Nmeasure, Ncat) >= 0) { + subcatalog[0].measureT[Nmeasure].McalPSF = 0.0; + subcatalog[0].measureT[Nmeasure].McalAPER = 0.0; + } + } + subcatalog[0].measureT[Nmeasure].dbFlags &= ~PHOTOM_FLAGS; + } + Nmeasure ++; + Nm ++; + if (Nmeasure == NMEASURE) { + NMEASURE += 1000; + REALLOCATE (subcatalog[0].measureT, MeasureTiny, NMEASURE); + } + myAssert (Nmeasure < NMEASURE, "realloc failure"); + } // end of catalog.average.Nmeasure loop + + // skip object if it is likely to be a galaxy + if (FALSE && (nEXT >= nPSF)) { + Nmeasure -= Nm; + Ngalaxy ++; + continue; + } + + NmMin = MIN (Nm, NmMin); + NmMax = MAX (Nm, NmMax); + + // XXXX test : what checks do I need to make elsewhere to avoid problems here? + if (Nm <= STAR_TOOFEW) { /* enough measurements total? */ + Nmeasure -= Nm; + Nfew ++; + continue; + } + int anySecfiltGood = FALSE; + for (Ns = 0; Ns < Nsecfilt; Ns++) { /* enough measurements in at least one band? */ + if (Nvalid[Ns] <= STAR_TOOFEW) continue; + anySecfiltGood = TRUE; + } + if (!anySecfiltGood) { + Nmeasure -= Nm; + Nfew ++; + continue; + } + + // for w-band photometry (& other cases?) convert gri photometry + // to a synthetic w-band magnitude + add_synthetic_mags (&subcatalog[0].averageT[Naverage], + &subcatalog[0].secfilt[Nsecfilt*Naverage], + &subcatalog[0].measureT[Nmeasure], + &Nmeasure, + &Nm); + if (Nmeasure == NMEASURE) { + NMEASURE += 1000; + REALLOCATE (subcatalog[0].measureT, MeasureTiny, NMEASURE); + } + myAssert (Nmeasure < NMEASURE, "realloc failure"); + + subcatalog[0].averageT[Naverage].Nmeasure = Nm; + Naverage ++; + if (Naverage == NAVERAGE) { + NAVERAGE += 50; + REALLOCATE (subcatalog[0].averageT, AverageTiny, NAVERAGE); + REALLOCATE (subcatalog[0].secfilt, SecFilt, NAVERAGE*Nsecfilt); + } + } + REALLOCATE (subcatalog[0].averageT, AverageTiny, MAX (Naverage, 1)); + REALLOCATE (subcatalog[0].measureT, MeasureTiny, MAX (Nmeasure, 1)); + REALLOCATE (subcatalog[0].secfilt, SecFilt, Nsecfilt*MAX (Naverage, 1)); + subcatalog[0].Naverage = Naverage; + subcatalog[0].Nmeasure = Nmeasure; + subcatalog[0].Nsecfilt = catalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * catalog[0].Nsecfilt; + + if (VERBOSE) { + fprintf (stderr, "using "OFF_T_FMT" stars ("OFF_T_FMT" measures) of "OFF_T_FMT" ("OFF_T_FMT" measures) for catalog %s, %d < Nm < %d\n", + subcatalog[0].Naverage, subcatalog[0].Nmeasure, catalog[0].Naverage, catalog[0].Nmeasure, catalog[0].filename, NmMin, NmMax); + fprintf (stderr, "rejections: %d stars have too few measures:\n %d code, %d time, %d dophot, %d mag, %d sigma, %d imag, %d psfqf, %d Nnan, %d galaxies, %d bad, %d poor\n", + Nfew, Ncode, Ntime, Ndophot, Nmag, Nsigma, Nimag, Npsfqf, Nnan, Ngalaxy, Nbad, Npoor); + } + + // limit the total number of stars in the catalog + if (MaxDensityUse) { + LimitDensityCatalog_ByNmeasure (subcatalog, catalog); + } + + free (Nvalid) + + return (TRUE); +} + +// sort by decreasing Nmeasure (X) +void sort_by_Nmeasure (int *X, off_t *Y, off_t N) { + +# define SWAPFUNC(A,B){ int tmpI; off_t tmpT; \ + tmpI = X[A]; X[A] = X[B]; X[B] = tmpI; \ + tmpT = Y[A]; Y[A] = Y[B]; Y[B] = tmpT; \ +} +# define COMPARE(A,B)(X[A] > X[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +int LimitDensityCatalog_ByNmeasure (Catalog *subcatalog, Catalog *oldcatalog) { + + Catalog tmpcatalog; + + double Rmin, Rmax, Dmin, Dmax; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + gfits_scan (&oldcatalog[0].header, "RA0", "%lf", 1, &Rmin); + gfits_scan (&oldcatalog[0].header, "DEC0", "%lf", 1, &Dmin); + gfits_scan (&oldcatalog[0].header, "RA1", "%lf", 1, &Rmax); + gfits_scan (&oldcatalog[0].header, "DEC1", "%lf", 1, &Dmax); + + if (VERBOSE2) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f\n", Rmin, Dmin, Rmax, Dmax); + + float AREA = fabs(Dmax - Dmin) * fabs(Rmax - Rmin) * cos (0.5*RAD_DEG*(Dmax + Dmin)); + assert (AREA > 0); + + off_t Nmax = MaxDensityValue * AREA; + if (subcatalog[0].Naverage <= Nmax) { + if (VERBOSE) { + fprintf (stderr, "subcatalog has less than the max density\n"); + } + return (TRUE); + } + + off_t Naverage = subcatalog[0].Naverage; + + // select a random subset of Nmax stars from subcatalog using Fisher-Yates + + // we are going to select Nmax entries by generating a random-sorted index list + int *value; + off_t *index, i, j, ave; + ALLOCATE (index, off_t, Naverage); + ALLOCATE (value, int, Naverage); + for (i = 0; i < Naverage; i++) { + index[i] = i; + value[i] = subcatalog[0].averageT[i].Nmeasure; + } + sort_by_Nmeasure (value, index, Naverage); + + // count the number of measurements this selection will yield + off_t NMEASURE = 0; + for (i = 0; i < Nmax; i++) { + ave = index[i]; + NMEASURE += subcatalog[0].averageT[ave].Nmeasure; + } + + // allocate the output data + ALLOCATE (tmpcatalog.averageT, AverageTiny, Nmax); + ALLOCATE (tmpcatalog.measureT, MeasureTiny, NMEASURE); + ALLOCATE (tmpcatalog.secfilt, SecFilt, Nmax * Nsecfilt); + + off_t Nmeasure = 0; + + // copy the Nmax selected entries from subcatalog to tmpcatalog (adjusting links) + for (i = 0; i < Nmax; i++) { + ave = index[i]; + tmpcatalog.averageT[i] = subcatalog[0].averageT[ave]; + tmpcatalog.averageT[i].measureOffset = Nmeasure; + for (j = 0; j < tmpcatalog.averageT[i].Nmeasure; j++) { + off_t offset = subcatalog[0].averageT[ave].measureOffset + j; + tmpcatalog.measureT[Nmeasure] = subcatalog[0].measureT[offset]; + tmpcatalog.measureT[Nmeasure].averef = i; + Nmeasure ++; + } + for (j = 0; j < Nsecfilt; j++) { + tmpcatalog.secfilt[i*Nsecfilt + j] = subcatalog[0].secfilt[ave*Nsecfilt + j]; + } + } + + if (VERBOSE) { + char *basename = filebasename (oldcatalog[0].filename); + fprintf (stderr, "limited to "OFF_T_FMT" ("OFF_T_FMT" subset, "OFF_T_FMT" total) stars, "OFF_T_FMT" ("OFF_T_FMT" subset, "OFF_T_FMT" total) measures for catalog %s\n", + Nmax, subcatalog[0].Naverage, oldcatalog[0].Naverage, Nmeasure, subcatalog[0].Nmeasure, oldcatalog[0].Nmeasure, basename); + free (basename); + } + + free (index); + free (value); + free (subcatalog[0].averageT); + free (subcatalog[0].measureT); + free (subcatalog[0].secfilt); + + subcatalog[0].averageT = tmpcatalog.averageT; + subcatalog[0].measureT = tmpcatalog.measureT; + subcatalog[0].secfilt = tmpcatalog.secfilt; + subcatalog[0].Naverage = Nmax; + subcatalog[0].Nmeasure = Nmeasure; + subcatalog[0].Nsecfilt = oldcatalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * oldcatalog[0].Nsecfilt; + + return (TRUE); +} + +int LimitDensityCatalog (Catalog *subcatalog, Catalog *catalog) { + + Catalog tmpcatalog; + + double Rmin, Rmax, Dmin, Dmax; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + gfits_scan (&catalog[0].header, "RA0", "%lf", 1, &Rmin); + gfits_scan (&catalog[0].header, "DEC0", "%lf", 1, &Dmin); + gfits_scan (&catalog[0].header, "RA1", "%lf", 1, &Rmax); + gfits_scan (&catalog[0].header, "DEC1", "%lf", 1, &Dmax); + + if (VERBOSE2) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f\n", Rmin, Dmin, Rmax, Dmax); + + float AREA = fabs(Dmax - Dmin) * fabs(Rmax - Rmin) * cos (0.5*RAD_DEG*(Dmax + Dmin)); + assert (AREA > 0); + + off_t Nmax = MaxDensityValue * AREA; + if (subcatalog[0].Naverage <= Nmax) { + if (VERBOSE) { + fprintf (stderr, "subcatalog has less than the max density\n"); + } + return (TRUE); + } + + off_t Naverage = subcatalog[0].Naverage; + + // select a random subset of Nmax stars from subcatalog using Fisher-Yates + + // we are going to select Nmax entries by generating a random-sorted index list + off_t *index, tmp, i, j, ave; + ALLOCATE (index, off_t, Naverage); + for (i = 0; i < Naverage; i++) { + index[i] = i; + } + for (i = 0; i < Naverage; i++) { + j = (Naverage - i) * drand48() + i; // a number between i and Naverage + tmp = index[j]; + index[j] = index[i]; + index[i] = tmp; + } + + // count the number of measurements this selection will yield + off_t NMEASURE = 0; + for (i = 0; i < Nmax; i++) { + ave = index[i]; + NMEASURE += subcatalog[0].averageT[ave].Nmeasure; + } + + // allocate the output data + ALLOCATE (tmpcatalog.averageT, AverageTiny, Nmax); + ALLOCATE (tmpcatalog.measureT, MeasureTiny, NMEASURE); + ALLOCATE (tmpcatalog.secfilt, SecFilt, Nmax * Nsecfilt); + + off_t Nmeasure = 0; + + // copy the Nmax selected entries from subcatalog to tmpcatalog (adjusting links) + for (i = 0; i < Nmax; i++) { + ave = index[i]; + tmpcatalog.averageT[i] = subcatalog[0].averageT[ave]; + tmpcatalog.averageT[i].measureOffset = Nmeasure; + for (j = 0; j < tmpcatalog.averageT[i].Nmeasure; j++) { + off_t offset = subcatalog[0].averageT[ave].measureOffset + j; + tmpcatalog.measureT[Nmeasure] = subcatalog[0].measureT[offset]; + tmpcatalog.measureT[Nmeasure].averef = i; + Nmeasure ++; + } + for (j = 0; j < Nsecfilt; j++) { + tmpcatalog.secfilt[i*Nsecfilt + j] = subcatalog[0].secfilt[ave*Nsecfilt + j]; + } + } + + if (VERBOSE) { + fprintf (stderr, "limited to "OFF_T_FMT" of "OFF_T_FMT" stars ("OFF_T_FMT" of "OFF_T_FMT" measures) for catalog %s\n", + Nmax, subcatalog[0].Naverage, Nmeasure, subcatalog[0].Nmeasure, catalog[0].filename); + } + + free (subcatalog[0].averageT); + free (subcatalog[0].measureT); + free (subcatalog[0].secfilt); + + subcatalog[0].averageT = tmpcatalog.averageT; + subcatalog[0].measureT = tmpcatalog.measureT; + subcatalog[0].secfilt = tmpcatalog.secfilt; + subcatalog[0].Naverage = Nmax; + subcatalog[0].Nmeasure = Nmeasure; + subcatalog[0].Nsecfilt = catalog[0].Nsecfilt; + subcatalog[0].Nsecfilt_mem = Naverage * catalog[0].Nsecfilt; + + return (TRUE); +} Index: /branches/eam_branches/relphot.20210521/src/client_logger.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/client_logger.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/client_logger.c (revision 41602) @@ -0,0 +1,39 @@ +# include "relphot.h" + +// I'm getting unlogged errors and failures. I need a log ouput that the clients can +// write independent of the master + +static FILE *logfile = NULL; +int client_logger_init (char *dirname) { + + char filename[DVO_MAX_PATH]; + + snprintf (filename, DVO_MAX_PATH, "%s/log.rlpc.XXXXXX", dirname); + + int fd = mkstemp (filename); + if (fd == -1) { + fprintf (stderr, "failed to open client logger %s, exiting\n", filename); + exit (50); + } + + logfile = fdopen (fd, "w"); + if (!logfile) { + fprintf (stderr, "failed to fdopen client logger, exiting\n"); + exit (51); + } + return TRUE; +} + +int client_logger_message (char *format,...) { + + if (!logfile) return FALSE; + + va_list argp; + + va_start (argp, format); + vfprintf (logfile, format, argp); + va_end (argp); + + fflush (logfile); + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/extra.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/extra.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/extra.c (revision 41602) @@ -0,0 +1,168 @@ +# include "relphot.h" + +int isMosaicChip (int photcode) { + + if (isGPC1chip(photcode)) return TRUE; + if (isGPC2chip(photcode)) return TRUE; + if (isHSCchip(photcode)) return TRUE; + if (isCFHchip(photcode)) return TRUE; // this is megacam + return FALSE; +} + +// for now (20140710) I need to identify gpc1 chips explicitly. generalize in the future +// note that the 4000, 14000, 15000 sets are SIMTEST (*not* synthetic) +int whichGPC1filter (int photcode) { + +# if (1) + // disable the corner chips: + if (((photcode > 10000) && (photcode < 10077)) || (photcode == 4100)) return PS1_g; // g-band + if (((photcode > 10100) && (photcode < 10177)) || (photcode == 4200)) return PS1_r; // r-band + if (((photcode > 10200) && (photcode < 10277)) || (photcode == 4300)) return PS1_i; // i-band + if (((photcode > 10300) && (photcode < 10377)) || (photcode == 4400)) return PS1_z; // z-band + if (((photcode > 10400) && (photcode < 10477)) || (photcode == 4500)) return PS1_y; // y-band + if (((photcode > 10500) && (photcode < 10577)) || (photcode == 4600)) return PS1_w; // w-band +# else + // for testing, enable the corner chips: + if (((photcode >= 10000) && (photcode <= 10077)) || (photcode == 4100)) return PS1_g; // g-band + if (((photcode >= 10100) && (photcode <= 10177)) || (photcode == 4200)) return PS1_r; // r-band + if (((photcode >= 10200) && (photcode <= 10277)) || (photcode == 4300)) return PS1_i; // i-band + if (((photcode >= 10300) && (photcode <= 10377)) || (photcode == 4400)) return PS1_z; // z-band + if (((photcode >= 10400) && (photcode <= 10477)) || (photcode == 4500)) return PS1_y; // y-band + if (((photcode >= 10500) && (photcode <= 10577)) || (photcode == 4600)) return PS1_w; // w-band +# endif + return PS1_none; +} + +// for now (20140710) I need to identify gpc1 chips explicitly. generalize in the future +int isGPC1chip (int photcode) { + +# if (1) + if (((photcode > 10000) && (photcode < 10077)) || (photcode == 4100)) return TRUE; // g-band + if (((photcode > 10100) && (photcode < 10177)) || (photcode == 4200)) return TRUE; // r-band + if (((photcode > 10200) && (photcode < 10277)) || (photcode == 4300)) return TRUE; // i-band + if (((photcode > 10300) && (photcode < 10377)) || (photcode == 4400)) return TRUE; // z-band + if (((photcode > 10400) && (photcode < 10477)) || (photcode == 4500)) return TRUE; // y-band + if (((photcode > 10500) && (photcode < 10577)) || (photcode == 4600)) return TRUE; // w-band +# else + if (((photcode >= 10000) && (photcode <= 10077)) || (photcode == 4100)) return TRUE; // g-band + if (((photcode >= 10100) && (photcode <= 10177)) || (photcode == 4200)) return TRUE; // r-band + if (((photcode >= 10200) && (photcode <= 10277)) || (photcode == 4300)) return TRUE; // i-band + if (((photcode >= 10300) && (photcode <= 10377)) || (photcode == 4400)) return TRUE; // z-band + if (((photcode >= 10400) && (photcode <= 10477)) || (photcode == 4500)) return TRUE; // y-band + if (((photcode >= 10500) && (photcode <= 10577)) || (photcode == 4600)) return TRUE; // w-band +# endif + + return FALSE; +} + +// for now, I need to identify gpc2 chips explicitly. generalize in the future +int isGPC2chip (int photcode) { + + if ((photcode >= 30000) && (photcode <= 30077)) return TRUE; // g-band + if ((photcode >= 30100) && (photcode <= 30177)) return TRUE; // r-band + if ((photcode >= 30200) && (photcode <= 30277)) return TRUE; // i-band + if ((photcode >= 30300) && (photcode <= 30377)) return TRUE; // z-band + if ((photcode >= 30400) && (photcode <= 30477)) return TRUE; // y-band + if ((photcode >= 30500) && (photcode <= 30577)) return TRUE; // w-band + + return FALSE; +} + +// for now (20140710) I need to identify gpc1 stacks explicitly. generalize in the future +int isGPC1stack (int photcode) { + + if ((photcode == 11000) || (photcode == 14100)) return TRUE; // g-band + if ((photcode == 11100) || (photcode == 14200)) return TRUE; // r-band + if ((photcode == 11200) || (photcode == 14300)) return TRUE; // i-band + if ((photcode == 11300) || (photcode == 14400)) return TRUE; // z-band + if ((photcode == 11400) || (photcode == 14500)) return TRUE; // y-band + if ((photcode == 11500) || (photcode == 14600)) return TRUE; // w-band + + return FALSE; +} + +// for now (20140710) I need to identify gpc1 stacks explicitly. generalize in the future +int isGPC1warp (int photcode) { + + // diff warps get stack-like photcodes (kind of lame) + if (IS_DIFF_DB) { + if ((photcode == 11000) || (photcode == 14100)) return TRUE; // g-band + if ((photcode == 11100) || (photcode == 14200)) return TRUE; // r-band + if ((photcode == 11200) || (photcode == 14300)) return TRUE; // i-band + if ((photcode == 11300) || (photcode == 14400)) return TRUE; // z-band + if ((photcode == 11400) || (photcode == 14500)) return TRUE; // y-band + if ((photcode == 11500) || (photcode == 14600)) return TRUE; // w-band + return FALSE; + } + + if ((photcode == 12000) || (photcode == 15100)) return TRUE; // g-band + if ((photcode == 12100) || (photcode == 15200)) return TRUE; // r-band + if ((photcode == 12200) || (photcode == 15300)) return TRUE; // i-band + if ((photcode == 12300) || (photcode == 15400)) return TRUE; // z-band + if ((photcode == 12400) || (photcode == 15500)) return TRUE; // y-band + if ((photcode == 12500) || (photcode == 15600)) return TRUE; // w-band + + return FALSE; +} + +int isGPC1synth (int photcode) { + + if ((photcode >= 3001) && (photcode <= 3006)) return TRUE; // g-band + + return FALSE; +} + +int is2MASS (int photcode) { + + if ((photcode >= 2011) && (photcode <= 2013)) return TRUE; + return FALSE; +} + +int isTYCHO (int photcode) { + + if ((photcode == 2020) || (photcode == 2021)) return TRUE; + return FALSE; +} + +// for now (20160925) I need to identify HSC chips explicitly. generalize in the future +int isHSCchip (int photcode) { + + if ((photcode >= 20000) && (photcode <= 20111)) return TRUE; // g-band + if ((photcode >= 21000) && (photcode <= 21111)) return TRUE; // r-band + if ((photcode >= 22000) && (photcode <= 22111)) return TRUE; // i-band + if ((photcode >= 23000) && (photcode <= 23111)) return TRUE; // z-band + if ((photcode >= 24000) && (photcode <= 24111)) return TRUE; // y-band + + return FALSE; +} + +// for now (20160925) I need to identify CFH chips explicitly. generalize in the future +int isCFHchip (int photcode) { + + if ((photcode >= 100) && (photcode <= 152)) return TRUE; // g-band + if ((photcode >= 200) && (photcode <= 252)) return TRUE; // r-band + if ((photcode >= 300) && (photcode <= 352)) return TRUE; // i-band + if ((photcode >= 400) && (photcode <= 452)) return TRUE; // z-band + if ((photcode >= 500) && (photcode <= 552)) return TRUE; // y-band + + return FALSE; +} + +double weight_cauchy (double x) { + double r = x / 2.385; + return (1.0 / (1.0 + SQ(r))); +} + +double VectorFractionInterpolate (double *values, float fraction, int Npts) { + + float F = fraction * Npts; + int N = fraction * Npts; + + if (N < 0 ) return NAN; + if (N >= Npts - 2) return NAN; + + // interpolate between N,N+1 + + double S = (F - N) * (values[N+1] - values[N]) + values[N]; + return S; +} Index: /branches/eam_branches/relphot.20210521/src/fit1d_irls.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/fit1d_irls.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/fit1d_irls.c (revision 41602) @@ -0,0 +1,387 @@ +# include "relphot.h" + +// These should probably be tunable: +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 +# define WEIGHT_THRESHOLD 0.3 + +# define IS_OLS TRUE + +void fit1d_copy_results (double **srcArray, double **tgtArray, int entry, int nterm); +void get_yfit (FitDataSet *dataset, int Npoints); +int fit1d_least_squares (FitDataSet *dataset, FitDataType *data, int isOLS, int Npoints); + +// 1D fit with arbitrary order. +// NOTE: the number of points considered for the fit (Npoints) +// may be different from the number of points stored in the array +int fit1d_irls (FitDataSet *dataset, int Npoints) { + + // initialize various things: + dataset->Nmeas = 0; + dataset->min = NAN; + dataset->max = NAN; + dataset->sigma = NAN; + dataset->chisq = NAN; + for (int i = 0; i < dataset->nterm; i++) { + dataset->bSigma[i] = 0; + } + + // initial fit with OLS, results in dataset->bArray + if (!fit1d_least_squares (dataset, dataset->alldata, IS_OLS, Npoints)) return FALSE; + + // apply fit to data and save in yFitVector & yOffVector + get_yfit (dataset, Npoints); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < dataset->MaxIterations); iterations++) { + + for (int i = 0; i < Npoints; i++) { + // we are only including the formal error, not the weight in the definition of wt[] + dataset->wtIRLS[i] = weight_cauchy (dataset->yOffVector[i] / dataset->alldata->dyVector[i]); + } + + // save this solution to entry 0 + fit1d_copy_results (dataset->bArray, dataset->bSaveArray, 0, dataset->nterm); + + // recalculate the least squares, but now apply the modified weight above (arg 3 = isOLS) + if (!fit1d_least_squares (dataset, dataset->alldata, !IS_OLS, Npoints)) { + // restore the saved solution + fit1d_copy_results (dataset->bSaveArray, dataset->bArray, 0, dataset->nterm); + break; + } + + // apply fit to data and save in yFitVector & yOffVector (used above to calculate wtIRLS + get_yfit (dataset, Npoints); + + // has the fit changed in a meaningful way? + converged = TRUE; + for (int i = 0; i < dataset->nterm; i++) { + if ((fabs(dataset->bArray[i][0] - dataset->bSaveArray[i][0]) > FIT_TOLERANCE * fabs(dataset->bArray[i][0])) && + (fabs(dataset->bArray[i][0] - dataset->bSaveArray[i][0]) > FLT_TOLERANCE)) + converged = FALSE; + } + } + + // save this solution to entry 0 + fit1d_copy_results (dataset->bArray, dataset->bSaveArray, 0, dataset->nterm); + + // calculate the weight thresholds to mask the bad points: + for (int i = 0; i < Npoints; i++) { + dataset->wtIRLS[i] = weight_cauchy (dataset->yOffVector[i] / dataset->alldata->dyVector[i]); + dataset->tmpVector[i] = dataset->wtIRLS[i]; + } + int midpt = 0.5 * Npoints; + dsort (dataset->tmpVector, Npoints); + double WtMedian = (Npoints % 2) ? dataset->tmpVector[midpt] : 0.5*(dataset->tmpVector[midpt] + dataset->tmpVector[midpt-1]); + double WtThreshold = WEIGHT_THRESHOLD * WtMedian; + + // points with too low weight are ignored in the bootstrap analysis below + int Nkeep = 0; + double dChi = 0.0; + double dSig = 0.0; + for (int i = 0; i < Npoints; i++) { + if ((dataset->wtIRLS[i] < WtThreshold) || !isfinite(dataset->alldata->yVector[i])) { + continue; // skip the masked points + } + // save values with sufficient weight for error analysis below (either basic OLS or bootstrap) + dataset->keepdata->yVector[Nkeep] = dataset->alldata->yVector[i]; + dataset->keepdata->dyVector[Nkeep] = dataset->alldata->dyVector[i]; + if (dataset->alldata->xVector) { + dataset->keepdata->xVector[Nkeep] = dataset->alldata->xVector[i]; // only defined if order > 0 + } + Nkeep ++; + + // record min, max, chisq, sigma, Nmeas for the unmasked points + dataset->min = isfinite(dataset->min) ? MIN(dataset->alldata->yVector[i], dataset->min) : dataset->alldata->yVector[i]; + dataset->max = isfinite(dataset->max) ? MAX(dataset->alldata->yVector[i], dataset->max) : dataset->alldata->yVector[i]; + // fprintf (stderr, "%d : %f %f %f\n", i, dataset->alldata->yVector[i], dataset->min, dataset->max); + double dValue2 = SQ(dataset->yOffVector[i]); + dChi += dValue2 / SQ (dataset->alldata->dyVector[i]); + dSig += dValue2; + } + dataset->Nmeas = Nkeep; + dataset->chisq = dChi / (Nkeep - 1); + dataset->sigma = sqrt (dSig / (Nkeep - 1)); + + if (dataset->Nbootstrap) { + int Nboot = 0; + for (int iboot = 0; iboot < dataset->Nbootstrap; iboot++) { + + // resample + for (int i = 0; i < Nkeep; i++) { + // I need to draw Npoints random entries from 'points' with replacement: + int N = Nkeep * drand48(); + dataset->sample->yVector[i] = dataset->keepdata-> yVector[N]; + dataset->sample->dyVector[i] = dataset->keepdata->dyVector[N]; + if (dataset->keepdata->xVector) { + dataset->sample->xVector[i] = dataset->keepdata->xVector[N]; // only defined if order > 0 + } + } + + if (!fit1d_least_squares (dataset, dataset->sample, IS_OLS, Nkeep)) continue; + // fprintf (stderr, "%d = %f : ", iboot, dataset->bArray[0][0]); + + // save this solution in entry Nboot + fit1d_copy_results (dataset->bArray, dataset->bBootArray, Nboot, dataset->nterm); + // fprintf (stderr, "%d = %f : ", Nboot, dataset->bBootArray[0][Nboot]); + Nboot ++; + } + + // loop over the nterm values + for (int i = 0; i < dataset->nterm; i++) { + // copy the bBootArray values to a tmp array + for (int j = 0; j < Nboot; j++) { + dataset->tmpVector[j] = dataset->bBootArray[i][j]; + } + dsort (dataset->tmpVector, Nboot); + double Slo = VectorFractionInterpolate (dataset->tmpVector, 0.158655, Nboot); + double Shi = VectorFractionInterpolate (dataset->tmpVector, 0.841345, Nboot); + dataset->bSigma[i] = (Shi - Slo) / 2.0; + } + } + + // calculate the formal fit error from the covariance matrix. if we do not do + // bootstrap, this is the reported parameter error. Even if we do use bootstrap, + // bootstrap with few values can sometimes yield an excessively-optimistic result for + // the error. Do not let the reported error be smaller than the formal error + if (fit1d_least_squares (dataset, dataset->keepdata, IS_OLS, Nkeep)) { + for (int i = 0; i < dataset->nterm; i++) { + double errvalue = sqrt(dataset->cArray[i][i]); + dataset->bSigma[i] = MAX (dataset->bSigma[i], errvalue); + } + } + return TRUE; +} + +// full least-squares fit function including weights +// results are returned to dataset->bArray +int fit1d_least_squares (FitDataSet *dataset, FitDataType *data, int isOLS, int Npoints) { + + // these are preallocated so repeated evaluations are not slowed down by alloc + double *s = dataset->sumVector; + double **b = dataset->bArray; + double **c = dataset->cArray; + + int nterm = dataset->nterm; // number of parameters being fitted, e.g., y = C0 + C1*x -> 1st order, 2 terms + int mterm = dataset->mterm; // number of x^n terms calculated (e.g., for 3rd order polynomial, we need x^0 - x^6, i.e., 7 x^n terms) + + double *x = data->xVector; + double *y = data->yVector; + double *dy = data->dyVector; + + // XXX : the wtIRLS vector has the same sequence as the data vectors + double *wt = isOLS ? NULL : dataset->wtIRLS; // use IRLS weight if requested + + // initialize summation registers for current pass + memset (s, 0, mterm*sizeof(double)); + for (int i = 0; i < nterm; i++) { + memset (c[i], 0, nterm*sizeof(double)); + memset (b[i], 0, sizeof(double)); + } + + // perform linear fit + for (int i = 0; i < Npoints; i++, y++, dy++) { + if (!finite(*y)) goto next; // skip invalid (NAN) data + if (x && !finite(*x)) goto next; // if we are fitting x, skip invalid (NAN) data + + // XXX in the opihi version, I combine weights; in the relphot version I have a second weight vector to apply + double dY = wt ? *wt / SQ(*dy) : 1.0 / SQ(*dy); + + double X = 1*dY; + double Y = *y*dY; + + for (int j = 0; j < nterm; j++) { + s[j] += X; + b[j][0] += Y; + if (x) { + // x is NULL is 0-order fit + X = X * (*x); + Y = Y * (*x); + } + } + for (int j = nterm; j < mterm; j++) { + s[j] += X; + if (x) { X = X * (*x); } + } + next: + // these elements are optional and NULL if not used: + if (x) { x++; } + if (wt) { wt++; } + } + for (int i = 0; i < nterm; i++) { + for (int j = 0; j < nterm; j++) { + c[i][j] = s[i + j]; + if (dataset->bPriorValue && (i == j)) { + if (!isnan(dataset->bPriorValue[i])) { + c[i][i] += 1.0 / SQ(dataset->bPriorSigma[i]); + b[i][0] += dataset->bPriorValue[i] / SQ(dataset->bPriorSigma[i]); + } + } + } + } + + if (!dgaussjordan (c, b, nterm, 1)) { + gprint (GP_ERR, "failed to fit data : ill-conditioned matrix\n"); + return FALSE; + } + return TRUE; +} + +void get_yfit (FitDataSet *dataset, int Npoints) { + + double *x = dataset->alldata->xVector; + double *y = dataset->alldata->yVector; + + // XXX: keep this or not? + // double *yfit = dataset->yFitVector; + double *yoff = dataset->yOffVector; + + int nterm = dataset->nterm; + double **b = dataset->bArray; + + /* generate fitted values */ + for (int i = 0; i < Npoints; i++, y++, yoff++) { + if (x && !finite(*x)) continue; + double yfit = 0; + double X = 1; + for (int j = 0; j < nterm; j++) { + yfit += b[j][0]*X; + if (x) { X = X * (*x); } + } + *yoff = *y - yfit; + if (x) { x ++; } + } + return; +} + +void fit1d_copy_results (double **srcArray, double **tgtArray, int entry, int nterm) { + for (int i = 0; i < nterm; i++) { + tgtArray[i][entry] = srcArray[i][0]; + } + // fprintf (stderr, "%d = %f -> %f: ", entry, srcArray[0][0], tgtArray[0][entry]); +} + +// allocate the vectors for this FitDataType entry (if order == 0, xVector = NULL) +FitDataType *FitDataTypeAlloc (int order, int N) { + + ALLOCATE_PTR (data, FitDataType, 1); + + ALLOCATE (data->yVector, double, N); + ALLOCATE (data->dyVector, double, N); + if (order) { + ALLOCATE (data->xVector, double, N); + } else { + data->xVector = NULL; + } + return data; +} + +// allocate the vectors for this FitDataType entry (if order == 0, xVector = NULL) +void FitDataTypeFree (FitDataType *data) { + if (!data) return; + FREE (data->yVector); + FREE (data->dyVector); + FREE (data->xVector); + FREE (data); +} + +void FitDataSetAlloc (FitDataSet *dataset, int Nmax, int order, int Nbootstrap) { + + dataset->order = order; + dataset->nterm = order + 1; + dataset->mterm = 2*order + 1; + dataset->Nbootstrap = Nbootstrap; + dataset->MaxIterations = 10; + + // arrays for actual data values + dataset->alldata = FitDataTypeAlloc (order, Nmax); + + // subset arrays to calculate formal errors + dataset->keepdata = FitDataTypeAlloc (order, Nmax); + + dataset->sample = NULL; + if (Nbootstrap) { + // arrays for bootstrap samples + dataset->sample = FitDataTypeAlloc (order, Nmax); + } + + // allocate internal vectors and arrays + ALLOCATE (dataset->bSigma, double, dataset->nterm); + ALLOCATE (dataset->sumVector, double, dataset->mterm); + ALLOCATE (dataset->cArray, double *, dataset->nterm); + ALLOCATE (dataset->bArray, double *, dataset->nterm); + ALLOCATE (dataset->bSaveArray, double *, dataset->nterm); + ALLOCATE (dataset->bBootArray, double *, dataset->nterm); + for (int i = 0; i < dataset->nterm; i++) { + ALLOCATE (dataset->cArray[i], double, dataset->nterm); + ALLOCATE (dataset->bArray[i], double, 1); + ALLOCATE (dataset->bSaveArray[i], double, 1); + ALLOCATE (dataset->bBootArray[i], double, Nbootstrap); + } + + ALLOCATE (dataset->yOffVector, double, Nmax); + ALLOCATE (dataset->tmpVector, double, MAX(Nmax, Nbootstrap)); + ALLOCATE (dataset->wtIRLS, double, Nmax); + + // set to NULL; user can activate + dataset->bPriorValue = NULL; + dataset->bPriorSigma = NULL; +} + +void FitDataSetFree (FitDataSet *dataset) { + + FREE (dataset->bPriorValue); + FREE (dataset->bPriorSigma); + FREE (dataset->wtIRLS); + FREE (dataset->tmpVector); + FREE (dataset->yOffVector); + + FitDataTypeFree (dataset->alldata); + FitDataTypeFree (dataset->keepdata); + if (dataset->Nbootstrap) { + FitDataTypeFree (dataset->sample); + } + + // allocate internal vectors and arrays + for (int i = 0; i < dataset->nterm; i++) { + FREE (dataset->cArray[i]); + FREE (dataset->bArray[i]); + FREE (dataset->bSaveArray[i]); + FREE (dataset->bBootArray[i]); + } + FREE (dataset->sumVector); + FREE (dataset->cArray); + FREE (dataset->bArray); + FREE (dataset->bSigma); + FREE (dataset->bSaveArray); + FREE (dataset->bBootArray); +} + +void FitDataSetAddPriors (FitDataSet *dataset) { + + ALLOCATE (dataset->bPriorValue, double, dataset->nterm); + ALLOCATE (dataset->bPriorSigma, double, dataset->nterm); + + for (int i = 0; i < dataset->nterm; i++) { + dataset->bPriorValue[i] = NAN; + dataset->bPriorSigma[i] = NAN; + } +} + +StatType FitDataSetSoften (FitDataSet *dataset, int Nvalues) { + + // use liststats to find the 20-pct, median, 80-pct points + StatType stats; + liststats_setmode (&stats, "MEDIAN"); + liststats (dataset->alldata->yVector, NULL, NULL, Nvalues, &stats); + + double altSigma = (stats.Upper80 - stats.Lower20) / 1.6; // 20% to 80% encompasses 60% of the values, corresponds to the range (-0.85 sigma : +0.85 sigma) + + // soften the individual errors with 10% of the scatter above + for (int j = 0; j < Nvalues; j++) { + double newSigma = hypot(dataset->alldata->dyVector[j], 0.1*altSigma); + dataset->alldata->dyVector[j] = newSigma; + } + + return stats; +} Index: /branches/eam_branches/relphot.20210521/src/global_stats.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/global_stats.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/global_stats.c (revision 41602) @@ -0,0 +1,74 @@ +# include "relphot.h" + +void global_stats (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr, int nloop) { + + StatType stN, stX, stS, imN, imX, imM, imD, msM, msX, msN, msD, tgM, tgX, tgD; + + liststats_init (&stN); + liststats_init (&stX); + liststats_init (&stS); + + liststats_init (&imN); + liststats_init (&imX); + liststats_init (&imM); + liststats_init (&imD); + + liststats_init (&msN); + liststats_init (&msX); + liststats_init (&msM); + liststats_init (&msD); + + liststats_init (&tgX); + liststats_init (&tgM); + liststats_init (&tgD); + + fprintf (stderr, "\n"); + fprintf (stderr, "STATS median mean sigma min max Nmeas\n"); + + int Ns; + for (Ns = 0; (nloop % 3 == 2) && (Ns < Nphotcodes); Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + int seccode = photcodes[Ns][0].code; + + stN = statsStarN (catalog, Ncatalog, Nsec, seccode, flatcorr); + stX = statsStarX (catalog, Ncatalog, Nsec); + stS = statsStarS (catalog, Ncatalog, Nsec); + + fprintf (stderr, " --- stats for %s ---\n", photcodes[Ns][0].name); + fprintf (stderr, "meas / star: %7.0f %7.1f %7.1f %7.0f %7.0f %6d\n", stN.median, stN.mean, stN.sigma, stN.min, stN.max, stN.Nmeas); + fprintf (stderr, "dMrel star: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", stS.median, stS.mean, stS.sigma, stS.min, stS.max, stS.Nmeas); + fprintf (stderr, "chisq star: %7.1f %7.1f %7.1f %7.1f %7.1f %6d\n", stX.median, stX.mean, stX.sigma, stX.min, stX.max, stX.Nmeas); + } + + imN = statsImageN (catalog, flatcorr); + imX = statsImageX (catalog); + imM = statsImageM (catalog); + imD = statsImagedM (catalog); + + msN = statsMosaicN (catalog, flatcorr); + msM = statsMosaicM (catalog); + msD = statsMosaicdM (catalog); + msX = statsMosaicX (catalog); + + tgM = statsMosaicM (catalog); + tgD = statsMosaicdM (catalog); + tgX = statsMosaicX (catalog); + + fprintf (stderr, "meas / image: %7.0f %7.0f %7.0f %7.0f %7.0f %6d\n", imN.median, imN.mean, imN.sigma, imN.min, imN.max, imN.Nmeas); + fprintf (stderr, "Mcal image: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", imM.median, imM.mean, imM.sigma, imM.min, imM.max, imM.Nmeas); + fprintf (stderr, "dMcal image: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", imD.median, imD.mean, imD.sigma, imD.min, imD.max, imD.Nmeas); + fprintf (stderr, "chisq image: %7.1f %7.1f %7.1f %7.1f %7.1f %6d\n", imX.median, imX.mean, imX.sigma, imX.min, imX.max, imX.Nmeas); + + fprintf (stderr, "meas / mosaic: %7.0f %7.0f %7.0f %7.0f %7.0f %6d\n", msN.median, msN.mean, msN.sigma, msN.min, msN.max, msN.Nmeas); + fprintf (stderr, "Mcal mosaic: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", msM.median, msM.mean, msM.sigma, msM.min, msM.max, msM.Nmeas); + fprintf (stderr, "dMcal mosaic: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", msD.median, msD.mean, msD.sigma, msD.min, msD.max, msD.Nmeas); + fprintf (stderr, "chisq mosaic: %7.1f %7.1f %7.1f %7.1f %7.1f %6d\n", msX.median, msX.mean, msX.sigma, msX.min, msX.max, msX.Nmeas); + + fprintf (stderr, "Mcal tgroup: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", tgM.median, tgM.mean, tgM.sigma, tgM.min, tgM.max, tgM.Nmeas); + fprintf (stderr, "dMcal tgroup: %7.4f %7.4f %7.4f %7.4f %7.4f %6d\n", tgD.median, tgD.mean, tgD.sigma, tgD.min, tgD.max, tgD.Nmeas); + fprintf (stderr, "chisq tgroup: %7.1f %7.1f %7.1f %7.1f %7.1f %6d\n", tgX.median, tgX.mean, tgX.sigma, tgX.min, tgX.max, tgX.Nmeas); +} + + Index: /branches/eam_branches/relphot.20210521/src/help.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/help.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/help.c (revision 41602) @@ -0,0 +1,85 @@ +# include "relphot.h" + +void relphot_usage (void) { + fprintf (stderr, "ERROR: USAGE: relphot (photcodes) -images\n"); + fprintf (stderr, " or: relphot -averages\n"); + fprintf (stderr, " or: relphot -apply-offsets\n"); + fprintf (stderr, " or: relphot -synthphot_means\n"); + fprintf (stderr, " or: relphot (photcodes) -parallel-regions -region-hosts (RegionFile)\n"); + fprintf (stderr, " or: relphot (photcodes) -parallel-images (ImageTable) -region-hosts (RegionFile)\n\n"); + fprintf (stderr, " regions: -region RA RA DEC DEC)\n"); + fprintf (stderr, " or: -catalog (name)\n"); + fprintf (stderr, " use -h for more usage information\n"); + exit (2); +} + +void relphot_help (int argc, char **argv) { + + /* check for help request */ + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + if (argc == 1) relphot_usage(); + return; + +show_help: + fprintf (stderr, "ERROR: USAGE: relphot (photcodes) -images\n"); + fprintf (stderr, " or: relphot -averages\n"); + fprintf (stderr, " or: relphot -apply-offsets\n"); + fprintf (stderr, " or: relphot (photcodes) -parallel-regions -region-hosts (RegionFile)\n"); + fprintf (stderr, " or: relphot (photcodes) -parallel-images (ImageTable) -region-hosts (RegionFile)\n\n"); + fprintf (stderr, " regions: -region RA RA DEC DEC)\n"); + fprintf (stderr, " or: -catalog (name)\n"); + fprintf (stderr, " options: \n"); + fprintf (stderr, " -time (start) (stop)\n"); + fprintf (stderr, " -v : verbose output\n"); + fprintf (stderr, " -vv : more verbose output\n"); + fprintf (stderr, " -outroot (outroot)\n"); + fprintf (stderr, " -plot\n"); + fprintf (stderr, " -plotdelay (seconds)\n"); + fprintf (stderr, " -statmode (mode)\n"); + fprintf (stderr, " -refcode (name) : give extra weight to this photcode\n"); + fprintf (stderr, " -n (nloop)\n"); + fprintf (stderr, " -reset\n"); + fprintf (stderr, " -update\n"); + fprintf (stderr, " -params\n"); + fprintf (stderr, " -mosaic (mosaic)\n"); + fprintf (stderr, " -imfreeze\n"); + fprintf (stderr, " -grid\n"); + fprintf (stderr, " -reset-ubercal : also reset ubercal-ed zero points (otherwise they are sacrosanct)\n"); + fprintf (stderr, " -area Xmin Xmax Ymin Ymax\n"); + fprintf (stderr, " -instmag min max\n"); + fprintf (stderr, " \n"); + exit (2); +} + +void relphot_client_usage (void) { + fprintf (stderr, "ERROR: USAGE: relphot (photcodes) -load (filename) -hostID (hostID) -hostdir (hostdir) [options]\n"); + fprintf (stderr, " or: relphot -update (filename) -hostID (hostID) -hostdir (hostdir) [options]\n"); + fprintf (stderr, " or: relphot -update-objects -hostID (hostID) -hostdir (hostdir) [options]\n"); + fprintf (stderr, " use -h for more usage information\n"); + exit (2); +} + +void relphot_client_help (int argc, char **argv) { + + /* check for help request */ + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + if (argc == 1) relphot_client_usage(); + return; + +show_help: + fprintf (stderr, "USAGE: relphot_client [-load / -update] (db info)\n\n"); + fprintf (stderr, " relphot_client -load (bcatalog) : extract the bright catalog subset from client's tables\n"); + fprintf (stderr, " (bcatalog) : location where the bright subset is saved\n"); + fprintf (stderr, " relphot_client -update (images) : apply calculated zero points to the client's tables\n\n"); + fprintf (stderr, " (images) : location of the table with the image zero points\n"); + fprintf (stderr, " relphot_client -update-objects : determine average magnitudes for objects\n\n"); + fprintf (stderr, " db info : -hostID (hostID) -hostdir (hostdir) -catdir (catdir)\n"); + fprintf (stderr, "other options:\n"); + fprintf (stderr, " -v : verbose output\n"); + fprintf (stderr, " -vv : extra verbose output\n"); + fprintf (stderr, " \n"); + exit (2); +} + Index: /branches/eam_branches/relphot.20210521/src/indexCatalog.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/indexCatalog.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/indexCatalog.c (revision 41602) @@ -0,0 +1,74 @@ +# include "relphot.h" + +static int catIDmax = 0; +static int *catIDseq = NULL; +static int *objIDmax = NULL; +static int **objIDseq = NULL; + +int indexCatalogs (Catalog *catalog, int Ncatalog) { + + int i; + off_t j; + + // find the max value of catID + for (i = 0; i < Ncatalog; i++) { + catIDmax = MAX (catalog[i].catID, catIDmax); + } + + ALLOCATE (catIDseq, int, catIDmax + 1); + for (i = 0; i < catIDmax + 1; i++) { + catIDseq[i] = -1; + } + + for (i = 0; i < Ncatalog; i++) { + int catID = catalog[i].catID; + catIDseq[catID] = i; + } + + ALLOCATE (objIDmax, int, Ncatalog); + ALLOCATE (objIDseq, int *, Ncatalog); + for (i = 0; i < Ncatalog; i++) { + objIDmax[i] = 0; + for (j = 0; j < catalog[i].Naverage; j++) { + objIDmax[i] = MAX (catalog[i].averageT[j].objID, objIDmax[i]); + } + + ALLOCATE (objIDseq[i], int, objIDmax[i] + 1); + for (j = 0; j < objIDmax[i] + 1; j++) { + objIDseq[i][j] = -1; + } + + for (j = 0; j < catalog[i].Naverage; j++) { + int objID = catalog[i].averageT[j].objID; + objIDseq[i][objID] = j; + } + } + return TRUE; +} + +int catID_and_objID_to_seq (int catID, int objID, int *catSeq, off_t *objSeq) { + + if (catID > catIDmax) return FALSE; + + int cat = catIDseq[catID]; + if (cat < 1) return FALSE; + + if (objID > objIDmax[cat]) return FALSE; + + int obj = objIDseq[cat][objID]; + if (obj < 1) return FALSE; + + *catSeq = cat; + *objSeq = obj; + return TRUE; +} + +void freeCatalogIndexes (int Ncatalog) { + + for (int i = 0; i < Ncatalog; i++) { + FREE (objIDseq[i]); + } + FREE (catIDseq); + FREE (objIDseq); + FREE (objIDmax); +} Index: /branches/eam_branches/relphot.20210521/src/initialize.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/initialize.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/initialize.c (revision 41602) @@ -0,0 +1,153 @@ +# include "relphot.h" + +RelphotMode initialize (int argc, char **argv) { + + init_error(); + relphot_help (argc, argv); + ConfigInit (&argc, argv); + RelphotMode mode = args (argc, argv); + if (!mode) exit (2); + + // UPDATE_AVERAGES always operates on all photcodes? + if (mode == UPDATE_AVERAGES) { + char tmpline1[256]; + int Ns; + Nphotcodes = GetPhotcodeNsecfilt (); + ALLOCATE (photcodes, PhotCode *, Nphotcodes); + ALLOCATE (PhotcodeList, char, 256); + for (Ns = 0; Ns < Nphotcodes; Ns++) { + photcodes[Ns] = GetPhotcodebyNsec (Ns); + if (Ns > 0) { + snprintf (tmpline1, 256, "%s,%s", PhotcodeList, photcodes[Ns][0].name); + } else { + snprintf (tmpline1, 256, "%s", photcodes[Ns][0].name); + } + strcpy (PhotcodeList, tmpline1); + } + } + + if (USE_GRID && (Nphotcodes > 1)) { + fprintf (stderr, "grid correction analysis currently can only operate on a single photcode\n"); + exit (1); + } + + IMAGE_BAD = ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_SKIP; + STAR_BAD = ID_OBJ_POOR | ID_OBJ_FEW; + MEAS_BAD = ID_MEAS_NOCAL | ID_MEAS_POOR_PHOTOM | ID_MEAS_SKIP_PHOTOM | ID_MEAS_AREA; + + if (SHOW_PARAMS) { + int Ns; + fprintf (stderr, "subset selection criteria:\n"); + fprintf (stderr, " photcodes "); + for (Ns = 0; Ns < Nphotcodes; Ns++) { + if (Ns == Nphotcodes - 1) { + fprintf (stderr, "%s\n", photcodes[Ns][0].name); + } else { + fprintf (stderr, "%s, ", photcodes[Ns][0].name); + } + } + if (TimeSelect) { + fprintf (stderr, "TimeSelect: TRUE (%s - %s)\n", ohana_sec_to_date (TSTART), ohana_sec_to_date (TSTOP)); + } else { + fprintf (stderr, "TimeSelect: FALSE\n"); + } + if (DophotSelect) { + fprintf (stderr, "DophotSelect: TRUE (%d)\n", DophotValue); + } else { + fprintf (stderr, "DophotSelect: FALSE\n"); + } + fprintf (stderr, "PSF_QF limit: 0.85 (hardwired)\n"); + + // fprintf (stderr, "Photom Bad Mask: 0x%08x, Photom Poor Mask: 0x%08x\n"); + + fprintf (stderr, "MAG_LIM: %f, SIGMA_LIM: %f\n", MAG_LIM, SIGMA_LIM); + fprintf (stderr, "INST_MAG_MIN: %f, INST_MAG_MAX: %f\n", ImagMin, ImagMax); + + fprintf (stderr, "STAR_TOOFEW: %d\n", STAR_TOOFEW); + + fprintf (stderr, "VERBOSE: %d, PLOTSTUFF: %d\n", VERBOSE, PLOTSTUFF); + fprintf (stderr, "GRID_X: %d, GRID_Y: %d, BINNING: %d == Nmx: %d, Nmy: %d\n", + RELPHOT_GRID_X, + RELPHOT_GRID_Y, + RELPHOT_GRID_BINNING, + (int)(RELPHOT_GRID_X/RELPHOT_GRID_BINNING), (int)(RELPHOT_GRID_Y/RELPHOT_GRID_BINNING)); + + fprintf (stderr, "STAR_SCATTER %lf\n", STAR_SCATTER); + fprintf (stderr, "IMAGE_SCATTER %lf\n", IMAGE_SCATTER); + fprintf (stderr, "IMAGE_OFFSET %lf\n", IMAGE_OFFSET); + fprintf (stderr, "IMAGE_CATALOG %s\n", ImageCat); + fprintf (stderr, "GSCFILE %s\n", GSCFILE); + fprintf (stderr, "CATDIR %s\n", CATDIR); + } + + // init the random seed + long A, B; + A = time(NULL); + for (B = 0; A == time(NULL); B++); + srand48(B); + + return mode; +} + +void initialize_client (int argc, char **argv) { + + // XXX need to determine which globals can affect relphot_client in either mode and pass appropriately + PhotcodeList = NULL; + photcodes = NULL; + + relphot_client_help (argc, argv); + ConfigInit (&argc, argv); + args_client (argc, argv); + + if (MODE == MODE_SYNTH_PHOT) return; + + if (MODE == MODE_UPDATE_OBJECTS) { + char tmpline1[256]; + int Ns; + Nphotcodes = GetPhotcodeNsecfilt (); + ALLOCATE (photcodes, PhotCode *, Nphotcodes); + ALLOCATE (PhotcodeList, char, 256); + for (Ns = 0; Ns < Nphotcodes; Ns++) { + photcodes[Ns] = GetPhotcodebyNsec (Ns); + if (Ns > 0) { + snprintf (tmpline1, 256, "%s,%s", PhotcodeList, photcodes[Ns][0].name); + } else { + snprintf (tmpline1, 256, "%s", photcodes[Ns][0].name); + } + strcpy (PhotcodeList, tmpline1); + } + return; + } + + // load the list of photcodes into the globals (photcodes, Nphotcodes) + PhotcodeList = strcreate (argv[1]); + photcodes = ParsePhotcodeList (PhotcodeList, &Nphotcodes, TRUE); // require SEC photcodes +} + +void ParsePhotcodeList_old (char *word) { + + Nphotcodes = 0; + photcodes = NULL; + int NPHOTCODES = 10; + ALLOCATE (photcodes, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodesKeep */ + char *myList = strcreate(word); + char *list = myList; + char *codename = NULL; + char *ptr = NULL; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + if ((photcodes[Nphotcodes] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + if (photcodes[Nphotcodes][0].type != PHOT_SEC) { + fprintf (stderr, "photcode %s is not an filter type (SEC)\n", codename); + exit (1); + } + Nphotcodes ++; + CHECK_REALLOCATE (photcodes, PhotCode *, NPHOTCODES, Nphotcodes, 10); + } +} + Index: /branches/eam_branches/relphot.20210521/src/launch_region_hosts.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/launch_region_hosts.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/launch_region_hosts.c (revision 41602) @@ -0,0 +1,125 @@ +# include "relphot.h" +# define DEBUG 0 + +int launch_region_hosts (RegionHostTable *regionHosts) { + + int i; + + // do not remove the sync and fits files if we do a manual run -- user must clear if needed + if (!PARALLEL_REGIONS_MANUAL) { + // clear the I/O files + for (i = 0; i < regionHosts->Nhosts; i++) { + char *syncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.sync"); + if (truncate (syncfile, 0)) fprintf (stderr, "trouble clearing syncfile %s\n", syncfile); + free (syncfile); + + char *fitsfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.fits"); + if (truncate (fitsfile, 0)) fprintf (stderr, "trouble clearing fitsfile %s\n", fitsfile ); + free (fitsfile); + + char *imsyncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.sync"); + if (truncate (imsyncfile, 0)) fprintf (stderr, "trouble clearing imsyncfile %s\n", imsyncfile); + free (imsyncfile); + + char *imfitsfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.fits"); + if (truncate (imfitsfile, 0)) fprintf (stderr, "trouble clearing imfitsfile %s\n", imfitsfile); + free (imfitsfile); + + char *loopsyncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "loop.sync"); + if (truncate (loopsyncfile, 0)) fprintf (stderr, "trouble clearing loopsyncfile %s\n", loopsyncfile); + free (loopsyncfile); + } + } + + for (i = 0; i < regionHosts->Nhosts; i++) { + + RegionHostInfo *host = ®ionHosts->hosts[i]; + + // communication files: + // subset images per host : CATDIR/Image.HOSTNAME.fits + char filename[1024]; + snprintf (filename, 1024, "%s/Image.%d.fits", CATDIR, host->hostID); + if (unlink (filename)) fprintf (stderr, "trouble clearing image %s\n", filename); + + // write the image subset for this host + ImageTableSave (filename, host->image, host->Nimage); + + char *command = NULL; + strextend (&command, "relphot %s", PhotcodeList); + strextend (&command, "-parallel-images %s", filename); + strextend (&command, "-region-hosts %s", REGION_FILE); + strextend (&command, "-region-hostID %d", host->hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-region %f %f %f %f", host->RminCat, host->RmaxCat, host->DminCat, host->DmaxCat); + strextend (&command, "-statmode %s", STATMODE); + strextend (&command, "-D CAMERA %s", CAMERA); + strextend (&command, "-D STAR_TOOFEW %d", STAR_TOOFEW); + strextend (&command, "-minerror %f", MIN_ERROR); + strextend (&command, "-cloud-limit %f", CLOUD_TOLERANCE); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (RESET) strextend (&command, "-reset"); + if (RESET_ZEROPTS) strextend (&command, "-reset-zpts"); + if (!KEEP_UBERCAL) strextend (&command, "-reset-ubercal"); + if (DophotSelect) strextend (&command, "-dophot %d", DophotValue); + if (ImagSelect) strextend (&command, "-instmag %f %f", ImagMin, ImagMax); + if (MaxDensityUse) strextend (&command, "-max-density %f", MaxDensityValue); + if (SyntheticPhotometry) strextend (&command, "-synthphot"); + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + + if (UPDATE) strextend (&command, "-update"); + if (MOSAIC_ZEROPT) strextend (&command, "-mosaic"); + if (FREEZE_IMAGES) strextend (&command, "-imfreeze"); + if (FREEZE_MOSAICS) strextend (&command, "-mosfreeze"); + if (CALIBRATE_STACKS_AND_WARPS) strextend (&command, "-only-stacks-and-warps"); + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } + + if (PARALLEL) strextend (&command, "-parallel"); + if (PARALLEL_MANUAL) strextend (&command, "-parallel-manual"); + if (PARALLEL_SERIAL) strextend (&command, "-parallel-serial"); + + // XXX deprecate this if we are happy with the new version + // if (SET_MREL_VERSION != 1) strextend (command, "-set-mrel-version %d", SET_MREL_VERSION); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_REGIONS_MANUAL) { + free (command); + continue; + } + + // launch the job, then wait for it to be done loading catalogs. force the remote + // client to generate the file + char *syncfile = make_filename (CATDIR, host->hostname, host->hostID, "loadcat.sync"); + clear_sync_file (syncfile); + + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", host->hostname, command, host->stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", host->hostname, errorInfo); + exit (1); + } + host->pid = pid; // save for future reference + + // remote client is done, go ahead with next client + check_sync_file (syncfile, 1); + free (syncfile); + free (command); + } + + int status = TRUE; + if (PARALLEL_REGIONS_MANUAL) { + fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); + getchar(); + } else { + RegionHostTableWaitJobsGetIO (regionHosts, __FILE__, __LINE__, VERBOSE); + + for (i = 0; i < regionHosts->Nhosts; i++) { + status = status && (regionHosts->hosts[i].status == 0); + } + } + + return status; +} Index: /branches/eam_branches/relphot.20210521/src/liststats.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/liststats.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/liststats.c (revision 41602) @@ -0,0 +1,435 @@ +# include "relphot.h" + +void liststats_setmode (StatType *stats, char *strmode) { + + stats->statmode = -1; + if (!strcmp (strmode, "MEAN")) { stats->statmode = STATS_MEAN; return; } + if (!strcmp (strmode, "MEDIAN")) { stats->statmode = STATS_MEDIAN; return; } + if (!strcmp (strmode, "WT_MEAN")) { stats->statmode = STATS_WT_MEAN; return; } + if (!strcmp (strmode, "INNER_MEAN")) { stats->statmode = STATS_INNER_MEAN; return; } + if (!strcmp (strmode, "INNER_WTMEAN")) { stats->statmode = STATS_INNER_WTMEAN; return; } + if (!strcmp (strmode, "CHI_INNER_MEAN")) { stats->statmode = STATS_CHI_INNER_MEAN; return; } + if (!strcmp (strmode, "CHI_INNER_WTMEAN")) { stats->statmode = STATS_CHI_INNER_WTMEAN; return; } + + fprintf (stderr, "ERROR: invalid stats mode: %s\n", strmode); + exit (1); +} + +int liststats_init (StatType *stats) { + stats->median = NAN; + stats->mean = NAN; + stats->sigma = NAN; + stats->error = NAN; + stats->chisq = NAN; + stats->min = NAN; + stats->max = NAN; + stats->Upper80 = NAN; + stats->Lower20 = NAN; + stats->Upper90 = NAN; + stats->Lower10 = NAN; + stats->total = NAN; + stats->Nmeas = 0; + return TRUE; +} + +int liststats (double *value, double *dvalue, double *weight, int N, StatType *stats) { + + int i, ks, ke; + double Mo, dMo, M, dM, Nm, X2, dS, R, W, *chi; + + myAssert (stats->statmode != STATS_NONE, "programming error, liststats mode not set"); + + liststats_init (stats); + + ke = ks = dMo = 0; + + stats[0].Nmeas = N; + stats[0].mean = 0; + stats[0].sigma = 0; + stats[0].error = 0; + stats[0].chisq = 0; + if (N < 1) return (FALSE); + + int N90 = MIN (N-1, 0.9*N); + int N80 = MIN (N-1, 0.8*N); + int N20 = MAX (0, 0.2*N); + int N10 = MAX (0, 0.1*N); + + if (weight) { + dsortthree (value, dvalue, weight, N); + } else { + if (dvalue) { + dsortpair (value, dvalue, N); + } else { + dsort (value, N); + } + } + + // these values do not depend on the errors or weighting scheme + stats[0].median = value[(int)(0.5*N)]; + stats[0].min = value[0]; + stats[0].max = value[N-1]; + stats[0].Upper80 = value[N80]; + stats[0].Lower20 = value[N20]; + stats[0].Upper90 = value[N90]; + stats[0].Lower10 = value[N10]; + + switch (stats->statmode) { + case STATS_MEDIAN: + ks = 0; + ke = N; + Mo = stats[0].median; + Nm = N; + goto chisq; + break; + case STATS_MEAN: + case STATS_WT_MEAN: + ks = 0; + ke = N; + break; + case STATS_INNER_MEAN: + case STATS_INNER_WTMEAN: + case STATS_CHI_INNER_MEAN: + case STATS_CHI_INNER_WTMEAN: + ks = 0.25*N + 0.50; + ke = 0.75*N + 0.25; + if (N <= 3) { + ks = 0; + ke = N; + } + break; + case STATS_NONE: + myAbort ("undefined stats"); + } + + // for these two modes, I need a vector of the chi-square contribution + // I'm actually just using chisq to get the correct sorting order + if ((stats->statmode == STATS_CHI_INNER_MEAN) || (stats->statmode == STATS_CHI_INNER_WTMEAN)) { + ALLOCATE (chi, double, N); + for (i = 0; i < N; i++) { + chi[i] = (value[i] - stats[0].median) / dvalue[i]; + } + if (weight) { + dsortfour (chi, value, dvalue, weight, N); + } else { + dsortthree (chi, value, dvalue, N); + } + free (chi); + } + + int WeightedMean = FALSE; + WeightedMean |= (stats->statmode == STATS_WT_MEAN); + WeightedMean |= (stats->statmode == STATS_INNER_WTMEAN); + WeightedMean |= (stats->statmode == STATS_CHI_INNER_WTMEAN); + if (!dvalue) WeightedMean = FALSE; // warn the user? + + /* calculating the per-star offset based on the desired weighting scheme */ + M = dM = Nm = W = R = 0; + if (weight) { + // the weight value is multiplied by whichever nominal weighting scheme is provided + // thus the user should set weight to 1.0 for nominal weight, and 100 for heavy weight (or so) + // and 0.01 for under-weight + if (WeightedMean) { + for (i = ks; i < ke; i++) { + M += value[i] * weight[i] / SQ(dvalue[i]); + W += weight[i] / SQ(dvalue[i]); + dM += SQ (weight[i] / dvalue[i]); + R += weight[i] / SQ(dvalue[i]); + Nm += 1.0; + } + Mo = M / W; + dMo = sqrt (dM) / R; + } else { + for (i = ks; i < ke; i++) { + M += value[i] * weight[i]; + W += weight[i]; + dM += dvalue ? SQ (weight[i] * dvalue[i]) : 0.0; + R += weight[i]; + Nm += 1.0; + } + Mo = M / W; + dMo = dvalue ? sqrt (dM) / R : NAN; + } + } else { + // NULL weight vector is supplied, revert to standard form (above reverts if weight[i] == 1) + if (WeightedMean) { + // weighted by inverse-variance + for (i = ks; i < ke; i++) { + M += value[i] / SQ (dvalue[i]); + dM += 1.0 / SQ (dvalue[i]); + Nm += 1.0; + } + Mo = M / dM; + dMo = sqrt (1.0 / dM); + } else { + // pure un-weighted + for (i = ks; i < ke; i++) { + M += value[i]; + dM += dvalue ? SQ (dvalue[i]) : 0.0; + Nm += 1.0; + } + Mo = M / Nm; + dMo = dvalue ? sqrt (dM) / Nm : NAN; + } + } + + chisq: + /* find sigma and chisq */ + X2 = dS = 0; + for (i = ks; i < ke; i++) { + M = SQ (value[i] - Mo); + if (dvalue) { + dM = SQ (dvalue[i]); + X2 += M / dM; + } + dS += M; + } + X2 = dvalue ? X2 / (Nm - 1) : NAN; + dS = sqrt (dS / (Nm - 1)); + + stats[0].mean = Mo; + stats[0].Nmeas = Nm; + stats[0].chisq = X2; + stats[0].sigma = dS; + stats[0].error = dMo; + + return (TRUE); +} + +int liststats_fit1d (double *value, double *err, double *x, int Npts, StatType *stats, double *dk) { + + double M = 0.0, MZ = 0.0, Z1 = 0.0, Z2 = 0.0, R = 0.0; + + for (int i = 0; i < Npts; i++) { + double wt = 1.0 / (err[i]*err[i]); + double V = value[i]; + double Z = x[i]; + M += V * wt; + MZ += V*Z * wt; + Z1 += Z * wt; + Z2 += Z*Z * wt; + R += 1.0 * wt; + } + + double DetInv = (R*Z2 - Z1*Z1); + double Det = (fabs(DetInv) < -1e6) ? 0.0 : 1.0 / DetInv; // do not allow unstable solutions + + double zp = (M*Z2 - Z1*MZ)*Det; + double dK = (MZ*R - M*Z1)*Det; + + /* find sigma and chisq */ + double Mo = 0.0, dM = 0.0; + double X2 = 0.0, dS = 0.0; + for (int i = 0; i < Npts; i++) { + Mo = zp + dK*x[i]; + M = SQ (value[i] - Mo); + dM = SQ (err[i]); + X2 += M / dM; + dS += M; + } + X2 = X2 / (Npts - 1); + dS = sqrt (dS / (Npts - 1)); + + *dk = dK; + stats->mean = zp; + stats->chisq = X2; + stats->error = dS; + return TRUE; +} + +// These should probably be tunable: +# define MAX_ITERATIONS 10 +# define FIT_TOLERANCE 1e-4 +# define FLT_TOLERANCE 1e-6 +# define WEIGHT_THRESHOLD 0.3 + +int fit_least_squares (double *fit, double *err, double *y, double *dy, double *wgt, double *wt, int Npts); + +// this is a zero-order fit (constant value only) +int liststats_irls (StatDataSet *dataset, int Npoints, StatType *stats) { + + liststats_init (stats); + + if (Npoints == 0) { + double value = NAN; + stats->mean = value; + stats->min = value; + stats->max = value; + stats->Nmeas = Npoints; + stats->chisq = NAN; + stats->sigma = NAN; + stats->error = NAN; + return TRUE; + } + + if (Npoints == 1) { + double value = dataset->flxlist[0]; + stats->mean = value; + stats->min = value; + stats->max = value; + stats->Nmeas = Npoints; + stats->chisq = NAN; + stats->sigma = NAN; + stats->error = dataset->errlist[0]; + return TRUE; + } + + int midpt = 0.5 * Npoints; + + // make the initial guess based on the median (not weighted mean) + // ALLOCATE_PTR (values, double, Npoints); + for (int i = 0; i < Npoints; i++) { + dataset->values[i] = dataset->flxlist[i]; + } + dsort (dataset->values, Npoints); + double value = (Npoints % 2) ? dataset->values[midpt] : 0.5*(dataset->values[midpt] + dataset->values[midpt-1]); + + // OLS (replace by median above) + // if (!fit_least_squares (&value, dataset->flxlist, dataset->errlist, dataset->wgtlist, NULL, Npoints)) return FALSE; + + // XXX add to dataset elements? + // ALLOCATE_PTR (wtvals, double, Npoints); + // ALLOCATE_PTR (wtlist, double, Npoints); + + int converged = FALSE; + for (int iterations = 0; !converged && (iterations < MAX_ITERATIONS); iterations++) { + + for (int i = 0; i < Npoints; i++) { + // we are only including the formal error, not the weight in the definition of wt[] + dataset->wtvals[i] = weight_cauchy ((dataset->flxlist[i] - value) / dataset->errlist[i]); + } + + double oldValue = value; + if (!fit_least_squares (&value, NULL, dataset->flxlist, dataset->errlist, dataset->wgtlist, dataset->wtvals, Npoints)) { + value = oldValue; + break; + } + + converged = TRUE; + if ((fabs(value - oldValue) > FIT_TOLERANCE * fabs(value)) && + (fabs(value - oldValue) > FLT_TOLERANCE)) { + converged = FALSE; + } + } + stats->mean = value; + + // calculate the weight thresholds to mask the bad points: + // double Sum_W = 0.0; + for (int i = 0; i < Npoints; i++) { + dataset->wtvals[i] = weight_cauchy ((dataset->flxlist[i] - value) / dataset->errlist[i]); + dataset->wtlist[i] = dataset->wtvals[i]; + // Sum_W += dataset->wtvals[i]; + } + dsort (dataset->wtlist, Npoints); + double WtMedian = (Npoints % 2) ? dataset->wtlist[midpt] : 0.5*(dataset->wtlist[midpt] + dataset->wtlist[midpt-1]); + // double WtThreshold = WEIGHT_THRESHOLD * Sum_W / (1.0 * Npoints); + double WtThreshold = WEIGHT_THRESHOLD * WtMedian; + + // generate the unmasked subset + // XXX add these to the dataset elements? + // ALLOCATE_PTR ( ykeep, double, Npoints); + // ALLOCATE_PTR (dykeep, double, Npoints); + // ALLOCATE_PTR (wtkeep, double, Npoints); + + // save unmasked points + int Nkeep = 0; + stats->min = NAN; + stats->max = NAN; + double dChi = 0.0; + double dSig = 0.0; + for (int i = 0; i < Npoints; i++) { + if ((dataset->wtvals[i] < WtThreshold) || !isfinite(dataset->flxlist[i])) { + dataset->msklist[i] = TRUE; // mark the masked points + continue; + } + dataset->ykeep[Nkeep] = dataset->flxlist[i]; + dataset->dykeep[Nkeep] = dataset->errlist[i]; + dataset->wtkeep[Nkeep] = dataset->wgtlist[i]; // externally-supplied weight + Nkeep ++; + + // record mean, error, chisq, min, max, sigma, Nmeas (unmasked points) + stats->min = isfinite(stats->min) ? MIN(dataset->flxlist[i], stats->min) : dataset->flxlist[i]; + stats->max = isfinite(stats->max) ? MAX(dataset->flxlist[i], stats->max) : dataset->flxlist[i]; + double dValue2 = SQ(dataset->flxlist[i] - value); + dChi += dValue2 / SQ (dataset->errlist[i]); + dSig += dValue2; + } + stats->Nmeas = Nkeep; + stats->chisq = dChi / (Nkeep - 1); + stats->sigma = sqrt (dSig / (Nkeep - 1)); + + // bootstrap resampling to generate the errorbars + // XXX add these to the dataset elements? + // ALLOCATE_PTR (ysample, double, Nkeep); + // ALLOCATE_PTR (dysample, double, Nkeep); + // ALLOCATE_PTR (wtsample, double, Nkeep); + // ALLOCATE_PTR (bvalue, double, NBOOTSTRAP); // vector to save the bootstrap values + + int Nboot = 0; + for (int iboot = 0; iboot < NBOOTSTRAP; iboot++) { + + // resample + for (int i = 0; i < Nkeep; i++) { + // I need to draw Npoints random entries from 'points' with replacement: + int N = Nkeep * drand48(); + dataset->ysample[i] = dataset->ykeep[N]; + dataset->dysample[i] = dataset->dykeep[N]; + dataset->wtsample[i] = dataset->wtkeep[N]; + } + + if (!fit_least_squares (&value, NULL, dataset->ysample, dataset->dysample, dataset->wtsample, NULL, Nkeep)) continue; + + dataset->bvalue[Nboot] = value; + Nboot ++; + } + + dsort (dataset->bvalue, Nboot); + + double Slo = VectorFractionInterpolate (dataset->bvalue, 0.158655, Nboot); + double Shi = VectorFractionInterpolate (dataset->bvalue, 0.841345, Nboot); + stats->error = (Shi - Slo) / 2.0; + + // bootstrap can sometimes yield an excessively-optimistic result for the error. Do not let + // the reported error be smaller than the formal error + double errvalue; + if (fit_least_squares (&value, &errvalue, dataset->ykeep, dataset->dykeep, dataset->wtkeep, NULL, Nkeep)) { + stats->error = MAX (stats->error, errvalue); + } + + return TRUE; +} + +// wgt is externally-supplied weight, wt is optional +int fit_least_squares (double *fit, double *err, double *y, double *dy, double *wgt, double *wt, int Npts) { + + int i; + + double S0 = 0; + double S1 = 0; + + /* perform linear fit */ + for (i = 0; i < Npts; i++, y++, dy++, wgt++) { + if (!finite(*y)) continue; + + // wt is optional + double dY = wt ? wt[i] * (*wgt) / SQ(*dy) : (*wgt) / SQ(*dy); + + S0 += dY; + S1 += *y*dY; + } + if (S0 == 0.0) return FALSE; + *fit = S1 / S0; + if (err) { *err = 1.0 / S0; } + return TRUE; +} + +// we could define the weight to be the only scale factor: +// \mu = \sum (value_i * weight_i) / \sum (weight_i) +// \sigma^2 = (1/R) \sum (weight_i^2 \sigma_i^2) +// R = \sum (weight_i^2) + +// or, we could define the weight to be a scale factor times the inverse error: +// \mu = \sum (value_i * weight_i / sigma_i) / \sum (weight_i) +// \sigma^2 = (1/R) \sum (weight_i^2 \sigma_i^2) +// R = \sum (weight_i^2) + Index: /branches/eam_branches/relphot.20210521/src/load_catalogs.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/load_catalogs.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/load_catalogs.c (revision 41602) @@ -0,0 +1,286 @@ +# include "relphot.h" +# include +# include + +// this function loops over the catalogs, loads the data, and extracts a bright subset +// the returned array (catalog, Ncatalog) has the same layout as the full database, but +// only a subset of the detetions & objects + +// if this function is called in parallel mode, it in turn calls load_catalogs_parallel, +// which distributes the work to the remote hosts and loads their results + +// if this function is called with a specified hostID, then only the fraction of the +// database hosted by that hostID is loaded +Catalog *load_catalogs (SkyList *skylist, int *Ncatalog, int hostID, char *hostpath, char *syncfile) { + + off_t i, Nmeas, Nstar, Nmeas_total, Nstar_total; + Catalog *catalog, tcatalog; + + // XXX need to decide how to determine PARALLEL mode... + if (PARALLEL && !hostID) { + catalog = load_catalogs_parallel (skylist, Ncatalog, syncfile); + return catalog; + } + + if (VERBOSE2) fprintf (stderr, "loading catalog data\n"); + + // a bit of an over-alloc, since we don't load all catalogs for a region + ALLOCATE (catalog, Catalog, skylist[0].Nregions); + + Nmeas_total = Nstar_total = 0; + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + // XXX keep in mind that not all catalogs are loaded + dvo_catalog_init (&catalog[i], TRUE); + dvo_catalog_init (&tcatalog, TRUE); + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + dvo_catalog_init (&tcatalog, TRUE); + tcatalog.filename = hostID ? hostfile : skylist[0].filename[i]; + tcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; // don't need to load all data at this point + tcatalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog + + if (!dvo_catalog_open (&tcatalog, skylist[0].regions[i], VERBOSE2, "r")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", tcatalog.filename); + exit (1); + } + if (!tcatalog.Naverage_disk) { + if (VERBOSE2) { fprintf (stderr, "no data in %s, skipping\n", tcatalog.filename); } + dvo_catalog_unlock (&tcatalog); + dvo_catalog_free (&tcatalog); + continue; + } + + if (!tcatalog.sorted) { + fprintf (stderr, "this database is not sorted. please sort using addstar -resort\n"); + exit (1); + } + + Nstar_total += tcatalog.Naverage; + Nmeas_total += tcatalog.Nmeasure; + + // select only the brighter stars + bcatalog (&catalog[i], &tcatalog, i); + dvo_catalog_unlock (&tcatalog); + dvo_catalog_free (&tcatalog); + } + + // XXX keep this test? + Nstar = Nmeas = 0; + for (i = 0; i < skylist[0].Nregions; i++) { + Nstar += catalog[i].Naverage; + Nmeas += catalog[i].Nmeasure; + if ((sizeof(IDX_T) == 8) && (catalog[i].Nmeasure > 0xffffffff)) { + fprintf (stderr, "ERROR: using small-sized IDX_T on data with more than 2G detections per table will cause errors\n"); + fprintf (stderr, " If you need to do this, and you can afford the RAM, rebuild with IDX_T set to off_t (see relphot.h, ImagesOps.c)\n"); + exit (3); + } + } + if (Nstar < 2) { + fprintf (stderr, "warning: insufficient stars "OFF_T_FMT"\n", Nstar); + } + + fprintf (stderr, "using "OFF_T_FMT" of "OFF_T_FMT" stars ("OFF_T_FMT" of "OFF_T_FMT" measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total); + if (!hostID && !REGION_HOST_ID && (Nstar < 1)) Shutdown ("%s", "ERROR: no stars match the minimum requirements; exiting \n"); + // in regular relphot, we shutdown here; in relphot_client, we generate and return an empty table (for consistency) + + // if we are running with parallel_images but not a parallel database, we need to + // release the lock so the next image host can proceed + if (!hostID && syncfile) { + update_sync_file (syncfile, 1); + } + + // XXX consider only returning the populated catalogs + *Ncatalog = skylist[0].Nregions; + return (catalog); +} + +/* this function loads all relevant catalog files for the first pass. it currently loads the data + read only (SOFT lock) since it assumes the image table has been locked. if we go to the new + addstar locking paradigm, in which the images and catalogs are updated independently, then we may + need to use an XCLD lock here. +*/ + +// CATDIR is supplied globally +# define DEBUG 1 +Catalog *load_catalogs_parallel (SkyList *sky, int *Ncatalog, char *syncfile) { + + char uniquer[12]; + int TIME = time(NULL); + int PID = getpid(); + + if (PARALLEL_MANUAL && MANUAL_UNIQUER) { + snprintf_nowarn (uniquer, 12, "%11s", MANUAL_UNIQUER); + } else { + snprintf_nowarn (uniquer, 12, "%05d.%05d", PID, TIME % 100000); + } + + int Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + ALLOCATE (table->hosts[i].results, char, 1024); + snprintf (table->hosts[i].results, 1024, "%s/relphot.catalog.%s.dat", table->hosts[i].pathname, uniquer); + + // options / arguments that can affect relphot_client -load: + // VERBOSE, VERBOSE2 + // KEEP_UBERCAL + // RESET (-reset) + // RESET_ZEROPTS (-reset-zpts) + // TimeSelect -time + // DophotSelect + // (note that psfQF is applied rigidly at 0.85, as is the galaxy test) + // MAG_LIM + // SIGMA_LIM + // ImagSelect, ImagMin, ImagMax + // MaxDensityUse, MaxDensityValue + + char *command = NULL; + strextend (&command, "relphot_client %s -load %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D CAMERA %s -D MAG_LIM %f -D SIGMA_LIM %f", + PhotcodeList, table->hosts[i].results, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, CAMERA, MAG_LIM, SIGMA_LIM); + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (RESET_ZEROPTS) { strextend (&command, "-reset-zpts"); } + if (!KEEP_UBERCAL) { strextend (&command, "-reset-ubercal"); } + if (DophotSelect) { strextend (&command, "-dophot %d", DophotValue); } + if (ImagSelect) { strextend (&command, "-instmag %f %f", ImagMin, ImagMax); } + if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } + if (SyntheticPhotometry) { strextend (&command, "-synthphot"); } + + if (TimeSelect) { + char *tstart = ohana_sec_to_date (TSTART); + char *tstop = ohana_sec_to_date (TSTOP); + strextend (&command, "-time %s %s", tstart, tstop); + free (tstart); + free (tstop); + } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relphot_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + continue; + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + // if any one of the remote jobs fails, we should fail and exit + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) { + fprintf (stderr, "at least one remote client job failed to load data, exiting\n"); + exit (3); + } + } + + // update syncfile here (save lots of I/O time): + + // at this point, the remote relastro_client jobs are done loading their data. in a + // parallel_images mode, the next image host can be launched while this image host now + // reads that + + // NOTE: if I let all hosts load blindly, I saturate the data clients with too many + // relphot_client requests. I need to have the master mediate this. the master + // will not launch the next remote job until this one says it is done + if (syncfile) { + update_sync_file (syncfile, 1); + } + + // each host generates a BrightCatalog structure, with the measure, average, etc value + // loaded into a single set of arrays (of MeasureTiny, AverageTiny, Secfilt). I need to + // split out the per-catalog measurements into separate catalog entries. + + // set up an initial array of catalogs + CatalogSplitter *catalogs = BrightCatalogSplitInit (Nsecfilt); + + for (i = 0; i < table->Nhosts; i++) { + + BrightCatalog *bcatalog = NULL; + while ((bcatalog = BrightCatalogLoad (table->hosts[i].results)) == NULL) { + // failed to get the data from this host. This can happen for various reasons. Give the user a chance to try again... + fprintf (stderr, "failed to read data from %s, stopping operations until this can be fixed\n", table->hosts[i].hostname); + fprintf (stderr, "you may run the command manually and send this process the CONT signal\n"); + int pid = getpid(); + kill (pid, SIGSTOP); + fprintf (stderr, "retrying %s\n", table->hosts[i].results); + } + free (table->hosts[i].results); + table->hosts[i].results = NULL; + + BrightCatalogSplit (catalogs, bcatalog); + + free (bcatalog->average); + free (bcatalog->measure); + free (bcatalog->secfilt); + free (bcatalog); + } + + Catalog *catalog = catalogs->catalog; + *Ncatalog = catalogs->Ncatalog; + + // need to free the place-holder catalogs: + for (i = catalogs->Ncatalog; i < catalogs->NCATALOG; i++) { + free (catalogs->catalog[i].averageT); + free (catalogs->catalog[i].measureT); + free (catalogs->catalog[i].secfilt); + } + + int Nmeasure = 0; + int Naverage = 0; + for (i = 0; i < catalogs->Ncatalog; i++) { + Nmeasure += catalogs->catalog[i].Nmeasure; + Naverage += catalogs->catalog[i].Naverage; + } + + fprintf (stderr, "loaded %d catalogs, using a total of %d stars (%d measures)\n", catalogs->Ncatalog, Naverage, Nmeasure); + client_logger_message ("loaded %d catalogs, using a total of %d stars (%d measures)\n", catalogs->Ncatalog, Naverage, Nmeasure); + + BrightCatalogSplitFree (catalogs); + FreeHostTable (table); + + return (catalog); +} Index: /branches/eam_branches/relphot.20210521/src/load_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/load_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/load_images.c (revision 41602) @@ -0,0 +1,185 @@ +# include "relphot.h" + +static AstromOffsetTable *table = NULL; + +// This function generates a subset of the images based on selections. Input db has already +// been loaded with the raw fits table data +int load_images (FITS_DB *db, SkyList *skylist, SkyRegion *region, int unlockImages, int UseAllImages) { + + Image *image, *subset; + off_t Nimage, Nsubset; + off_t *LineNumber; + char *inSubset; + + INITTIME; + + // convert database table to internal structure (binary to Image) + // 'image' points to the same memory as db->ftable->buffer + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].scaledValue, &db[0].nativeOrder); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + MARKTIME("read image table: %f sec\n", dtime); + + BuildChipMatch (image, Nimage); + MARKTIME("build chip match for %d images: %f sec\n", (int) Nimage, dtime); + + // allocate and init an array to identify the images included in the subset + ALLOCATE (inSubset, char, Nimage); + memset (inSubset, 0, Nimage); + + // select the images which overlap the selected sky regions + // 'subset' points to a new copy of the data (different from 'image') if a subset is selected + if (UseAllImages) { + ALLOCATE (LineNumber, off_t, Nimage); + for (off_t i = 0; i < Nimage; i++) { + LineNumber[i] = i; + inSubset[i] = TRUE; + } + subset = image; + Nsubset = Nimage; + } else { + subset = select_images (skylist, image, Nimage, inSubset, &LineNumber, &Nsubset, region); + MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime); + + if (Nimage == 0) { + fprintf (stderr, "no images selected for analysis : problem with region or photcode?\n"); + exit (4); + } + + if (Nsubset == Nimage) { + free (subset); + subset = image; + } + } + + // reset image values as needed. always allow 'few' images to succeed, if possible (new + // images / detections may have been added + for (off_t i = 0; i < Nsubset; i++) { + // reset these two regardless (we will re-determine) + subset[i].flags &= ~(ID_IMAGE_PHOTOM_NOCAL | ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_SKIP | ID_IMAGE_PHOTOM_FEW); + if (RESET) { + if (RESET_ZEROPTS) { + if (!KEEP_UBERCAL || !(subset[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) { + subset[i].McalPSF = 0.0; + subset[i].McalAPER = 0.0; + subset[i].dMcal = NAN; + subset[i].flags &= ~ID_IMAGE_PHOTOM_UBERCAL; + } + } + subset[i].flags &= ~(ID_IMAGE_MOSAIC_POOR | ID_IMAGE_NIGHT_POOR); + subset[i].flags &= ~(ID_IMAGE_IMAGE_PHOTCAL | ID_IMAGE_MOSAIC_PHOTCAL | ID_IMAGE_TGROUP_PHOTCAL); + subset[i].ubercalDist = 1000; + } + } + + // XXX consider allowing relphot to skip the AstroMap load (do we need precise astrometry here?) + + // NOTE: unlock & free when running relphot_client (update-objects), but not when running relphot_images + // note that if we have MOSAIC_ZEROPT selected, we cannot free images as the full array is needed by initMosaic + + /* unlock, if we can (else, unlocked below) and free, if we can */ + if (unlockImages && !MOSAIC_ZEROPT) { + if (subset != image) { + // if we have generated an image subset and we are running UPDATE_OFFSETS, the we can free images here + free (image); + db[0].ftable.buffer = NULL; + BuildChipMatch (subset, Nsubset); + } + dvo_image_unlock (db); + // allocate and init an array to identify the images included in the subset + } + + char mapfile[DVO_MAX_PATH]; + snprintf (mapfile, DVO_MAX_PATH, "%s/AstroMap.fits", CATDIR); + table = AstromOffsetMapLoad (mapfile, 100000, VERBOSE); + + // assign images.coords.offsetMap -> table->map[i] + if (table) { + AstromOffsetTableMatchChips (subset, Nsubset, table); + } + + // match chips to mosaics (if applicable) + initMosaics (subset, Nsubset, image, inSubset, Nimage); + MARKTIME("init mosaics: %f sec\n", dtime); + free (inSubset); + + // save the subset of images in the static reference in ImageOps, set up indexes + initImages (subset, LineNumber, Nsubset); + MARKTIME("init images: %f sec\n", dtime); + + // assign chip images to tgroups + initTGroups (subset, Nsubset); + + return TRUE; +} + +int reload_images (FITS_DB *db) { + + Image *image; + off_t Nimage, Nx, i, *LineNumber; + VTable *vtable; + + image = getimages (&Nimage, &LineNumber); + + gfits_vtable_from_ftable (&db[0].ftable, &db[0].vtable, LineNumber, Nimage); + vtable = &db[0].vtable; + + gfits_scan (vtable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + for (i = 0; i < Nimage; i++) { + memcpy (vtable[0].buffer[i], &image[i], Nx); + } + return (TRUE); +} + +void put_astrom_table (AstromOffsetTable *myTable) { + table = myTable; +} + +void free_astrom_table () { + if (!table) return; + AstromOffsetTableFree (table); + free (table); +} + + + +// *** old example code *** + // determine the populated SkyRegions overlapping the requested area +# if (0) + // XXX note : this chunk selects catalogs for a region across the 0,360 boundary (e.g., 350 10) + // it has largely been obviated, but I'm keeping the code in case I need to reinstate it. + if (region[0].Rmin > region[0].Rmax) { + SkyRegion subregion; + subregion = *region; + subregion.Rmin = 0.0; subregion.Rmax = region[0].Rmax; + skylist = SkyListByPatch (sky, -1, &subregion); + + // select the images which overlap the selected sky regions + // 'subset' points to a new copy of the data (different from 'image') + subset = select_images (skylist, image, Nimage, inSubset, &LineNumber, &Nsubset, &subregion); + MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime); + + Image *subsetExtra; + off_t NsubsetExtra; + off_t *LineNumberExtra, i; + + subregion.Rmin = region[0].Rmin; subregion.Rmax = 360.0; + SkyList *extraList = SkyListByPatch (sky, -1, &subregion); + + subsetExtra = select_images (extraList, image, Nimage, inSubset, &LineNumberExtra, &NsubsetExtra, &subregion); + MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime); + + REALLOCATE (subset, Image, MAX (Nsubset + NsubsetExtra, 1)); + REALLOCATE (LineNumber, off_t, MAX (Nsubset + NsubsetExtra, 1)); + + for (i = 0; i < NsubsetExtra; i++) { + subset[i+Nsubset] = subsetExtra[i]; + LineNumber[i+Nsubset] = LineNumberExtra[i]; + } + Nsubset += NsubsetExtra; + MARKTIME("selected %d overlapping images: %f sec\n", (int) NsubsetExtra, dtime); + } +# endif + Index: /branches/eam_branches/relphot.20210521/src/myIndex.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/myIndex.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/myIndex.c (revision 41602) @@ -0,0 +1,77 @@ +# include "relphot.h" + +myIndexType *myIndexInit () { + + myIndexType *myIndex; + ALLOCATE (myIndex, myIndexType, 1); + + myIndex->minID = MAX_INT; + myIndex->maxID = 0; + + myIndex->NINDEX = 1000; + myIndex->Nindex = 0; + + ALLOCATE (myIndex->index, int, myIndex->NINDEX); + + return myIndex; +} + +int myIndexFree (myIndexType *myIndex) { + + if (!myIndex) return TRUE; + free (myIndex->index); + free (myIndex); + return TRUE; +} + +// set minID and maxID externally +int myIndexUpdateLimits (myIndexType *myIndex, int value) { + + myIndex->maxID = MAX(value, myIndex->maxID); + myIndex->minID = MIN(value, myIndex->minID); + + return TRUE; +} + +// once minID and maxID are set, this function defines the range and inits +int myIndexSetRange (myIndexType *myIndex) { + + myIndex->Nindex = myIndex->maxID - myIndex->minID + 1; + myIndex->NINDEX = myIndex->Nindex; + + REALLOCATE (myIndex->index, int, myIndex->NINDEX); + + int i; + for (i = 0; i < myIndex->Nindex; i++) { + myIndex->index[i] = -1; + } + + return TRUE; +} + +// once minID and maxID are set, this function defines the range and inits +int myIndexSetEntry (myIndexType *myIndex, int value, int entry) { + + int n = value - myIndex->minID; + + myAssert (n >= 0, "impossible!"); + + if (myIndex->index[n] != -1) { + fprintf (stderr, "skipping duplicate ID value: %d\n", value); + return -1; + } + + myIndex->index[n] = entry; + return n; +} + +// once minID and maxID are set, this function defines the range and inits +int myIndexGetEntry (myIndexType *myIndex, int value) { + + if (value < myIndex->minID) return -1; // not in this index + if (value > myIndex->maxID) return -1; // not in this index + + int n = value - myIndex->minID; + + return myIndex->index[n]; +} Index: /branches/eam_branches/relphot.20210521/src/plot_scatter.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/plot_scatter.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/plot_scatter.c (revision 41602) @@ -0,0 +1,80 @@ +# include "relphot.h" + +void plot_scatter (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) { + + off_t i, j, k, m, N, Ntot; + float Mrel, Mcal, Mmos, Mgrid; + double *xlist, *ylist, *ilist; + Graphdata graphdata; + + Ntot = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + Ntot += catalog[i].averageT[j].Nmeasure; + } + } + ALLOCATE (xlist, double, Ntot); + ALLOCATE (ylist, double, Ntot); + ALLOCATE (ilist, double, Ntot); + + int Nsecfilt = GetPhotcodeNsecfilt (); + + int Ns; + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + N = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + /* calculate the average value for a single star */ + if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue; + m = catalog[i].averageT[j].measureOffset; + + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + // skip measurements that do not match the current photcode + int ecode = GetPhotcodeEquivCodebyCode (catalog[i].measureT[m].photcode); + if (ecode != thisCode) { continue; } + + if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue; + Mcal = getMcal (m, i, MAG_CLASS_PSF); + if (isnan(Mcal)) continue; + Mmos = getMmos (m, i); + if (isnan(Mmos)) continue; + Mgrid = getMgrid (m, i); + if (isnan(Mgrid)) continue; + + Mrel = catalog[i].secfilt[Nsecfilt*j+Nsec].MpsfChp; + if (isnan(Mrel)) continue; + + xlist[N] = Mrel; + ylist[N] = PhotSysTiny (&catalog[i].measureT[m], &catalog[i].averageT[j], &catalog[i].secfilt[j*Nsecfilt], MAG_CLASS_PSF) - Mcal - Mmos - Mgrid - Mrel; + ilist[N] = PhotInstTiny (&catalog[i].measureT[m], MAG_CLASS_PSF); + N++; + } + } + } + + if (N == 0) { + fprintf (stderr, "no valid average values yet\n"); + continue; + } + + plot_defaults (&graphdata); + graphdata.xmin = PlotMmin; + graphdata.xmax = PlotMmax; + graphdata.ymin = PlotdMmin; + graphdata.ymax = PlotdMmax; + plot_list (&graphdata, xlist, ylist, N, "mag vs dmag", "%s.Mag.png", OUTROOT); + + plot_defaults (&graphdata); + graphdata.ymin = PlotdMmin; + graphdata.ymax = PlotdMmax; + plot_list (&graphdata, ilist, ylist, N, "imag vs dmag", "%s.iMag.png", OUTROOT); + } + free (xlist); + free (ylist); + free (ilist); +} Index: /branches/eam_branches/relphot.20210521/src/plotstuff.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/plotstuff.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/plotstuff.c (revision 41602) @@ -0,0 +1,194 @@ +# include "relphot.h" +# include + +static int Xgraph[5] = {0,0,0,0,0}; +static int active; + +/* +static union { unsigned char c[4]; float f; } f_undef = { {0xff, 0xff, 0xff, 0xfe} }; +# define fUNDEF (f_undef.f) +*/ + +static union { unsigned char c[8]; float d; } d_undef = { {0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00} }; +# define dUNDEF (d_undef.d) + +void XDead () { + signal (SIGPIPE, XDead); + fprintf (stderr, "kapa is dead, must restart\n"); + Xgraph[active] = -1; +} + +int get_graph (int N) { + return Xgraph[N]; +} + +int open_graph (int N) { + + char name[100]; + + sprintf (name, "gastro:%d", N); + + // if -plot is supplied, the plots are shown on the screen; + // otherwise, only the final plots are generated and saved without display + if (PLOTSTUFF) { + Xgraph[N] = KapaOpen ("kapa", name); + } else { + Xgraph[N] = KapaOpen ("kapa -noX", name); + } + + if (Xgraph[N] < 0) { + fprintf (stderr, "error starting kapa\n"); + return (FALSE); + } + return (TRUE); +} + +void DonePlotting (Graphdata *graphmode, int N) { + + if (Xgraph[N] == 0) return; + KapaBox (Xgraph[N], graphmode); + return; +} + +void JpegPlot (Graphdata *graphmode, int N, char *filename) { + OHANA_UNUSED_PARAM(graphmode); + + if (Xgraph[N] == 0) return; + + KapaPNG (Xgraph[N], filename); + return; +} + +void PSPlot (Graphdata *graphmode, int N, char *filename) { + OHANA_UNUSED_PARAM(graphmode); + + if (Xgraph[N] == 0) return; + + KiiPS (Xgraph[N], filename, TRUE, KAPA_PS_NEWPLOT, "default"); + return; +} + +void PrepPlotting (int Npts, Graphdata *graphmode, int N) { + + if (Npts < 1) return; + + if (Xgraph[N] < 1) if (!open_graph(N)) return; + + KapaClearSections (Xgraph[N]); + + KapaSetLimits (Xgraph[N], graphmode); + KapaSetFont (Xgraph[N], "helvetica", 14); + KapaBox (Xgraph[N], graphmode); + // KapaSendLabel (Xgraph[N], "PSF Mag", KAPA_LABEL_XM); + // KapaSendLabel (Xgraph[N], "Ap Mag - PSF Mag", KAPA_LABEL_YM); + + KapaPrepPlot (Xgraph[N], Npts, graphmode); + return; +} + +void PlotLabel (char *string, int N) { + + if (Xgraph[N] == 0) return; + + KapaSendLabel (Xgraph[N], string, 2); +} + +void PlotVector (int Npts, double *vect, int mode, int N, char *type) { + OHANA_UNUSED_PARAM(mode); + + float *values; + int i; + + if (Npts < 1) return; + + ALLOCATE (values, float, Npts); + for (i = 0; i < Npts; i++) { + values[i] = vect[i]; + } + + KapaPlotVector (Xgraph[N], Npts, values, type); + free (values); + return; +} + +void plot_wait () { + if (PLOTDELAY > 500000) { + fprintf (stdout, "press return\n"); + if (fscanf (stdin, "%*c") != 1) fprintf (stderr, "\n"); + } else { + usleep (PLOTDELAY); + } +} + +// plot the vector pair to a file with name defined by the varargs format +void plot_list (Graphdata *graphdata, double *xlist, double *ylist, int N, char *label, char *format, ...) { + + char tmp, *filename; + int i, Nbyte; + StatType stats; + va_list argp; + + va_start (argp, format); + Nbyte = vsnprintf (&tmp, 0, format, argp); + va_end (argp); + if (!Nbyte) return; + Nbyte ++; + + ALLOCATE (filename, char, Nbyte); + va_start (argp, format); + vsnprintf (filename, Nbyte, format, argp); + va_end (argp); + + stats.min = stats.max = xlist[0]; + for (i = 0; i < N; i++) { + stats.min = MIN (stats.min, xlist[i]); + stats.max = MAX (stats.max, xlist[i]); + } + if (graphdata[0].xmin == dUNDEF) graphdata[0].xmin = 1.05*stats.min - 0.05*stats.max; + if (graphdata[0].xmax == dUNDEF) graphdata[0].xmax = 1.05*stats.max - 0.05*stats.min; + + stats.min = stats.max = ylist[0]; + for (i = 0; i < N; i++) { + stats.min = MIN (stats.min, ylist[i]); + stats.max = MAX (stats.max, ylist[i]); + } + if (graphdata[0].ymin == dUNDEF) graphdata[0].ymin = 1.05*stats.min - 0.05*stats.max; + if (graphdata[0].ymax == dUNDEF) graphdata[0].ymax = 1.05*stats.max - 0.05*stats.min; + + PrepPlotting (N, graphdata, 0); + PlotVector (N, xlist, 0, 0, "x"); + PlotVector (N, ylist, 1, 0, "y"); + if (label != NULL) PlotLabel (label, 0); + DonePlotting (graphdata, 0); + + if (SAVEPLOT) JpegPlot (graphdata, 0, filename); + plot_wait(); + free (filename); +} + +// plot the vector pair to a file with name defined by the varargs format +void plot_list_add (Graphdata *graphdata, double *xlist, double *ylist, int Npts) { + + KapaPrepPlot (Xgraph[0], Npts, graphdata); + PlotVector (Npts, xlist, 0, 0, "x"); + PlotVector (Npts, ylist, 1, 0, "y"); + plot_wait(); +} + +void plot_defaults (Graphdata *graphdata) { + + KapaInitGraph (graphdata); + graphdata[0].style = KAPA_PLOT_POINTS; /* points */ + graphdata[0].ptype = 2; + graphdata[0].ltype = 0; + graphdata[0].etype = 0; + graphdata[0].color = black; + graphdata[0].lweight = 0; + graphdata[0].size = 1.0; + + graphdata[0].xmin = dUNDEF; + graphdata[0].xmax = dUNDEF; + graphdata[0].ymin = dUNDEF; + graphdata[0].ymax = dUNDEF; + +} Index: /branches/eam_branches/relphot.20210521/src/psps_ids.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/psps_ids.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/psps_ids.c (revision 41602) @@ -0,0 +1,43 @@ +# include "relphot.h" + +// Compute IDs for PSPS based on recipe they defined + +uint64_t CreatePSPSDetectionID(double tobs, int ccdid, int detID) { + // t0 for detection id is 2007-01-01 00:00:00 utc + double t0 = 54101.0; + double diff = floor( 100000. * (tobs - t0) ); + int itmp = diff; + + // ccdid must be < 100 + uint64_t detectid = 1000000000*((uint64_t) itmp) + 10000000 * ((uint64_t) ccdid) + + ((uint64_t) detID); + + return detectid; +} + +uint64_t CreatePSPSStackDetectionID(int sourceID, int imageID, int detID) { + // sourceID : ID of database + table that tracked the image (< 0x7f = 127) + // imageID : external ID of the image which provided the detections (< 0x1000.0000 ~ 2.7e8) + // detID : detection sequence in image (< 0x1000.0000 ~ 2.7e8) + + assert (detID < 0x10000000); + assert (imageID < 0x10000000); + assert (sourceID < 0x7f); + + uint64_t detectid = ((uint64_t)sourceID << 56) + ((uint64_t)imageID << 28) + (uint64_t)detID; + return detectid; +} + +uint64_t CreatePSPSObjectID(double ra, double dec) { + double zh = 0.0083333; + double zid = (dec + 90.) / zh; // 0 - 180*60*2 = 21600 < 15 bits + int izone = (int) floor(zid); + double zresid = zid - ((float) izone); // 0.0 - 1.0 + + uint64_t part1, part2, part3; + part1 = (uint64_t)( izone * 10000000000000LL) ; + part2 = ((uint64_t)(ra * 1000000.)) * 10000 ; // 0 - 360*1e6 = 3.6e8 (< 29 bits) + part3 = (int) (zresid * 10000.0) ; // 0 - 10000 (1 bit == 30/10000 arcsec) (< 14 bits) + + return part1 + part2 + part3; +} Index: /branches/eam_branches/relphot.20210521/src/reload_catalogs.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/reload_catalogs.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/reload_catalogs.c (revision 41602) @@ -0,0 +1,317 @@ +# include "relphot.h" + +# define DEBUG 1 + +# define TIMESTAMP(TIME) \ + gettimeofday (&stop, (void *) NULL); \ + dtime = DTIME (stop, start); \ + TIME += dtime; \ + gettimeofday (&start, (void *) NULL); + +void reload_catalogs (SkyList *skylist, FlatCorrectionTable *flatcorr, int hostID, char *hostpath) { + + int i; + int status; + struct stat filestat; + Catalog catalog; + + struct timeval start, stop; + double dtime = 0.0; + double time1 = 0.0; + double time2 = 0.0; + double time3a = 0.0; + double time3b = 0.0; + double time4 = 0.0; + double time5 = 0.0; + double time6 = 0.0; + double time7 = 0.0; + + // XXX need to decide how to determine PARALLEL mode... + if (PARALLEL && !hostID) { + reload_catalogs_parallel (skylist); + return; + } + + // load the ZP corrections here + if (SYNTH_ZERO_POINTS) SynthZeroPointsLoad (SYNTH_ZERO_POINTS); + + if (VERBOSE) fprintf (stderr, "re-loading catalog data\n"); + + /* load data from each region file */ + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + gettimeofday (&start, (void *) NULL); + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // only update existing db tables + status = stat (catalog.filename, &filestat); + if ((status == -1) && (errno == ENOENT)) { + if (VERBOSE) fprintf (stderr, "no file %s, skipping\n", catalog.filename); + continue; + } + TIMESTAMP(time1); + + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } + catalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s, skipping\n", catalog.filename); + continue; + } + if (VERBOSE && (catalog.Naverage_disk == 0)) { + fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + TIMESTAMP(time2); + + populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE); + TIMESTAMP(time3a); + + // XXX need to worry about the image subset data + initImageBins (&catalog, 1, FALSE); + initMosaicBins (&catalog, 1, FALSE); + initTGroupBins (&catalog, 1); + + initGridBins (&catalog, 1); + TIMESTAMP(time3b); + + findImages (&catalog, 1, FALSE); + findMosaics (&catalog, 1, FALSE); + findTGroups (&catalog, 1); + + TIMESTAMP(time4); + + initMrel (&catalog, 1); + setMrelFinal (&catalog, flatcorr, FALSE); + TIMESTAMP(time5); + + // modify the output format as desired (ignore current format on disk) + if (UPDATE_CATFORMAT) { + catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } + + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "RELPHOT", "%s", 1, moddate); + free (moddate); + + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + + free_tiny_values(&catalog); + dvo_catalog_free (&catalog); + TIMESTAMP(time6); + + // XXX freeTgroupBins here: + + freeImageBins (1, FALSE); + freeMosaicBins (1, FALSE); + freeTGroupBins (1); + freeGridBins (1); + TIMESTAMP(time7); + + if (hostID) { + client_logger_message ("updated catalog file %s\n", catalog.filename); + } + } + + fprintf (stderr, "time step 1 %10.3f sec : find catalog\n", time1); + fprintf (stderr, "time step 2 %10.3f sec : load catalog\n", time2); + fprintf (stderr, "time step 3 %10.3f sec : make tiny values\n", time3a); + fprintf (stderr, "time step 4 %10.3f sec : init imbins\n", time3b); + fprintf (stderr, "time step 5 %10.3f sec : find images\n", time4); + fprintf (stderr, "time step 6 %10.3f sec : set Mrel\n", time5); + fprintf (stderr, "time step 7 %10.3f sec : save catalog\n", time6); + fprintf (stderr, "time step 8 %10.3f sec : free catalog\n", time7); +} + +int reload_catalog_parallel_group (HostTableGroup *group, SkyList *sky, char *imageFile); + +// XXX Image to Image Subset +int reload_catalogs_parallel (SkyList *sky) { + + // name of image subset file: + char imageFile[512]; + snprintf (imageFile, 512, "%s/Images.subset.dat", CATDIR); + + // the ApplyOffsets option re-uses an existing Images.subset.dat file + if (!ApplyOffsets) { + off_t Nimage; + ImageSubset *image = getimages_subset (&Nimage); + if (!ImageSubsetSave (imageFile, image, Nimage)) { + fprintf (stderr, "failed to write image subset\n"); + exit (1); + } + free (image); + } + + // now launch the relphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: problem with parallel host table\n"); + exit (2); + } + + if (BOUNDARY_TREE) { + char *tmppath = abspath(BOUNDARY_TREE, DVO_MAX_PATH); + free (BOUNDARY_TREE); + BOUNDARY_TREE = tmppath; + } + if (SYNTH_ZERO_POINTS) { + char *tmppath = abspath(SYNTH_ZERO_POINTS, DVO_MAX_PATH); + free (SYNTH_ZERO_POINTS); + SYNTH_ZERO_POINTS = tmppath; + } + + int Ngroups; + HostTableGroup *groups = HostTableGroupsUniqueMachines (table, &Ngroups); + // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines) + + int i; + for (i = 0; i < Ngroups; i++) { + // update only a group of unique machines at a time + if (i < SKIP_PARALLEL_GROUPS) continue; + reload_catalog_parallel_group (&groups[i], sky, imageFile); + } + + for (i = 0; i < Ngroups; i++) { + free (groups[i].hosts); + } + free (groups); + FreeHostTable (table); + + return TRUE; +} + +int reload_catalog_parallel_group (HostTableGroup *group, SkyList *sky, char *imageFile) { + + int i, j; + for (i = 0; i < group->Nhosts; i++) { + + if (sky->Nregions < group->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], group->hosts[i][0].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", group->hosts[i][0].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH); + free (group->hosts[i][0].pathname); + group->hosts[i][0].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relphot_client %s -update-catalogs %s", PhotcodeList, imageFile); + strextend (&command, "-hostID %d -D CATDIR %s -hostdir %s", group->hosts[i][0].hostID, CATDIR, group->hosts[i][0].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "-statmode %s", STATMODE); + strextend (&command, "-D CAMERA %s", CAMERA); + strextend (&command, "-D STAR_TOOFEW %d -minerror %f", STAR_TOOFEW, MIN_ERROR); + + // options & configs which affect relphot_client -update-catalogs + // VERBOSE, VERBOSE2 + // RESET + // RESET_ZEROPTS + // TimeSelect + // AreaSelect + // STATMODE + // STAR_TOOFEW + // MIN_ERROR + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (RESET_ZEROPTS) { strextend (&command, "-reset-zpts"); } + if (PRESERVE_PS1) { strextend (&command, "-preserve-ps1"); } + if (UPDATE) { strextend (&command, "-update"); } + if (IS_DIFF_DB) { strextend (&command, "-is-diff-db"); } + if (!KEEP_UBERCAL) { strextend (&command, "-reset-ubercal"); } + if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } + if (BOUNDARY_TREE) { strextend (&command, "-boundary-tree %s", BOUNDARY_TREE); } + if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); } + if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } + + if (!(STAGES & STAGE_CHIP)) { strextend (&command, "-skip-chip"); } + if (!(STAGES & STAGE_WARP)) { strextend (&command, "-skip-warp"); } + if (!(STAGES & STAGE_STACK)) { strextend (&command, "-skip-stack"); } + + // deprecate + // if (SET_MREL_VERSION != 1) { snprintf (tmpline, 1024, "%s -set-mrel-version %d", SET_MREL_VERSION); } // XXXX deprecate this... + + if (AreaSelect) { strextend (&command, "-area %f %f %f %f", AreaXmin, AreaXmax, AreaYmin, AreaYmax); } + if (TimeSelect) { + char *tstart = ohana_sec_to_date (TSTART); + char *tstop = ohana_sec_to_date (TSTOP); + strextend (&command, "-time %s %s", tstart, tstop); + free (tstart); + free (tstop); + } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relphot_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", group->hosts[i][0].hostname, command, group->hosts[i][0].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", group->hosts[i][0].hostname, errorInfo); + exit (1); + } + group->hosts[i][0].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableGroupWaitJobsGetIO (group, __FILE__, __LINE__, VERBOSE); + if (!status) { + fprintf (stderr, "at least one remote client job failed to load data, exiting\n"); + exit (3); + } + } + return (TRUE); +} Index: /branches/eam_branches/relphot.20210521/src/relphot.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot.c (revision 41602) @@ -0,0 +1,87 @@ +# include "relphot.h" + +int main (int argc, char **argv) { + + // get configuration info, args + SetSignals (); + RelphotMode mode = initialize (argc, argv); + if (!mode) exit (2); + + SkyList *skylist = NULL; + + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE); + if (!sky) { + fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR); + exit (2); + } + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + if ((mode != PARALLEL_REGIONS) && (mode != PARALLEL_IMAGES)) { + skylist = NULL; + if (UserCatalog) { + int Nchar = strlen(UserCatalog); + if (!strcmp (&UserCatalog[Nchar-4], ".cpt")) UserCatalog[Nchar-4] = 0; + skylist = SkyListByName (sky, UserCatalog); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + if (!skylist) { + fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR); + exit (2); + } + } + + switch (mode) { + case UPDATE_IMAGES: + // IF CALLED WITH NLOOP == 0, DOES NOT LOAD bcatalog, just loads image table and generates ImageSubset.dat + relphot_images (skylist); + relphot_free (sky, skylist); + exit (0); + + case UPDATE_AVERAGES: + // take the current set of detections and set the mean magnitudes + relphot_objects (skylist, 0, NULL); + relphot_free (sky, skylist); + exit (0); + + case SYNTH_PHOT: + // set the mean magnitudes ONLY for SYNPHOT objects + relphot_synthphot (skylist, 0, NULL); + relphot_free (sky, skylist); + exit (0); + + case PARALLEL_REGIONS: + // run image updates in parallel across multiple remote machines + relphot_parallel_regions (sky); + relphot_free (sky, skylist); + exit (0); + + case PARALLEL_IMAGES: + // operation on the remote machines in the PARALLEL_REGION mode + relphot_parallel_images (sky); + relphot_free (sky, skylist); + exit (0); + + case APPLY_OFFSETS: + // re-run this step from a previous attempt (assumes an existing Images.subset.dat file) + if (!PARALLEL) { + fprintf (stderr, "-apply-offsets only makes sense in an parallel context\n"); + exit (2); + } + reload_catalogs (skylist, NULL, 0, NULL); + relphot_free (sky, skylist); + exit (0); + + default: + fprintf (stderr, "ERROR: no valid relphot mode chosen\n"); + exit (2); + } + fprintf (stderr, "IMPOSSIBLE: skipped out of switch?\n"); + exit (1); +} + +/*** + + I would like to merge the functionality of relphot_images (Nloop == 0), relphot_objects, and apply_offsets + +***/ Index: /branches/eam_branches/relphot.20210521/src/relphot_client.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_client.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_client.c (revision 41602) @@ -0,0 +1,113 @@ +# include "relphot.h" + +// relphot_client is run on a remote host and is responsible for updating the catalogs +// owned by that host. + +// there are two modes: + +// relphot_client -load : extract the bright catalog subset from the client's tables and +// save to a local FITS table for relphot to load and used + +// (relphot loads the bright catalog subsets from all clients, then uses this to determine +// the per-image zero points (and potentially the flat-field corrections) + +// relphot_client -update : load image table containing the update zero point information +// and apply this to the client's tables + +int main (int argc, char **argv) { + + // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, etc) + SetSignals (); + initialize_client (argc, argv); + client_logger_init (HOSTDIR); + + // load the current sky table (layout of all SkyRegions) if needed + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE); + if (!sky) { + fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR); + exit (2); + } + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + SkyList *skylist = SkyListByPatch (sky, -1, &UserPatch); + if (!skylist) { + fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR); + exit (2); + } + client_logger_message ("loaded sky table and defined patch\n"); + + switch (MODE) { + case MODE_LOAD: { + int Ncatalog; + Catalog *catalog = load_catalogs (skylist, &Ncatalog, HOST_ID, HOSTDIR, NULL); + if (!catalog) { + fprintf (stderr, "ERROR loading catalogs from %s\n", CATDIR); + exit (2); + } + client_logger_message ("loaded catalogs\n"); + + BrightCatalog *bcatalog = BrightCatalogMerge (catalog, Ncatalog); + client_logger_message ("generated subset data\n"); + + if (!BrightCatalogSave (BCATALOG, bcatalog)) { + fprintf (stderr, "ERROR saving bright catalog from %s\n", CATDIR); + exit (2); + } + + // free memory associated with the catalogs + for (int i = 0; i < Ncatalog; i++) { + free_tiny_values (&catalog[i]); + dvo_catalog_free (&catalog[i]); + } + free (catalog); + BrightCatalogFree (bcatalog); + client_logger_message ("generated subset table\n"); + relphot_client_free (sky, skylist); + break; + } + + case MODE_UPDATE: { + // load the image subset table from the specified location + off_t Nimage; + ImageSubset *image = ImageSubsetLoad (IMAGES, &Nimage); + if (!image) { + fprintf (stderr, "ERROR loading image subset %s\n", CATDIR); + exit (2); + } + client_logger_message ("loaded image subset data\n"); + + // save the available image information in the static array in ImageOps.c + initImagesSubset (image, NULL, Nimage); + + // load the flat-field correction table from CATDIR + FlatCorrectionTable *flatcorr = NULL; + reload_catalogs (skylist, flatcorr, HOST_ID, HOSTDIR); + freeImages ((char *)image); + free (image); + client_logger_message ("updated catalogs\n"); + relphot_client_free (sky, skylist); + break; + } + + case MODE_UPDATE_OBJECTS: { + // take the current set of detections and set the mean magnitudes + relphot_objects (skylist, HOST_ID, HOSTDIR); + relphot_client_free (sky, skylist); + client_logger_message ("updated objects\n"); + break; + } + + case MODE_SYNTH_PHOT: + // set the mean magnitudes ONLY for SYNPHOT objects + relphot_synthphot (skylist, HOST_ID, HOSTDIR); + client_logger_message ("set synth photometry\n"); + break; + + default: + fprintf (stderr, "impossible!"); + abort(); + } + client_logger_message ("done with relphot_client\n"); + exit (0); +} + Index: /branches/eam_branches/relphot.20210521/src/relphot_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_images.c (revision 41602) @@ -0,0 +1,211 @@ +# include "relphot.h" + +// 25.0 is the nominal zero point for measurements in the database +// for all measurements except PHOT_REF: +// measure.M contains: -2.5*log(DN) + 2.5*log(exptime) + 25.0 +// for measurements using PHOT_REF: +// measure.M contains: -2.5*log(DN) + 2.5*log(exptime) + True Zero Point (but exptime and ZP are known) + +int relphot_images (SkyList *skylist) { + + int i, status, Ncatalog; + Catalog *catalog = NULL; + FITS_DB db; + + INITTIME; + + /* register database handle with shutdown procedure */ + set_db (&db); + gfits_db_init (&db); + db.mode = dvo_catalog_catmode (CATMODE); + db.format = dvo_catalog_catformat (CATFORMAT); + + /* lock and load the image db table */ + status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); + if (!status && UPDATE) { + fprintf (stderr, "error\n"); + Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + } + + // if the file is missing, db.dbstate will have a value of either: + // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE) + if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) { + Shutdown ("ERROR: database %s contains no image data", CATDIR); + } + + // read data from Image.dat file + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + MARKTIME("-- load image data: %f sec\n", dtime); + + /* load regions and images based on specified sky patch and/or catalog */ + load_images (&db, skylist, &UserPatch, FALSE, USE_ALL_IMAGES); + MARKTIME("-- load images: %f sec\n", dtime); + + /* unlock, if we can (else, unlocked below) */ + if (!UPDATE) dvo_image_unlock (&db); + + // load the flat correction table (if defined) + // we no longer need to do this: the flatcorrection is now saved in measure.Mflat +# if (0) + char flatcorrfile[256]; + sprintf (flatcorrfile, "%s/flatcorr.fits", CATDIR); + FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrfile, VERBOSE); +# endif + FlatCorrectionTable *flatcorr = NULL; + + if (CALIBRATE_STACKS_AND_WARPS || (NLOOP > 0)) { + /* load catalog data from region files (hostID is 0 since we are not a client */ + catalog = load_catalogs (skylist, &Ncatalog, 0, NULL, NULL); + MARKTIME("-- load catalog data: %f sec\n", dtime); + + /* add in a loop over the catalogs calling dvo_catalog_chipcoords */ + + /* match measurements with images, mosaics */ + initImageBins (catalog, Ncatalog, TRUE); + MARKTIME("-- make image bins: %f sec\n", dtime); + + initMosaicBins (catalog, Ncatalog, TRUE); + initTGroupBins (catalog, Ncatalog); + + initGridBins (catalog, Ncatalog); + initMrel (catalog, Ncatalog); + + findImages (catalog, Ncatalog, TRUE); + MARKTIME("-- set up image indexes: %f sec\n", dtime); + + findMosaics (catalog, Ncatalog, TRUE); /* also sets Grid values */ + MARKTIME("-- set up mosaic indexes: %f sec\n", dtime); + + findTGroups (catalog, Ncatalog); + MARKTIME("-- set up mosaic indexes: %f sec\n", dtime); + + SAVEPLOT = FALSE; + + setExclusions (catalog, Ncatalog, TRUE); + + global_stats (catalog, Ncatalog, flatcorr, 0); + + if (PLOTSTUFF) { + plot_star_coords (catalog, Ncatalog); + // plot_mosaic_fields (catalog); + } + + // if we are measuring the flat-field correction grid, we need to perform a number of iterations first: + if (USE_GRID) { + int star_toofew; + + // until we finish the grid analysis, do not reject stars out-of-hand based on ID_OBJ_FEW + // XXX this is kind of poor: need to have a better distinctions about STAR_BAD in setMrel vs getMrel + star_toofew = STAR_TOOFEW; + STAR_TOOFEW = 0; + for (i = 0; i < NGRID; i++) { + STAR_BAD = ID_OBJ_POOR; + setMrel (catalog, Ncatalog, flatcorr); + STAR_BAD = ID_OBJ_POOR | ID_OBJ_FEW; + setMgrid (catalog, flatcorr); + } + STAR_BAD = ID_OBJ_POOR | ID_OBJ_FEW; + STAR_TOOFEW = star_toofew; + } + + /* determine fit values */ + for (i = 0; i < NLOOP; i++) { + SetZptIteration (i); + + setMrel (catalog, Ncatalog, flatcorr); // threaded + + setMcal (catalog, flatcorr); + setMmos (catalog, flatcorr); + setMgrp (catalog, flatcorr); + + setMgrid (catalog, flatcorr); + MARKTIME("-- set Mrel, Mcal, Mmos, Mgrid : %f sec\n", dtime); + + if (PLOTSTUFF) { + plot_scatter (catalog, Ncatalog, flatcorr); + plot_grid (catalog, flatcorr); + plot_mosaics (); + plot_images (); + plot_stars (catalog, Ncatalog); + plot_chisq (catalog, Ncatalog); + } + + global_stats (catalog, Ncatalog, flatcorr, i); + SetZeroPointModes (); + + MARKTIME("-- finished loop %d: %f sec\n", i, dtime); + fprintf (stderr, "----- continue loop %d -----\n", i); + } + + if (PLOTSTUFF) { + plot_scatter (catalog, Ncatalog, flatcorr); + plot_grid (catalog, flatcorr); + plot_mosaics (); + plot_images (); + plot_stars (catalog, Ncatalog); + plot_chisq (catalog, Ncatalog); + } + + if (USE_GRID) dump_grid (); + + MARKTIME("-- finalize Mcal values: %f sec\n", dtime); + + setMcalFromMosaics (); // copy per-mosaic calibrations to the images + setMcalFromTGroups (); // copy per-tgroup calibrations to the images + + // calculate and save per-chip residuals here: + MagResidSave ("mag.resid.fits", catalog); + + save_images_updates (&db); + + /* at this point, we have correct cal coeffs in the image/mosaic structures */ + for (i = 0; i < Ncatalog; i++) { + // these tiny values are set by BrightCatalogSplit from load_catalogs + free_tiny_values (&catalog[i]); + dvo_catalog_free (&catalog[i]); + } + free (catalog); + freeImageBins (Ncatalog, TRUE); + freeMosaicBins (Ncatalog, TRUE); + freeTGroupBins (Ncatalog); + freeGridBins (Ncatalog); + + // end of if (NLOOP > 0) block : this loop determines the offsets per chip + } else { + // If nloop == 0, the above pass is not performed, in which case + // the Mcal values passed to the mosaics are not returned to the images... + setMcalFromMosaics (); // copy per-mosaic calibrations to the images + setMcalFromTGroups (); // copy per-tgroup calibrations to the images + } + + reload_images (&db); + + // only change the real database files if -update is requested + if (!UPDATE) exit (0); + + /* Load catalog data from region files, update Mrel include all data. In a parallel + context, this function writes the image parameters as a subset table for the remote + clients */ + if (!CALIBRATE_STACKS_AND_WARPS) { + reload_catalogs (skylist, flatcorr, 0, NULL); + MARKTIME("-- updated all catalogs: %f sec\n", dtime); + } + + if (UPDATE_CATFORMAT) { + // ensure the db format is updated + db.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); + } + if (CALIBRATE_STACKS_AND_WARPS || (NLOOP > 0)) { + // do not save changes if we did not make changes. + dvo_image_update (&db, VERBOSE); + } + dvo_image_unlock (&db); + + freeImages (db.ftable.buffer); + freeMosaics (); + gfits_db_free (&db); + + return TRUE; +} + Index: /branches/eam_branches/relphot.20210521/src/relphot_objects.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_objects.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_objects.c (revision 41602) @@ -0,0 +1,308 @@ +# include "relphot.h" + +// apply the current calibration values to the measurements to determine the average +// magnitudes, applying the desired clipping analysis. Note that this does not depend on +// the form of the image zero point measurement. If there is a grid or map term, and this +// has been correctly propagated to the measurement, the average will respect that value. + +int relphot_objects_parallel (SkyList *sky); + +int relphot_objects (SkyList *skylist, int hostID, char *hostpath) { + + off_t i, j, k; + int Nsecfilt; + struct stat filestat; + + Catalog catalog; + + INITTIME; + + // XXX need to decide how to determine PARALLEL mode... + if (PARALLEL && !hostID) { + relphot_objects_parallel (skylist); + return TRUE; + } + + FITS_DB db; + + set_db (&db); + gfits_db_init (&db); + + /* lock and load the image db table */ + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_SOFT); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + MARKTIME("read image table: %f sec\n", dtime); + // the raw FITS data is freed by dvo_image_load, leaving only the Image structure data + + /* load regions and images based on specified sky patch (default depth) */ + load_images (&db, skylist, &UserPatch, TRUE, USE_ALL_IMAGES); + MARKTIME("loaded images: %f sec\n", dtime); + + // load the ZP corrections here + if (SYNTH_ZERO_POINTS) SynthZeroPointsLoad (SYNTH_ZERO_POINTS); + + if (REPAIR_WARPS) { + FindWarpGroups (); + MARKTIME("setup warp groups: %f sec\n", dtime); + MakeStackIndex (); + MARKTIME("setup stack index: %f sec\n", dtime); + RepairWarpMeasuresOpenLogfile (); + } + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // only update existing db tables + int status = stat (catalog.filename, &filestat); + if ((status == -1) && (errno == ENOENT)) { + if (VERBOSE) fprintf (stderr, "no file %s, skipping\n", catalog.filename); + continue; + } + + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + // if we are going to update the format, we should update all tables + if (UPDATE_CATFORMAT) { + catalog.catflags |= DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; + } + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s, skipping\n", catalog.filename); + continue; + } + if (!catalog.Naverage_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + // reset the calculated average magnitudes (does not affect ubercal-tied measurements or images) + if (RESET) { + Nsecfilt = catalog.Nsecfilt; + + DVOAverageFlags photomBits = + ID_OBJ_EXT | // extended in our data (eg, PS) + ID_OBJ_EXT_ALT | // extended in external data (eg, 2MASS) + ID_OBJ_GOOD | // good-quality measurement in our data (eg,PS) + ID_OBJ_GOOD_ALT | // good-quality measurement in external data (eg, 2MASS) + ID_OBJ_GOOD_STACK | // good-quality object in the stack (> 1 good stack) + ID_OBJ_BEST_STACK | // the primary stack measurement are the best measurements + ID_OBJ_SUSPECT_STACK | // suspect object in the stack (> 1 good or suspect stack, < 2 good) + ID_OBJ_BAD_STACK; // good-quality object in the stack (> 1 good stack) + + for (j = 0; j < catalog.Naverage; j++) { + catalog.average[j].flags &= ~photomBits; // reset all except astrometry bits + catalog.average[j].psfQF = NAN; // reset (will be re-calculated here) + catalog.average[j].psfQFperf = NAN; // reset (will be re-calculated here) + catalog.average[j].stargal = NAN; // reset (will be re-calculated here) + catalog.average[j].photFlagsUpper = 0; // reset (will be re-calculated here) + catalog.average[j].photFlagsLower = 0; // reset (will be re-calculated here) + catalog.average[j].NwarpOK = 0; // reset (will be re-calculated here) + for (k = 0; k < Nsecfilt; k++) { + dvo_secfilt_init (&catalog.secfilt[j*Nsecfilt + k], SECFILT_RESET_ALL); + } + } + } + + populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE); + + // XXX need to worry about the image subset data + initImageBins (&catalog, 1, FALSE); + initMosaicBins (&catalog, 1, FALSE); + initGridBins (&catalog, 1); + + findImages (&catalog, 1, FALSE); + findMosaics (&catalog, 1, FALSE); + + // update the detection coordinates using the new image parameters + if (REPAIR_WARPS) RepairWarpMeasures (&catalog); + + initMrel (&catalog, 1); + setMrelFinal (&catalog, NULL, TRUE); + + // uncomment for extra verbosity + // MARKTIME("setMrelFinal for "OFF_T_FMT" average "OFF_T_FMT" measure : %f sec\n", catalog.Naverage, catalog.Nmeasure, dtime); + // gettimeofday (&startTimer, NULL); // reset timer + + // XXX if we want to have options for setting warp, chip, stack independently, we need to init only the desired ones + // NOTE flatcorr == NULL, but it should have been applied already by setphot + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + free_tiny_values(&catalog); + dvo_catalog_free (&catalog); + freeImageBins (1, FALSE); + freeMosaicBins (1, FALSE); + freeGridBins (1); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + char history[128]; + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "RELPHOT", "%s", 1, moddate); + if (REPAIR_WARPS) { + snprintf (history, 128, "repair warp measure.imageID: %s", moddate); + gfits_modify_alt (&catalog.header, "HISTORY", "%S", 0, history); // adds a new entry + } + free (moddate); + + // we can optionally convert output format here + // but it would be better to define a dvo crawler program to do this + // catalog.catformat = DVO_FORMAT_PS1_V1; + + // modify the output format as desired (ignore current format on disk) + if (UPDATE_CATFORMAT) { + catalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT); + } + + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + + free_tiny_values(&catalog); + dvo_catalog_free (&catalog); + + freeImageBins (1, FALSE); + freeMosaicBins (1, FALSE); + freeGridBins (1); + } + + if (REPAIR_WARPS) { + FreeWarpGroups (); + FreeStackGroups (); + RepairWarpMeasuresCloseLogfile (); + } + + freeImages(db.ftable.buffer); + gfits_db_free (&db); + + return (TRUE); +} + +// CATDIR is supplied globally +# define DEBUG 1 +int relphot_objects_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + if (BOUNDARY_TREE) { + char *tmppath = abspath(BOUNDARY_TREE, DVO_MAX_PATH); + free (BOUNDARY_TREE); + BOUNDARY_TREE = tmppath; + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + // options / arguments that can affect relphot_client -update-objects: + // VERBOSE, VERBOSE2 + // KEEP_UBERCAL + // RESET (-reset) + // TimeSelect -time + // DophotSelect + // (note that psfQF is applied rigidly at 0.85, as is the galaxy test) + // MAG_LIM + // SIGMA_LIM + // ImagSelect, ImagMin, ImagMax + // MaxDensityUse, MaxDensityValue + + char *command = NULL; + strextend (&command, "relphot_client -update-objects"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "-statmode %s", STATMODE); + strextend (&command, "-D CAMERA %s", CAMERA); + strextend (&command, "-D STAR_TOOFEW %d", STAR_TOOFEW); + strextend (&command, "-minerror %f", MIN_ERROR); + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (RESET) { strextend (&command, "-reset"); } + if (RESET_ZEROPTS) { strextend (&command, "-reset-zpts"); } + if (REPAIR_WARPS) { strextend (&command, "-repair-warps"); } + if (PRESERVE_PS1) { strextend (&command, "-preserve-ps1"); } + if (IS_DIFF_DB) { strextend (&command, "-is-diff-db"); } + if (UPDATE) { strextend (&command, "-update"); } + if (!KEEP_UBERCAL) { strextend (&command, "-reset-ubercal"); } + if (UPDATE_CATFORMAT) { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); } + if (BOUNDARY_TREE) { strextend (&command, "-boundary-tree %s", BOUNDARY_TREE); } + if (SYNTH_ZERO_POINTS) { strextend (&command, "-synthphot-zpts %s", SYNTH_ZERO_POINTS); } + if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } + if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } + if (USE_MCAL_PSF_FOR_STACK_APER) { strextend (&command, "-use-mcal-psf-for-stack-aper"); } + + if (!(STAGES & STAGE_CHIP)) { strextend (&command, "-skip-chip"); } + if (!(STAGES & STAGE_WARP)) { strextend (&command, "-skip-warp"); } + if (!(STAGES & STAGE_STACK)) { strextend (&command, "-skip-stack"); } + + // if (SET_MREL_VERSION != 1) { strextend (&command, "-set-mrel-version %d", SET_MREL_VERSION); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relphot_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable (table); + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/relphot_parallel_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_parallel_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_parallel_images.c (revision 41602) @@ -0,0 +1,173 @@ +# include "relphot.h" + +/* This function is essentially identical to relphot_images, except: + + * load the subset images saved by the master node + * distinguish detections we own (touch our images) and those we don't + * distinguish objects we own (in region) and those we don't + * update the unowned detections for owned objects from neighbor regions + * update the unowned objects for owned detections + */ + +int dumpObjects (char *filename, Catalog *catalog, int Ncatalog); + +int relphot_parallel_images (SkyTable *sky) { + + int i, Ncatalog; + Catalog *catalog = NULL; + + INITTIME; + + client_logger_init (CATDIR); + + // load the RegionTable (UserRegion should not be used at this level) + RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE); + int myHost = regionHosts->index[REGION_HOST_ID]; + RegionHostFindNeighbors (regionHosts, myHost); + client_logger_message ("started parallel images on %s\n", regionHosts->hosts[myHost].hostname); + + // load the subset images belonging to this host + off_t Nimage; + Image *image = ImageTableLoad (IMAGE_TABLE, &Nimage); + if (!image) { + fprintf (stderr, "ERROR loading image %s\n", IMAGE_TABLE); + exit (2); + } + client_logger_message ("loaded images\n"); + + makeMosaics (image, Nimage, TRUE); + + initImages (image, NULL, Nimage); + + // load the flat correction table (if defined) + // we no longer need to do this: the flatcorrection is now saved in measure.Mflat +# if (0) + char flatcorrfile[256]; + sprintf (flatcorrfile, "%s/flatcorr.fits", CATDIR); + FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrfile, VERBOSE); +# endif + FlatCorrectionTable *flatcorr = NULL; + + // UserPatch.Rmin,Rmax may have range from a few degrees < 0.0 to few degrees > 360.0. + // the following function correctly chooses the sky regions on the 0,360 boundary + SkyList *skylist = SkyListByBounds (sky, -1, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + client_logger_message ("prepare to load catalogs\n"); + + /* load catalog data from region files (hostID is 0 since we are not a client */ + char *syncfile = make_filename (CATDIR, regionHosts->hosts[myHost].hostname, REGION_HOST_ID, "loadcat.sync"); + catalog = load_catalogs (skylist, &Ncatalog, 0, NULL, syncfile); + MARKTIME("-- load catalog data: %f sec\n", dtime); + free (syncfile); + + client_logger_message ("loaded catalog data\n"); + + // generate tables go from catID,objID -> catSeq,objSeq + indexCatalogs (catalog, Ncatalog); + client_logger_message ("indexed Catalogs\n"); + + /* match measurements with images, mosaics */ + initImageBins (catalog, Ncatalog, TRUE); + MARKTIME("-- make image bins: %f sec\n", dtime); + + initMosaicBins (catalog, Ncatalog, TRUE); + initMrel (catalog, Ncatalog); + + findImages (catalog, Ncatalog, TRUE); + MARKTIME("-- set up image indexes: %f sec\n", dtime); + + findMosaics (catalog, Ncatalog, TRUE); /* also sets Grid values */ + MARKTIME("-- set up mosaic indexes: %f sec\n", dtime); + + client_logger_message ("done setting up indexes\n"); + + // dumpObjects ("test.obj.dat", catalog, Ncatalog); + markObjects (catalog, Ncatalog); + + SAVEPLOT = FALSE; + + setExclusions (catalog, Ncatalog, TRUE); + + global_stats (catalog, Ncatalog, flatcorr, 0); + + if (PLOTSTUFF) { + plot_star_coords (catalog, Ncatalog); + // plot_mosaic_fields (catalog); + } + + /* determine fit values */ + client_logger_message ("starting the loops : %d \n", NLOOP); + for (i = 0; i < NLOOP; i++) { + SetZptIteration (i); + + // set the mean stellar mags given the measurements and the image calibrations + setMrel (catalog, Ncatalog, flatcorr); + + // share mean mags for objects at the boundary (number of unowned meas > 0) + share_mean_mags (catalog, Ncatalog, regionHosts, i); + client_logger_message ("shared mean mag data : loop %d \n", i); + + // load mean mags from other region hosts + slurp_mean_mags (catalog, Ncatalog, regionHosts, i); + client_logger_message ("slurped mean mag data : loop %d \n", i); + + // set the image (Mcal) and mosaic (Mmos) zero point offsets given the mean mags and measurements + setMcal (catalog, flatcorr); + setMmos (catalog, flatcorr); + // setMgrp (catalog, FALSE, flatcorr); XXX think this through: this may not make sense for tgroups + MARKTIME("-- set Mrel, Mcal, Mmos, Mgrid : %f sec\n", dtime); + + // share image mags for images with non-zero unowned detections + share_image_mags (regionHosts, i); + client_logger_message ("shared image data : loop %d \n", i); + + slurp_image_mags (regionHosts, i); + client_logger_message ("slurped image data : loop %d \n", i); + + global_stats (catalog, Ncatalog, flatcorr, i); + + SetZeroPointModes (); + MARKTIME("-- finished loop %d: %f sec\n", i, dtime); + } + client_logger_message ("done with loops\n"); + + // this is a checkpoint to make sure all hosts have finished the loop above + char *loopsyncfile = make_filename (CATDIR, regionHosts->hosts[myHost].hostname, REGION_HOST_ID, "loop.sync"); + update_sync_file (loopsyncfile, 0); + free (loopsyncfile); + + for (i = 0; i < regionHosts->Nhosts; i++) { + if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue; + char *loopsync = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "loop.sync"); + check_sync_file (loopsync, 0); + free (loopsync); + } + client_logger_message ("all hosts are done the loops\n"); + + MARKTIME("-- finalize Mcal values: %f sec\n", dtime); + + setMcalFromMosaics (); // copy per-mosaic calibrations to the images + // XXX keep? setMcalFromTGroups (); // copy per-tgroup calibrations to the images + + share_image_mags (regionHosts, -1); + + for (i = 0; i < Ncatalog; i++) { + // these tiny values are set by BrightCatalogSplit from load_catalogs + free_tiny_values (&catalog[i]); + dvo_catalog_free (&catalog[i]); + } + free (catalog); + + freeCatalogIndexes (Ncatalog); + freeImageBins(Ncatalog, TRUE); + freeMosaicBins (Ncatalog, TRUE); + freeGridBins (Ncatalog); + freeImages((char *)image); + free (image); + + SkyListFree(skylist); + FreeRegionHostTable (regionHosts); + + client_logger_message ("done with parallel images\n"); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/relphot_parallel_regions.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_parallel_regions.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_parallel_regions.c (revision 41602) @@ -0,0 +1,101 @@ +# include "relphot.h" + +int relphot_parallel_regions (SkyTable *sky) { + + int status; + FITS_DB db; + + INITTIME; + + // load the RegionTable (UserRegion should not be used at this level) + RegionHostTable *regionHosts = RegionHostTableLoad (CATDIR, REGION_FILE); + if (!regionHosts) { + fprintf (stderr, "ERROR: problem with region host table\n"); + exit (2); + } + + // load the flat correction table (if defined) +# if (0) + char flatcorrfile[256]; + sprintf (flatcorrfile, "%s/flatcorr.fits", CATDIR); + FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrfile, VERBOSE); +# endif + FlatCorrectionTable *flatcorr = NULL; + + // register database handle with shutdown procedure + set_db (&db); + gfits_db_init (&db); + db.mode = dvo_catalog_catmode (CATMODE); + db.format = dvo_catalog_catformat (CATFORMAT); + + // lock and load the image db table + status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); + if (!status && UPDATE) { + fprintf (stderr, "error\n"); + Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + } + + if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) { + // if the Image.dat file is missing, db.dbstate will have a value of either: + // LCK_EMPTY (if UPDATE) or LCK_MISSING (if !UPDATE) + Shutdown ("ERROR: database %s contains no image data", CATDIR); + } + + // read data from Image.dat file + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + MARKTIME("-- load image data: %f sec\n", dtime); + + // save Images.dat using the copied structure + if (UPDATE_CATFORMAT) { + // ensure the db format is updated + db.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&db.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); + } + + /* assign the images to the different region hosts */ + if (!assign_images (&db, regionHosts)) Shutdown ("error assigning images to region hosts"); + MARKTIME("-- assign images: %f sec\n", dtime); + + /* launch processing on the parallel region hosts */ + if (!launch_region_hosts (regionHosts)) Shutdown ("error launching region hosts"); + MARKTIME ("finished relphot -parallel-regions: %f sec total\n", dtime); + + // retrieve updated image parameters from the remote hosts (also set Image.mcal) + if (!slurp_image_mags (regionHosts, -1)) Shutdown ("error loading image updates"); + + if (!UPDATE) { + dvo_image_unlock (&db); + MARKTIME ("finished relphot -parallel-regions: %f sec total\n", dtime); + fprintf (stderr, "NOTE: UPDATE is OFF (results are not saved)\n"); + exit (0); + } + + SkyList *skylist = SkyListByBounds (sky, -1, regionHosts->Rmin, regionHosts->Rmax, regionHosts->Dmin, regionHosts->Dmax); + UserPatch.Rmin = regionHosts->Rmin; + UserPatch.Rmax = regionHosts->Rmax; + UserPatch.Dmin = regionHosts->Dmin; + UserPatch.Dmax = regionHosts->Dmax; + + // I have to save a copy because dvo_image_save and _unlock swap and free the data + save_images_backup (&db); + + /* update catalogs (in parallel) */ + reload_catalogs (skylist, flatcorr, 0, NULL); + + // save the changes to the image parameters + dvo_image_save (&db, VERBOSE); + + // dvo_image_save frees db.ftable.buffer (== image) and replaces it: do not free stored Image table + clearImages (); + + dvo_image_unlock (&db); + MARKTIME ("finished relphot -parallel-regions: %f sec total\n", dtime); + + freeImages(db.ftable.buffer); + gfits_db_free (&db); + + SkyListFree(skylist); + FreeRegionHostTable (regionHosts); + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/relphot_synthphot.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_synthphot.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_synthphot.c (revision 41602) @@ -0,0 +1,145 @@ +# include "relphot.h" + +// 1) Look for objects which have only synthetic photometry for any of the grizy filters. +// 2) apply spatial zero point offsets to synth photometry, then set mean mag based on synth. + +int relphot_synthphot_parallel (SkyList *sky); + +int relphot_synthphot (SkyList *skylist, int hostID, char *hostpath) { + + off_t i; + + Catalog catalog; + + // XXX need to decide how to determine PARALLEL mode... + if (PARALLEL && !hostID) { + relphot_synthphot_parallel (skylist); + return TRUE; + } + + // load the ZP corrections here + SynthZeroPointsLoad (SYNTH_ZERO_POINTS); + SynthZeroPoints *zpts = SynthZeroPointsGet (); + if (!zpts) exit (2); + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naverage_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + relphot_synthphot_catalog (&catalog, zpts); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + // we can optionally convert output format here + // but it would be better to define a dvo crawler program to do this + // catalog.catformat = DVO_FORMAT_PS1_V1; + SetProtect (TRUE); + if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); } + if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); } + SetProtect (FALSE); + dvo_catalog_free (&catalog); + } + + return (TRUE); +} + +// CATDIR is supplied globally +# define DEBUG 1 +int relphot_synthphot_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + // ensure that the path for the zero point is absolute + { + char *tmppath = abspath (SYNTH_ZERO_POINTS, DVO_MAX_PATH); + free (SYNTH_ZERO_POINTS); + SYNTH_ZERO_POINTS = tmppath; + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + // options / arguments that can affect relphot_client -synthphot_means: + // VERBOSE, VERBOSE2 + + char *command = NULL; + strextend (&command, "relphot_client -synthphot_means %s", SYNTH_ZERO_POINTS); + strextend (&command, "-hostID %d -D CATDIR %s -hostdir %s", table->hosts[i].hostID, CATDIR, table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) { strextend (&command, "-v"); } + if (VERBOSE2) { strextend (&command, "-vv"); } + if (UPDATE) { strextend (&command, "-update"); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relphot_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relphot_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/relphot_synthphot_catalog.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/relphot_synthphot_catalog.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/relphot_synthphot_catalog.c (revision 41602) @@ -0,0 +1,103 @@ +# include "relphot.h" + +// we've just reloaded the data from disk; we now need to apply the Image/Mosaic/Grid +// calibrations determined by the rest of the program. We also need to set the final +// output dbFlags values + +int relphot_synthphot_catalog (Catalog *catalog, SynthZeroPoints *zpts) { + + off_t j; + + int Nsecfilt = catalog[0].Nsecfilt; + + for (j = 0; j < catalog[0].Naverage; j++) { + off_t m = catalog[0].average[j].measureOffset; + Average *average = &catalog[0].average[j]; + Measure *measure = &catalog[0].measure[m]; + SecFilt *secfilt = &catalog[0].secfilt[j*Nsecfilt]; + + relphot_synthphot_average (average, secfilt, measure, zpts); + } + return TRUE; +} + +// set mean of chip measurements (selected by photcode range for now): +int relphot_synthphot_average (Average *average, SecFilt *secfilt, Measure *measure, SynthZeroPoints *zpts) { + + int i; + off_t k; + + // first, determine the state of things for each of the main filters: + // in this bit, we are using hardwired photcode values: + // PS1_g, PS1_r, PS1_i, PS1_z, PS1_y, PS1_w == (1,2,3,4,5,6); 0 == none + // note that Nsec = PS_N - 1 + + int havePS1[7]; + int haveSYN[7]; + int needSYN[7]; + int measSYN[7]; + + // was: 13.64, 13.76, 13.74, 12.94, 12.01 + float MaxMagForceSynth[] = {0.0, 13.5, 13.5, 13.5, 13.0, 12.0, 13.5}; + + for (i = 0; i < 7; i++) { + havePS1[i] = FALSE; // do we have any PS1 measurement for this filter? + haveSYN[i] = FALSE; // do we have any synthetic measurement for this filter? + needSYN[i] = FALSE; // force the use of the synthetic measurement for this filter? + measSYN[i] = -1; // which entries carries the synthetic measurement for this filter? + } + + // need to look up the (X,Y) coords from this (R,D) location + double X, Y; + double R = ohana_normalize_angle_to_midpoint (average->R, 0.0); + RD_to_XY (&X, &Y, R, average->D, &zpts->coords); + if (X < 0) return FALSE; + if (Y < 0) return FALSE; + if (X >= zpts->Nx) return FALSE; + if (Y >= zpts->Ny) return FALSE; + + int Xpix = X; + int Ypix = Y; + int Npix = Xpix + Ypix*zpts->Nx; + + for (k = 0; k < average[0].Nmeasure; k++) { + + // skip measurements that do not have a valid photcode + PhotCode *code = GetPhotcodebyCode (measure[k].photcode); + if (!code) continue; + + // is this synth_grizy? + // is this PS1? + + // are we a PS1 exposure photcode? + if (isGPC1chip(measure[k].photcode)) { + int Nfilter = whichGPC1filter (measure[k].photcode); + havePS1[Nfilter] = TRUE; + } + if (isGPC1synth(measure[k].photcode)) { + int Nfilter = measure[k].photcode - 3000; + haveSYN[Nfilter] = TRUE; + measSYN[Nfilter] = k; + + // force the use of SYN even if we have PS1 mags? + if (measure[k].M < MaxMagForceSynth[Nfilter]) { + needSYN[Nfilter] = TRUE; + } + } + } + + for (i = 1; i < 6; i++) { + if (needSYN[i] || (haveSYN[i] && !havePS1[i])) { + int Nsec = i - 1; + float *value = (float *) zpts->matrix[Nsec].buffer; + float ZP = !isnan(value[Npix]) ? value[Npix] : 0.0; + secfilt[Nsec].MpsfChp = measure[measSYN[i]].M + ZP; + secfilt[Nsec].dMpsfChp = 0.6; + secfilt[Nsec].Mchisq = NAN; + secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; + } + } + + return (TRUE); +} + Index: /branches/eam_branches/relphot.20210521/src/save_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/save_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/save_images.c (revision 41602) @@ -0,0 +1,108 @@ +# include "relphot.h" + +// XXX there are two versions of this code: +// save_image_updates () is called by relphot_images +// save_image_backup () is called by relphot_parallel_regions + +// rationalize these to a single function... + +int save_images_updates (FITS_DB *db) { + + if (!SAVE_IMAGE_UPDATES) return TRUE; + + FITS_DB dbX; + gfits_db_init (&dbX); + dbX.lockstate = LCK_XCLD; + dbX.timeout = 60.0; + dbX.mode = db->mode; + dbX.format = db->format; + + char filename[1024]; + snprintf (filename, 1024, "%s.bck", ImageCat); // ImageCat is global + if (!gfits_db_lock (&dbX, filename)) { + fprintf (stderr, "can't lock backup image image catalog\n"); + return FALSE; + } + + // apply the changes from the image subset to the full table: + + // convert database table to internal structure (binary to Image) + // 'image' points to the same memory as db->ftable->buffer + + off_t Nimage, Nx; + Image *image = gfits_table_get_Image (&db->ftable, &Nimage, &db->scaledValue, &db->nativeOrder); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + gfits_scan (db->ftable.header, "NAXIS1", OFF_T_FMT, 1, &Nx); + + off_t *LineNumber; + off_t Nsubset; + Image *subset = getimages (&Nsubset, &LineNumber); + for (off_t i = 0; i < Nsubset; i++) { + if (LineNumber[i] == -1) continue; + memcpy (&image[LineNumber[i]], &subset[i], Nx); + } + + // copy Images.dat data (all or only the subset / vtable elements?) I think I need to dump + // the entire Image table, but with the updates in place I think this says: re-work the + // ftable/vtable usage in this program to be more sensible... + gfits_copy_header (&db->header, &dbX.header); + gfits_copy_matrix (&db->matrix, &dbX.matrix); + gfits_copy_header (&db->theader, &dbX.theader); + gfits_copy_ftable (&db->ftable, &dbX.ftable); + + dbX.ftable.header = &dbX.theader; + dbX.virtual = FALSE; + + // save Images.dat using the copied structure + if (UPDATE_CATFORMAT) { + // ensure the db format is updated + dbX.format = dvo_catalog_catformat (UPDATE_CATFORMAT); + gfits_modify (&dbX.header, "FORMAT", "%s", 1, UPDATE_CATFORMAT); + + char photcodeFile[1024]; + sprintf (photcodeFile, "%s/Photcodes.dat", CATDIR); + SavePhotcodesFITS (photcodeFile); + } + + dvo_image_save (&dbX, VERBOSE); + dvo_image_unlock (&dbX); + gfits_db_free (&dbX); + return TRUE; +} + +int save_images_backup (FITS_DB *db) { + + FITS_DB dbX; + char filename[1024]; + + gfits_db_init (&dbX); + dbX.lockstate = LCK_XCLD; + dbX.timeout = 60.0; + dbX.mode = db->mode; + dbX.format = db->format; + + snprintf (filename, 1024, "%s.bck", ImageCat); + if (!gfits_db_lock (&dbX, filename)) { + fprintf (stderr, "can't lock backup image image catalog\n"); + return (FALSE); + } + + // copy Images.dat data (all or only the subset / vtable elements?) I think I need to dump + // the entire Image table, but with the updates in place I think this says: re-work the + // ftable/vtable usage in this program to be more sensible... + gfits_copy_header (&db->header, &dbX.header); + gfits_copy_matrix (&db->matrix, &dbX.matrix); + gfits_copy_header (&db->theader, &dbX.theader); + gfits_copy_ftable (&db->ftable, &dbX.ftable); + + dbX.ftable.header = &dbX.theader; + + dvo_image_save (&dbX, VERBOSE); + dvo_image_unlock (&dbX); + + gfits_db_free (&dbX); + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/select_images.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/select_images.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/select_images.c (revision 41602) @@ -0,0 +1,441 @@ +# include "relphot.h" + +/* this function returns a list of all images which overlap the given SkyList (set of + SkyRegions). All images in the image catalog are tested once, so there is no check that an + image already has been included. LineNum stores the locations in the Image database of the + list of images */ + +typedef struct { + double Xc[5]; + double Yc[5]; + double Rc; + double Dc; +} SkyRegionCoords; + +void MakeSkyCoordIndex (SkyRegionCoords **skycoords_out, double **RmaxSky_out, off_t **index_out, SkyList *skylist); +void dsortindex (double *X, off_t *Y, int N); +off_t getRegionStartByRA (double R, double *Rref, off_t Nregions); + +Image *select_images (SkyList *skylist, Image *timage, off_t Ntimage, char *inSubset, off_t **LineNumber, off_t *Nimage, SkyRegion *region) { + + Image *image; + off_t i, j, k, m, nStart, iSky, nimage, NIMAGE, D_NIMAGE; + off_t *line_number; + int InRange, found; + double Ri[5], Di[5], Xi[5], Yi[5]; + Coords tcoords; + + if (skylist[0].Nregions < 1) { + *Nimage = 0; + *LineNumber = NULL; + if (VERBOSE) fprintf (stderr, "no matching sky regions\n"); + return NULL; + } + + INITTIME; + + // FILE *ftest = fopen ("image.region.dat", "w"); + + // the comparison is made in the catalog local projection. below we set crval1,2 + InitCoords (&tcoords, "DEC--TAN"); + tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; + + double RminSkyRegion = region[0].Rmin; + double RmaxSkyRegion = region[0].Rmax; + double DminSkyRegion = region[0].Dmin; + double DmaxSkyRegion = region[0].Dmax; + + double dD = CALIBRATE_STACKS_AND_WARPS ? 0.0 : 5.0; + double dR = dD / cos(RAD_DEG*DminSkyRegion); + RminSkyRegion -= dR; + RmaxSkyRegion += dR; + DminSkyRegion -= dD; + DmaxSkyRegion += dD; + + double RmidSkyRegion = 0.5*(RminSkyRegion + RmaxSkyRegion); + + // generate a set of Rmin/Rmax values using this RmidSkyRegion for DEC bands of width + // dDecBand (0.1 deg). DecIndex = (Dec + 90) / dDecBand + double *RminBand, *RmaxBand; + float dDecBand = 0.1; + int NDecBands = 180 / dDecBand + 1; + ALLOCATE (RminBand, double, NDecBands); + ALLOCATE (RmaxBand, double, NDecBands); + for (i = 0; i < NDecBands; i++) { + RminBand[i] = +360.0; + RmaxBand[i] = -360.0; + } + + /* compare with each region file */ + for (i = 0; i < skylist[0].Nregions; i++) { + int iDecBandMin = (skylist[0].regions[i][0].Dmin + 90.0) / dDecBand; + int iDecBandMax = (skylist[0].regions[i][0].Dmax + 90.0) / dDecBand; + + double RminAlt = ohana_normalize_angle_to_midpoint (skylist[0].regions[i][0].Rmin, RmidSkyRegion); + double RmaxAlt = ohana_normalize_angle_to_midpoint (skylist[0].regions[i][0].Rmax, RmidSkyRegion); + + for (j = iDecBandMin; j <= iDecBandMax; j++) { + RminBand[j] = MIN(RminBand[j], RminAlt); + RmaxBand[j] = MAX(RmaxBand[j], RmaxAlt); + } + } + + SkyRegionCoords *skycoords = NULL; + double *RmaxSky = NULL; + off_t *index = NULL; + + if (!USE_BASIC_CHECK) { + MakeSkyCoordIndex (&skycoords, &RmaxSky, &index, skylist); + } + + if (VERBOSE) fprintf (stderr, "finding images\n"); + + D_NIMAGE = 6000; + + nimage = 0; + NIMAGE = D_NIMAGE; + ALLOCATE (image, Image, NIMAGE); + ALLOCATE (line_number, off_t, NIMAGE); + + // go through the complete list of images, selecting ones which overlap any region + for (i = 0; i < Ntimage; i++) { + + if (!(i % 300000)) fprintf (stderr, "."); + + // XXX I am just going for force this for the moment: + // goto found_it; + + /* exclude images by photcode (we are not doing this, but I'm not sure why) */ + if (timage[i].photcode) { + int Ns = GetActivePhotcodeIndex (timage[i].photcode); + if (Ns < 0) continue; + } + + /* exclude images by time */ + if (TimeSelect) { + if (timage[i].tzero < TSTART) continue; + if (timage[i].tzero > TSTOP) continue; + } + + /* define image corners - note the DIS images (mosaic phu) are special */ + if (!strcmp(&timage[i].coords.ctype[4], "-DIS")) { + Xi[0] = -0.5*timage[i].NX; Yi[0] = -0.5*timage[i].NY; + Xi[1] = +0.5*timage[i].NX; Yi[1] = -0.5*timage[i].NY; + Xi[2] = +0.5*timage[i].NX; Yi[2] = +0.5*timage[i].NY; + Xi[3] = -0.5*timage[i].NX; Yi[3] = +0.5*timage[i].NY; + Xi[4] = -0.5*timage[i].NX; Yi[4] = -0.5*timage[i].NY; + } else { + Xi[0] = 0; Yi[0] = 0; + Xi[1] = timage[i].NX; Yi[1] = 0; + Xi[2] = timage[i].NX; Yi[2] = timage[i].NY; + Xi[3] = 0; Yi[3] = timage[i].NY; + Xi[4] = 0; Yi[4] = 0; + } + found = FALSE; + + /* transform corners to ra,dec -- costs ~3sec for 3M images (pikake) */ + double RminImage = RmidSkyRegion + 180.0; + double RmaxImage = RmidSkyRegion - 180.0; + double DminImage = +90.0; + double DmaxImage = -90.0; + for (j = 0; j < 5; j++) { + XY_to_RD (&Ri[j], &Di[j], Xi[j], Yi[j], &timage[i].coords); + Ri[j] = ohana_normalize_angle_to_midpoint (Ri[j], RmidSkyRegion); + + RminImage = MIN(RminImage, Ri[j]); + RmaxImage = MAX(RmaxImage, Ri[j]); + DminImage = MIN(DminImage, Di[j]); + DmaxImage = MAX(DmaxImage, Di[j]); + } + if (RmaxImage - RminImage > 180.0) { + double tmp = RminImage; + RminImage = RmaxImage - 360.0; + RmaxImage = tmp; + } + + // check that this image is even in range of the searched region + if (USE_FULL_OVERLAP) { + // for full overlap, the image must be completely inside the region of interest + if (DmaxImage > DmaxSkyRegion) continue; + if (DminImage < DminSkyRegion) continue; + + // the sky region RA is defined to be 0 - 360.0 + if (RmaxImage > RmaxSkyRegion) continue; + if (RminImage < RminSkyRegion) continue; + } else { + if (DminImage > DmaxSkyRegion) continue; + if (DmaxImage < DminSkyRegion) continue; + + // the sky region RA is defined to be 0 - 360.0 + if (RminImage > RmaxSkyRegion) continue; + if (RmaxImage < RminSkyRegion) continue; + } + + // the above checks are only valid for the outermost region. however, at a given + // declination, the range of RA is limited by the actual catalog boundaries + // check if this image is in range for the Dmax and Dmin locations + if (USE_FULL_OVERLAP && strcmp(&timage[i].coords.ctype[4], "-DIS")) { + + int iDecBandMin = (DminImage + 90.0) / dDecBand; + int iDecBandMax = (DmaxImage + 90.0) / dDecBand; + + // the sky region RA is defined to be 0 - 360.0 + if (RminImage < RminBand[iDecBandMin]) { + if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage)); + continue; + } + if (RminImage < RminBand[iDecBandMax]) { + if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage)); + continue; + } + if (RmaxImage > RmaxBand[iDecBandMin]) { + if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage)); + continue; + } + if (RmaxImage > RmaxBand[iDecBandMax]) { + if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage)); + continue; + } + } + + // fprintf (ftest, "%f %f : %f %f : %d %d\n", RminImage, RmaxImage, DminImage, DmaxImage, timage[i].tzero, timage[i].photcode); + + // image overlaps region, keep it + if (USE_BASIC_CHECK) goto found_it; + + // RA(nStart) is guaranteed to be < RminImage: -- costs 0.5sec for 3M images + nStart = getRegionStartByRA (RminImage, RmaxSky, skylist[0].Nregions); + + /* compare with each region file */ + for (iSky = nStart; (iSky < skylist[0].Nregions) && !found; iSky++) { + + m = index[iSky]; + + /* we make positional comparisons in the projection of catalog */ + tcoords.crval1 = skycoords[m].Rc; + tcoords.crval2 = skycoords[m].Dc; + + /* transform corner coords to X,Y in this catalog system */ + InRange = TRUE; + for (j = 0; (j < 5) && InRange; j++) { + InRange = RD_to_XY (&Xi[j], &Yi[j], Ri[j], Di[j], &tcoords); + } + if (!InRange) continue; + + /* check if image corner inside catalog */ + for (j = 0; (j < 4) && !found; j++) { + found = corner_check (&Xi[j], &Yi[j], &skycoords[m].Xc[0], &skycoords[m].Yc[0]); + if (found) goto found_it; + } + /* check if catalog corner inside image */ + for (j = 0; (j < 4) && !found; j++) { + found = corner_check (&skycoords[m].Xc[j], &skycoords[m].Yc[j], &Xi[0], &Yi[0]); + if (found) goto found_it; + } + /* check if edges cross */ + for (j = 0; (j < 4) && !found; j++) { + for (k = 0; (k < 4) && !found; k++) { + found = edge_check (&Xi[j], &Yi[j], &skycoords[m].Xc[k], &skycoords[m].Yc[k]); + if (found) goto found_it; + } + } + } + if (!found) continue; + + found_it: + image[nimage] = timage[i]; + inSubset[i] = TRUE; + line_number[nimage] = i; + nimage ++; + if (nimage == NIMAGE) { + NIMAGE += D_NIMAGE; + D_NIMAGE = MAX (100000, D_NIMAGE * 1.5); + REALLOCATE (image, Image, NIMAGE); + REALLOCATE (line_number, off_t, NIMAGE); + } + } + MARKTIME("\nfinish image selection: %f sec\n", dtime); + + // fclose (ftest); + + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage); + + REALLOCATE (image, Image, MAX (nimage, 1)); + REALLOCATE (line_number, off_t, MAX (nimage, 1)); + + FREE (RmaxBand); + FREE (RminBand); + + if (!USE_BASIC_CHECK) { + free (skycoords); + free (RmaxSky); + free (index); + } + + *Nimage = nimage; + *LineNumber = line_number; + return (image); +} + +/* check if line between points 0 and 1 of x1 + crosses line between points 0 and 1 of x2 */ +int edge_check (double *x1, double *y1, double *x2, double *y2) { + + double theta1, theta2; + + theta1 = opening_angle (x1[0], y1[0], x2[0], y2[0], x1[1], y1[1]); + theta2 = opening_angle (x1[0], y1[0], x2[0], y2[0], x2[1], y2[1]); + + if (theta1*theta2 < 0.0) { + return (FALSE); + } + + if (fabs(theta1) < fabs(theta2)) { + return (FALSE); + } + + theta1 = opening_angle (x2[0], y2[0], x1[1], y1[1], x2[1], y2[1]); + theta2 = opening_angle (x2[0], y2[0], x1[1], y1[1], x1[0], y1[0]); + + + if (theta1*theta2 < 0.0) { + return (FALSE); + } + + if (fabs(theta1) < fabs(theta2)) { + return (FALSE); + } + + return (TRUE); + +} + +/* check if point x1,y1 is in box formed by x2[0-4] */ +int corner_check (double *x1, double *y1, double *x2, double *y2) { + + int i; + double theta; + + theta = 0; + + for (i = 0; i < 4; i++) { + theta += opening_angle (x2[i], y2[i], x1[0], y1[0], x2[i+1], y2[i+1]); + } + if (fabs(theta) > 6) { + return (TRUE); + } else { + return (FALSE); + } +} + +/* returns the opening angle between the three points (2 is in middle) + in range -pi to pi */ + +double opening_angle (double x1, double y1, double x2, double y2, double x3, double y3) { + + double dx1, dy1, dx2, dy2, ct, st, theta; + + dx1 = x1 - x2; + dy1 = y1 - y2; + + dx2 = x3 - x2; + dy2 = y3 - y2; + + ct = (dx1*dx2 + dy1*dy2); + st = (dx1*dy2 - dx2*dy1); + + theta = atan2 (st, ct); + + return (theta); + +} + +void dsortindex (double *X, off_t *Y, int N) { + +# define SWAPFUNC(A,B){ double tmpf; off_t tmpi; \ + tmpf = X[A]; X[A] = X[B]; X[B] = tmpf; \ + tmpi = Y[A]; Y[A] = Y[B]; Y[B] = tmpi; \ +} +# define COMPARE(A,B)(X[A] < X[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +off_t getRegionStartByRA (double R, double *Rref, off_t Nregions) { + + // use bisection to find the overlapping mosaic + + off_t Nlo, Nhi, N; + + // find the last mosaic before start + Nlo = 0; Nhi = Nregions; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (Rref[N] < R) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N, Nregions); + } + } + return (Nlo); +} + +void MakeSkyCoordIndex (SkyRegionCoords **skycoords_out, double **RmaxSky_out, off_t **index_out, SkyList *skylist) { + + off_t i; + SkyRegionCoords *skycoords; + Coords tcoords; + double *RmaxSky; + off_t *index; + double dx, dy; + + // the comparison is made in the catalog local projection. below we set crval1,2 + InitCoords (&tcoords, "DEC--TAN"); + tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; + + /* compare with each region file */ + ALLOCATE (skycoords, SkyRegionCoords, skylist[0].Nregions); + + ALLOCATE (RmaxSky, double, skylist[0].Nregions); + ALLOCATE (index, off_t, skylist[0].Nregions); + + for (i = 0; i < skylist[0].Nregions; i++) { + + /* we make positional comparisons in the projection of catalog */ + skycoords[i].Rc = 0.5*(skylist[0].regions[i][0].Rmax + skylist[0].regions[i][0].Rmin); + skycoords[i].Dc = 0.5*(skylist[0].regions[i][0].Dmax + skylist[0].regions[i][0].Dmin); + tcoords.crval1 = skycoords[i].Rc; + tcoords.crval2 = skycoords[i].Dc; + + /* define catalog corners */ + RD_to_XY (&skycoords[i].Xc[0], &skycoords[i].Yc[0], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmin, &tcoords); + RD_to_XY (&skycoords[i].Xc[1], &skycoords[i].Yc[1], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmin, &tcoords); + RD_to_XY (&skycoords[i].Xc[2], &skycoords[i].Yc[2], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmax, &tcoords); + RD_to_XY (&skycoords[i].Xc[3], &skycoords[i].Yc[3], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmax, &tcoords); + skycoords[i].Xc[4] = skycoords[i].Xc[0]; + skycoords[i].Yc[4] = skycoords[i].Yc[0]; + + RmaxSky[i] = skylist[0].regions[i][0].Rmax; + index[i] = i; + + dx = 0.02*(skycoords[i].Xc[2] - skycoords[i].Xc[0]); + dy = 0.02*(skycoords[i].Yc[2] - skycoords[i].Yc[0]); + skycoords[i].Xc[0] -= dx; skycoords[i].Yc[0] -= dy; + skycoords[i].Xc[1] += dx; skycoords[i].Yc[1] -= dy; + skycoords[i].Xc[2] += dx; skycoords[i].Yc[2] += dy; + skycoords[i].Xc[3] -= dx; skycoords[i].Yc[3] += dy; + skycoords[i].Xc[4] -= dx; skycoords[i].Yc[4] -= dy; + } + + dsortindex (RmaxSky, index, skylist[0].Nregions); + + *RmaxSky_out = RmaxSky; + *index_out = index; + *skycoords_out = skycoords; + + return; +} Index: /branches/eam_branches/relphot.20210521/src/setExclusions.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/setExclusions.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/setExclusions.c (revision 41602) @@ -0,0 +1,60 @@ +# include "relphot.h" + +// this function sets the NOCAL and AREA dbFlags bits for the MeasureTiny elements these +// are used elsewhere (StarOps.c, ImageOps.c, MosaicOps.c, GridOps.c, etc) to skip bad +// measurements. The only exception is 'setMave' which is called by 'relphot_objects', +// and uses the bits read from disk as the test + +int setExclusions (Catalog *catalog, int Ncatalog, int verbose) { + + off_t i, j, k, m, Narea, Nnocal, Ngood; + Coords *coords; + double r, d, x, y; + + Ngood = Nnocal = Narea = 0; + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + m = catalog[i].averageT[j].measureOffset; + for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) { + + /* select measurements by photcode */ + int Ns = GetActivePhotcodeIndex (catalog[i].measureT[m].photcode); + if (Ns < 0) goto mark_nocal; + + /* select measurements by time */ + if (TimeSelect) { + if (catalog[i].measureT[m].t < TSTART) goto mark_nocal; + if (catalog[i].measureT[m].t > TSTOP) goto mark_nocal; + } + + /* select measurements by mag limit */ + if (AreaSelect) { + r = catalog[i].measureT[m].R; + d = catalog[i].measureT[m].D; + if ((coords = getCoords (m, i)) == NULL) goto markbad; + RD_to_XY (&x, &y, r, d, coords); + if (x < AreaXmin) goto markbad; + if (x > AreaXmax) goto markbad; + if (y < AreaYmin) goto markbad; + if (y > AreaYmax) goto markbad; + } + Ngood ++; + continue; + + markbad: + catalog[i].measureT[m].dbFlags |= ID_MEAS_AREA; + Narea ++; + continue; + + mark_nocal: + catalog[i].measureT[m].dbFlags |= ID_MEAS_NOCAL; + Nnocal ++; + continue; + } + } + } + if (verbose) fprintf (stderr, OFF_T_FMT" measurements marked by area\n", Narea); + if (verbose) fprintf (stderr, OFF_T_FMT" measurements marked nocal\n", Nnocal); + if (verbose) fprintf (stderr, OFF_T_FMT" measurements kept for analysis\n", Ngood); + return (TRUE); +} Index: /branches/eam_branches/relphot.20210521/src/setMrelCatalog.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/setMrelCatalog.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/setMrelCatalog.c (revision 41602) @@ -0,0 +1,1282 @@ +# include "relphot.h" + +# if (0) +# define TEST_OBJ_ID 0x000149b0 +# define TEST_CAT_ID 0x00000001 +# else +# define TEST_OBJ_ID 0 +# define TEST_CAT_ID 0 +# endif + +int magStatsByRanking (StatDataSet *dataset, StatType *stats); +int magStatsByRankingClipped (StatDataSet *dataset, StatType *stats); +int magStatsByRankingIRLS (StatDataSet *dataset, StatType *stats); +int markMeasureByRanking (StatDataSet *dataset, Measure *measure, int minrank, DVOMeasureFlags keepflag, DVOMeasureFlags maskflag); +void GetPhotFlagStats (uint32_t *photFlagUpper, uint32_t *photFlagLower, uint32_t *photflag_list, int Nphotflag); +void sort_entry_by_offset (double *offset, int *entry, int N); +void sort_StatDataSet (StatDataSet *dataset); +int liststats_irls (StatDataSet *dataset, int Npoints, StatType *stats); + +// # define MAG_STATS_BY_RANKING magStatsByRankingIRLS +// # define MAG_STATS_BY_RANKING magStatsByRanking + +# define UBERCAL_WEIGHT 100.0 + +# define SKIP_THIS_MEAS(REASON) { \ + results->REASON ++; \ + continue; } + +# define SKIP_THIS_MEAS_STACK(REASON) { \ + continue; } + +# define CHECK_VALID_MAG(MAG,D_MAG) (isfinite(MAG) && isfinite(D_MAG) && (MAG > -5.0) && (MAG < 30.0)) +# define CHECK_VALID_FLUX(FLUX,D_FLUX) (isfinite(FLUX) && isfinite(D_FLUX)) + +// set A to B if B > A or if A is NAN, as long as B is not NAN +# define MAX_NOT_NAN(A,B) { if (isfinite(B) && (!isfinite(A) || (B > A))) A = B; } + +static float MagToFlux (float Mag) { + float Flux = pow(10.0, -0.4*(Mag)); + return (Flux); +} + +int print_measure_set_alt (Average *average, SecFilt *secfilt, Measure *measure) { + + int Ns; + off_t k; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + for (k = 0; k < average[0].Nmeasure; k++) { + fprintf (stderr, "meas: %08x\n", measure[k].dbFlags); + } + + for (Ns = 0; Ns < Nsecfilt; Ns++) { + fprintf (stderr, "secf: %08x\n", secfilt[Ns].flags); + } + return (TRUE); +} + +int useOLS = TRUE; // chosen based on iteration below + +int setMrel_catalog_alt (Catalog *catalog, int Nc, int isSetMrelFinal, FlatCorrectionTable *flatcorr, SetMrelInfo *results, int Nsecfilt) { + + off_t j; + + // the same STATMODE is used for chip and warp averages + liststats_setmode (&results->psfstats, STATMODE); + liststats_setmode (&results->apstats, STATMODE); + liststats_setmode (&results->kronstats, STATMODE); + + SetMrelInfoReset (results); // reset the counters + + useOLS = UseStandardOLS (ZPT_STARS); + for (j = 0; j < catalog[Nc].Naverage; j++) { + + if (STAGES & STAGE_CHIP) { + setMrelAverageExposure (&catalog[Nc], Nc, j, Nsecfilt, isSetMrelFinal, flatcorr, results); + } + + // only apply Stack operation on setMrelFinal in first pass + if (isSetMrelFinal && (STAGES & STAGE_STACK) && !IS_DIFF_DB) { + setMrelAverageStack (&catalog[Nc], Nc, j, Nsecfilt, flatcorr); + } + + // only measure force-warp mean values if issetMrelFinal (make it optional?) + if (isSetMrelFinal && (STAGES & STAGE_WARP)) { + setMrelAverageForcedWarp (&catalog[Nc], Nc, j, Nsecfilt, flatcorr, results); + } + } + return (TRUE); +} + +// NOTE: +// Msys is measure[i].M + zp corrections +// Mcal is image[j].Mcal +// Mmos and Mgrid are offsets for mosaic and grid + +// tie down reference photometry if the -refcode (code) option is selected +// eg, -refcode g_SDSS +// this probably makes no sense in the context of multifilter analysis +// XXX probably need to use the photcode table to assign reference mag weights. + +// dlist gives the error per measurement, wlist gives the weight +// we can modify the error and weight in a few ways: +// 1) MIN_ERROR guarantees a floor +// 2) photomErrSys is added in quadrature as a sytematic error, set per photcode +// 3) UBERCAL measurements can have their weight increased by a big factor to help tie down the averages +// 4) some reference photcode of some kind can be specified as fixed and have a high weight + +// Although I calculate McalAPER for exposures, I am only using McalPSF for chips. Note +// in StarOps.c:setMcalOutput I am setting measure->McalAPER to image->McalPSF for chips +// and warps, but not stacks + +// set mean of chip measurements (selected by photcode range for now): +int setMrelAverageExposure (Catalog *catalog, int cat, off_t ave, int Nsecfilt, int isSetMrelFinal, FlatCorrectionTable *flatcorr, SetMrelInfo *results) { + + int Nsec; + off_t k; + + // we are guaranteed to have averageT, but not average + AverageTiny *averageT = &catalog[0].averageT[ave]; + + off_t measureOffset = averageT->measureOffset; + MeasureTiny *measureT = &catalog[0].measureT[measureOffset]; + + // we are NOT guaranteed to have average, measure, secfilt + Average *average = catalog[0].average ? &catalog[0].average[ave] : NULL; + Measure *measure = catalog[0].measure ? &catalog[0].measure[measureOffset] : NULL; + SecFilt *secfilt = catalog[0].secfilt ? &catalog[0].secfilt[ave*Nsecfilt] : NULL; + char *measureRank = catalog[0].measureRank ? &catalog[0].measureRank[measureOffset] : NULL; + + // we are measuring means for 3 types of mags: psf, ap, kron. + + // in the final assignment, set the mean mags even if only 1 measurements exists + int Nminmeas = isSetMrelFinal ? 1 : STAR_TOOFEW + 1; + + // isSetMrelFinal : in the final pass, we set psf, kron, ap mag values and extra stats; other passes only do psf mags + + // option for a test print + if ((averageT[0].objID == TEST_OBJ_ID) && (averageT[0].catID == TEST_CAT_ID)) { + fprintf (stderr, "test obj\n"); + print_measure_set_alt (average, secfilt, measure); + } + + SetMrelInfoResetObject (results); // reset the per-object arrays + + int NextPS1 = 0; + int NpsfPS1 = 0; + int GoodPS1 = FALSE; + int Good2MASS = FALSE; + int Galaxy2MASS = FALSE; + int haveTYCHO = FALSE; + int haveHSC = FALSE; + int haveCFH = FALSE; + + float stargalmax = 0.0; + + // assign measurements to the photcode lists + for (k = 0; k < averageT[0].Nmeasure; k++) { + + // these bits should not be set unless we use them in this pass + // (note that we can only un-set them in the final pass when we actually have measure, not just measureT + if (measure) measure[k].dbFlags &= ~(ID_MEAS_PHOTOM_PSF | ID_MEAS_PHOTOM_APER | ID_MEAS_PHOTOM_KRON); + + // skip measurements that do not have a valid photcode (raise exception?) + PhotCode *code = GetPhotcodebyCode (measureT[k].photcode); + myAssert (code, "invalid photcode??"); + + // SKIP gpc1 stack data (hard-wired photcodes) + if (isGPC1stack(measureT[k].photcode)) continue; + + // SKIP gpc1 forced-warp data (hard-wired photcodes) + if (isGPC1warp(measureT[k].photcode)) continue; + + if (isTYCHO(measureT[k].photcode)) { haveTYCHO = TRUE; } + if (isHSCchip(measureT[k].photcode)) { haveHSC = TRUE; } + if (isCFHchip(measureT[k].photcode)) { haveCFH = TRUE; } + + if (is2MASS(measureT[k].photcode)) { + if (measureT[k].photFlags & 0x00c00000) { + Galaxy2MASS = TRUE; // per object value + } + if (measureT[k].photFlags & 0x00000007) { + Good2MASS = TRUE; // per object value + } + } + + // assign the Nsec value so we can assign to the right lists + int Nsec = GetPhotcodeNsec (code->equiv); + if (Nsec < 0) continue; // skip measurements which do not have an equiv average photcode + + results->Nmeas[Nsec] ++; + + // various steps only relevant to the final pass + if (isSetMrelFinal) { + MAX_NOT_NAN (results->psfQfMax[Nsec], measure[k].psfQF); + MAX_NOT_NAN (results->psfQfPerfMax[Nsec], measure[k].psfQFperf); + + // are we a PS1 exposure photcode? (hard-wired photcodes) + if (isGPC1chip(measure[k].photcode)) { + results->NexpPS1[Nsec] ++; + results->havePS1[Nsec] = TRUE; + + // only count psfQF > 0.85 here + if (isfinite(measure[k].extNsigma) && isfinite(measure[k].psfQF) && (measure[k].psfQF > 0.85)) { + stargalmax = MAX (stargalmax, measure[k].extNsigma); + results->stargal_list[results->Nstargal] = measure[k].extNsigma; + results->Nstargal ++; + } + + // for all GPC1 measurements, track the photflags + results->photflag_list[results->Nphotflags] = measure[k].photFlags; + results->Nphotflags ++; + } + + // force the use of SYN even if we have PS1 mags? + if (isGPC1synth(measure[k].photcode)) { + results->haveSYN[Nsec] = TRUE; + results->measSYN[Nsec] = k; + measureT[k].dbFlags |= ID_MEAS_SYNTH_MAG; // redundant with photcode + // note that synthetic mags have the real mags in measure.M (no zero point offset) + if (measure[k].M < results->minSYN[Nsec]) results->needSYN[Nsec] = TRUE; + continue; + } + } + + // ** Choose the calibration (depends on the mode : do I have an image reference or not?) + float Mcal = 0.0, Mmos = 0.0, Mgrid = 0.0, Mflat = 0.0, Mgrp = 0.0; + float dMgrp = 0.0; + off_t meas = measureOffset + k; + if (getImageEntry (meas, cat) < 0) { + // measurements without an image are either external reference photometry or + // data for which the associated image has not been loaded (probably because of + // overlaps). Msys + measure.Mcal is our best guess of the true magnitude + Mmos = Mgrid = 0; + Mcal = measureT[k].McalPSF; // check that this is zero for loaded REF value + } else { + // getMcal returns image[].Mcal; note the flat-field correction is stored in measure.Mflat + Mcal = getMcal (meas, cat, MAG_CLASS_PSF); + if (isnan(Mcal)) SKIP_THIS_MEAS(Ncal); + Mmos = getMmos (meas, cat); + if (isnan(Mmos)) SKIP_THIS_MEAS(Nmos); + Mgrp = getMgrp (meas, cat, measureT[k].airmass, &dMgrp); + if (isnan(Mgrp)) SKIP_THIS_MEAS(Ngrp); + Mgrid = getMgrid (meas, cat); + if (isnan(Mgrid)) SKIP_THIS_MEAS(Ngrid); + Mflat = isnan (measureT[k].Mflat) ? 0.0 : measureT[k].Mflat; + } + + int myUbercalDist = getUbercalDist(meas, cat); + results->minUbercalDist[Nsec] = MIN(results->minUbercalDist[Nsec], myUbercalDist); + + int isUbercal = (measureT[k].dbFlags & ID_MEAS_PHOTOM_UBERCAL); + + if (isUbercal) results->haveUbercal[Nsec] = TRUE; // haveUbercal is set per secfilt, isUbercal is per measure XXX define this array + + // logic for choosing a modified weight: + float modifiedWeight = 1.0; + if (isUbercal) modifiedWeight = UBERCAL_WEIGHT; + if (refPhotcode && (code->code == refPhotcode->code)) modifiedWeight = UBERCAL_WEIGHT; + if (USE_REFERENCE_WEIGHT && (code->type == PHOT_REF)) modifiedWeight = code->photomErrSys; // we are overloading this field for now + + // XXX I should probably do something more clever with the measured scatter in the nights, mosaics, images + + // combine the various errors in quadrature here: + float dMsys = hypot (code->photomErrSys, dMgrp); + + float Map = NAN; + float dMap = NAN; + + // XXX I am not convinced the sign on Mflat here is correct + float Moff = Mflat + Mcal + Mmos + Mgrp + Mgrid; + + if (isSetMrelFinal) { + Map = PhotCat (&measure[k], MAG_CLASS_APER); + + // NOTE: for PV3, apFluxErr is broken (see pmSourcePhotometry.c:245). we are going to + // use PSF flux error instead here: + // dMap = MAX (hypot(measure[k].dMap, dMsys), MIN_ERROR); + dMap = MAX (hypot(measure[k].dM, dMsys), MIN_ERROR); + + if (CHECK_VALID_MAG(Map, dMap)) { + int Nap = results->aperData[Nsec].Nlist; + results->aperData[Nsec].flxlist[Nap] = Map - Moff; // this is consistent with PhotRel + results->aperData[Nsec].errlist[Nap] = dMap; + results->aperData[Nsec].wgtlist[Nap] = modifiedWeight; + results->aperData[Nsec].ranking[Nap] = measureRank[k]; + results->aperData[Nsec].measSeq[Nap] = k; + results->aperData[Nsec].msklist[Nap] = 0; + results->aperData[Nsec].Nlist ++; + } + } + + // NOTE: we need to calculate the averge chip Kron on each pass to be able to calibrate the stacks + float Mkron = PhotCatTiny (&measureT[k], MAG_CLASS_KRON); + float dMkron; + if (isSetMrelFinal) { + dMkron = MAX (hypot(measure[k].dMkron, dMsys), MIN_ERROR); + } else { + dMkron = MAX (hypot(measureT[k].dM, dMsys), MIN_ERROR); + } + if (CHECK_VALID_MAG(Mkron, dMkron)) { + int Nkron = results->kronData[Nsec].Nlist; + results->kronData[Nsec].flxlist[Nkron] = Mkron - Moff; // this is consistent with PhotRel + results->kronData[Nsec].errlist[Nkron] = dMkron; + results->kronData[Nsec].wgtlist[Nkron] = modifiedWeight; + results->kronData[Nsec].ranking[Nkron] = measureRank ? measureRank[k] : 0; + results->kronData[Nsec].measSeq[Nkron] = k; + results->kronData[Nsec].msklist[Nkron] = 0; + results->kronData[Nsec].Nlist ++; + } + + float Mpsf = PhotSysTiny (&measureT[k], &averageT[0], &secfilt[0], MAG_CLASS_PSF); + float dMpsf = MAX (hypot(measureT[k].dM, dMsys), MIN_ERROR); + if (CHECK_VALID_MAG(Mpsf, dMpsf)) { + int Npsf = results->psfData[Nsec].Nlist; + results->psfData[Nsec].flxlist[Npsf] = Mpsf - Moff; // this is consistent with PhotRel + results->psfData[Nsec].errlist[Npsf] = dMpsf; + results->psfData[Nsec].wgtlist[Npsf] = modifiedWeight; + results->psfData[Nsec].ranking[Npsf] = measureRank ? measureRank[k] : 0; + results->psfData[Nsec].measSeq[Npsf] = k; + results->psfData[Nsec].msklist[Npsf] = 0; + results->psfData[Nsec].Nlist ++; + } + + // count the extended detections (all PS1 bands) + if (isSetMrelFinal && isGPC1chip(measureT[k].photcode) && !isnan(Map) && !isnan(Mpsf)) { + float dMagAp = Mpsf - Map; + float SigmaAp = hypot(0.1, 2.5*dMpsf); + // XXX this is still quite ad hoc, but at least it: + // (a) converges to 0.1 mag offset at the bright end + // (b) converges to 0.5 mag offset at the faint end (dM = 0.2) + if (dMagAp > SigmaAp) { + results->Next[Nsec] ++; + NextPS1 ++; + } else { + NpsfPS1 ++; + } + } + } + + SynthZeroPoints *synthzpts = SynthZeroPointsGet(); + + // find max values across filters + float psfQfMax = 0.0; + float psfQfPerfMax = 0.0; + + // now calculate the mean stats for the Nsec bands. + for (Nsec = 0; Nsec < Nsecfilt; Nsec++) { + + // if we detected this object in PS1, or do not request -preserve-ps1, keep the mean photometry values + if (!PRESERVE_PS1 || !(secfilt[Nsec].flags & ID_SECF_HAS_PS1)) { + dvo_secfilt_init (&secfilt[Nsec], SECFILT_RESET_CHIP); // this does not reset astrometry or STACK bits + } + + if (haveTYCHO) { + secfilt[Nsec].flags |= ID_SECF_HAS_TYCHO; + } + if (haveHSC) { + secfilt[Nsec].flags |= ID_SECF_HAS_HSC; + } + if (haveCFH) { + secfilt[Nsec].flags |= ID_SECF_HAS_CFH; + } + + if (PRESERVE_PS1 && (secfilt[Nsec].flags & ID_SECF_HAS_PS1)) continue; + // if -preserve-ps1 is set and this object has PS1 data, skip the rest of the steps: + + // XXX hardwired grizy = (01234) JHK = (567) w = (8) + if ((Nsec < 5) || (Nsec == 8)) { + secfilt[Nsec].Ncode = results->NexpPS1[Nsec]; + } else { + secfilt[Nsec].Ncode = results->Nmeas[Nsec]; // 2MASS data if it exists + } + + // use the synthetic magnitude for mean photometry (apply zpt calibration if no tycho + // photometry exists) + if (results->needSYN[Nsec] || (!results->havePS1[Nsec] && results->haveSYN[Nsec])) { + if (!isSetMrelFinal) continue; + + // use the single SYNTH value instead of the other mags here + float Mpsf = measure[results->measSYN[Nsec]].M; + float ZP = 0.0; + + // if we have loaded the synth zpt correction, and do NOT have a tycho measurement, apply the correction + if (!haveTYCHO && synthzpts) { + // need to look up the (X,Y) coords from this (R,D) location + double X, Y; + double R = ohana_normalize_angle_to_midpoint (measure->R, 0.0); // XXX 0.0 or 180.0 for center? + RD_to_XY (&X, &Y, R, measure->D, &synthzpts->coords); + if (X < 0) return FALSE; + if (Y < 0) return FALSE; + if (X >= synthzpts->Nx) return FALSE; + if (Y >= synthzpts->Ny) return FALSE; + + int Xpix = X; + int Ypix = Y; + int Npix = Xpix + Ypix*synthzpts->Nx; + + float *value = (float *) synthzpts->matrix[Nsec].buffer; + ZP = !isnan(value[Npix]) ? value[Npix] : 0.0; + if (isfinite(value[Npix])) { + secfilt[Nsec].flags |= ID_SECF_FIX_SYNTH; + } + } + secfilt[Nsec].MpsfChp = Mpsf + ZP; + secfilt[Nsec].dMpsfChp = 0.6; + secfilt[Nsec].Mchisq = 0.0; + secfilt[Nsec].flags |= ID_SECF_USE_SYNTH; + continue; + } + + // if too few valid measurements meet the minimum criteria, go to the next entry + StatType *psfstats = &results->psfstats; + + int NrankingPSF = useOLS ? magStatsByRanking (&results->psfData[Nsec], psfstats) : magStatsByRankingIRLS (&results->psfData[Nsec], psfstats); + if (NrankingPSF < Nminmeas) { + secfilt[Nsec].flags |= ID_OBJ_FEW; + } else { + secfilt[Nsec].MpsfChp = psfstats->mean; + secfilt[Nsec].dMpsfChp = psfstats->error; + secfilt[Nsec].Mchisq = (psfstats->Nmeas > 1) ? psfstats->chisq : NAN; + } + + // when running -averages, we have no information about the images, so we cannot set this + if (results->minUbercalDist[Nsec] > -1) { + secfilt[Nsec].ubercalDist = results->minUbercalDist[Nsec]; + } + + StatType *kronstats = &results->kronstats; + int NrankingKRON = useOLS ? magStatsByRanking (&results->kronData[Nsec], kronstats) : magStatsByRankingIRLS (&results->kronData[Nsec], kronstats); + if (NrankingKRON) { + secfilt[Nsec].MkronChp = kronstats->mean; + secfilt[Nsec].dMkronChp = kronstats->error; + secfilt[Nsec].sMkronChp = kronstats->sigma; + secfilt[Nsec].NusedKron = NrankingKRON; + } + + if (isSetMrelFinal) { + if ((average[0].objID == TEST_OBJ_ID) && (average[0].catID == TEST_CAT_ID)) { + fprintf (stderr, "test obj\n"); + } + + if (NrankingPSF) { + secfilt[Nsec].sMpsfChp = psfstats->sigma; + secfilt[Nsec].Nused = psfstats->Nmeas; + secfilt[Nsec].Mmax = psfstats->max; + secfilt[Nsec].Mmin = psfstats->min; + } + secfilt[Nsec].psfQfMax = results->psfQfMax[Nsec]; + secfilt[Nsec].psfQfPerfMax = results->psfQfPerfMax[Nsec]; + + MAX_NOT_NAN (psfQfMax, secfilt[Nsec].psfQfMax); + MAX_NOT_NAN (psfQfPerfMax, secfilt[Nsec].psfQfPerfMax); + + // mark the measurements matching this ranking + int minRankPSF = (NrankingPSF > 0) ? results->psfData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->psfData[Nsec], measure, minRankPSF, ID_MEAS_PHOTOM_PSF, ID_MEAS_MASKED_PSF); + + int minRankKron = (NrankingKRON > 0) ? results->kronData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->kronData[Nsec], measure, minRankKron, ID_MEAS_PHOTOM_KRON, ID_MEAS_MASKED_KRON); + + StatType *apstats = &results->apstats; + int NrankingAPER = useOLS ? magStatsByRanking (&results->aperData[Nsec], apstats) : magStatsByRankingIRLS (&results->aperData[Nsec], apstats); + if (NrankingAPER) { + secfilt[Nsec].MapChp = apstats->mean; + secfilt[Nsec].dMapChp = apstats->error; + secfilt[Nsec].sMapChp = apstats->sigma; + secfilt[Nsec].NusedAp = NrankingAPER; + } + int minRankAper = (NrankingAPER > 0) ? results->aperData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->aperData[Nsec], measure, minRankAper, ID_MEAS_PHOTOM_APER, ID_MEAS_MASKED_APER); + + // does this object appear extended in > 50% of measurements? + if ((results->Next[Nsec] > 0) && (results->Next[Nsec] > 0.5*psfstats->Nmeas)) { + secfilt[Nsec].flags |= ID_SECF_OBJ_EXT; + } + + switch (minRankPSF) { + case 0: + secfilt[Nsec].flags |= ID_SECF_RANK_0; + if (results->havePS1[Nsec]) GoodPS1 = TRUE; + break; + case 1: + secfilt[Nsec].flags |= ID_SECF_RANK_1; + if (results->havePS1[Nsec]) GoodPS1 = TRUE; + break; + case 2: + secfilt[Nsec].flags |= ID_SECF_RANK_2; + if (results->havePS1[Nsec]) GoodPS1 = TRUE; + break; + case 3: + secfilt[Nsec].flags |= ID_SECF_RANK_3; + break; + case 4: + secfilt[Nsec].flags |= ID_SECF_RANK_4; + break; + default: + break; + } + if (results->havePS1[Nsec]) { + secfilt[Nsec].flags |= ID_SECF_HAS_PS1; + } + if (results->haveUbercal[Nsec]) { + secfilt[Nsec].flags |= ID_SECF_USE_UBERCAL; + } + } + } + + if (isSetMrelFinal) { + if (NextPS1 && (NextPS1 > NpsfPS1)) { + average[0].flags |= ID_OBJ_EXT; + } + if (GoodPS1) { + average[0].flags |= ID_OBJ_GOOD; + } + if (Galaxy2MASS) { + average[0].flags |= ID_OBJ_EXT_ALT; + } + if (Good2MASS) { + average[0].flags |= ID_OBJ_GOOD_ALT; + } + + GetPhotFlagStats (&average[0].photFlagsUpper, &average[0].photFlagsLower, results->photflag_list, results->Nphotflags); + + // calculate the median of the stargal (ExtNsigma) values + dsort(results->stargal_list, results->Nstargal); + double stargal_median; + if (results->Nstargal % 2) { + int Ncenter = results->Nstargal / 2; + stargal_median = results->stargal_list[Ncenter]; + } else { + int Ncenter = results->Nstargal / 2 - 1; + stargal_median = 0.5*(results->stargal_list[Ncenter] + results->stargal_list[Ncenter + 1]); + } + average[0].stargal = stargal_median; + + average[0].psfQF = psfQfMax; + average[0].psfQFperf = psfQfPerfMax; + } + return (TRUE); +} + +// only apply Stack operation on setMrelFinal in first pass +// this function has 3 goals, not to be confused: +// 1) find and mark the PRIMARY detections (regardless of the quality of the detection) +// 2) select the BEST detections per filter (regardless of PRIMARY) +// 3) apply the zero point and AB->Jy transformations + +// I calculate McalAPER and McalPSF independently for stacks. I use McalAPER for Mkron +// and Map, and McalPSF for Mpsf. Note in StarOps.c:setMcalOutput I am setting +// measure->McalAPER to image->McalPSF for chips and warps, but not stacks + +int setMrelAverageStack (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr) { + + // we are guaranteed to have average, measure, secfilt + Average *average = &catalog[0].average[ave]; + off_t measureOffset = average->measureOffset; + int Nmeasure = average->Nmeasure; + + Measure *measure = &catalog[0].measure[measureOffset]; + SecFilt *secfilt = &catalog[0].secfilt[ave*Nsecfilt]; + + float McalPSF = 0, McalAPER = 0, Mmos = 0, Mgrid = 0; + + // set the primary projection cell and skycell for this coordinate + int tessID = -1; + int projectID = -1; + int skycellID = -1; + get_tess_ids(&tessID, &projectID, &skycellID, average[0].R, average[0].D); + + average[0].tessID = tessID; + average[0].projectionID = projectID; + average[0].skycellID = skycellID; + + int NstackGood = 0; + int NstackSuspect = 0; + int haveStackObject = FALSE; + + for (int Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + /* calculate the average mag in this SEC photcode for a single star */ + + int haveStack = FALSE; + + int isBad = FALSE; + int isSuspect = FALSE; + + int NstackMeas = 0; // number of stack measurements for this photcode + int NstackDet = 0; // number of stack detections for this photcode (not forced) + + int primaryLevelMax = 0; + off_t primaryEntryMax = -1; + float primaryValueMax = 0.0; + + int bestLevelMax = 0; + off_t bestEntryMax = -1; + float bestValueMax = 0.0; + + int Nprimary = 0; + + // reset all stack-related values for this secfilt: + secfilt[Nsec].stackBestOff = -1; + secfilt[Nsec].stackPrmryOff = -1; + secfilt[Nsec].FpsfStk = NAN; + secfilt[Nsec].dFpsfStk = NAN; + secfilt[Nsec].FkronStk = NAN; + secfilt[Nsec].dFkronStk = NAN; + secfilt[Nsec].FapStk = NAN; + secfilt[Nsec].dFapStk = NAN; + secfilt[Nsec].MpsfStk = NAN; + secfilt[Nsec].MkronStk = NAN; + secfilt[Nsec].MapStk = NAN; + secfilt[Nsec].Nstack = 0; + secfilt[Nsec].NstackDet = 0; + secfilt[Nsec].flags &= ~ID_SECF_STACK_FLAGS; // reset the stack flags + + for (off_t k = 0; k < Nmeasure; k++) { + + // only examine gpc1 stack data + if (!isGPC1stack(measure[k].photcode)) continue; + + // skip measurements that do not match the current photcode + PhotCode *code = GetPhotcodebyCode (measure[k].photcode); + if (!code) continue; + if (code->equiv != thisCode) { continue; } + + NstackMeas ++; + if ((measure[k].photFlags2 & 0x00000004) == 0) NstackDet ++; // 0x04 = PM_SOURCE_MODE2_MATCHED + + // clear this bit for all measurements + measure[k].dbFlags &= ~ID_MEAS_STACK_PRIMARY; + measure[k].dbFlags &= ~ID_MEAS_STACK_PHOT_SRC; + + haveStack = TRUE; + haveStackObject = TRUE; + + // if (measure[k].dbFlags & MEAS_BAD) SKIP_THIS_MEAS_STACK(Nbad); + + int isPrimary = FALSE; + + // ** is this a PRIMARY measurement? (there may be more than one) + + // if we request the primary (USE_TREE_FOR_PRIMARY), this is true if the measurement + // is from the primary skycell for this position. (note that MatchImageSkycellID + // requires the entry in the full Measure table, not just this object) + if (MatchImageSkycellID (measureOffset + k, cat, tessID, projectID, skycellID)) { + measure[k].dbFlags |= ID_MEAS_STACK_PRIMARY; + isPrimary = TRUE; + Nprimary ++; + } + + // soften the error floor (can dM be 0.0?) + // XXX EAM : this was int SNvalue -- isfinite(SNvalue) would always be finite + float SNvalue = isfinite(measure[k].dM) ? 1.0 / hypot (measure[k].dM, MIN_ERROR) : NAN; + int psfQFperfAboveLimit = isfinite(SNvalue) && isfinite(measure[k].psfQFperf) && (measure[k].psfQFperf > 0.95); + + // ** determine the BEST level + int bestLevel = 0; + if (!isPrimary && !psfQFperfAboveLimit) bestLevel = 1; + if ( isPrimary && !psfQFperfAboveLimit) bestLevel = 2; + if (!isPrimary && psfQFperfAboveLimit) bestLevel = 3; + if ( isPrimary && psfQFperfAboveLimit) bestLevel = 4; + + // here is the rule for choosing the best value: + float bestValue = (bestLevel > 2) ? SNvalue : measure[k].psfQFperf; + + // if we have not reached this level before, set the new level + if (bestLevelMax < bestLevel) { + bestValueMax = bestValue; + bestLevelMax = bestLevel; + bestEntryMax = k; + } + // if we have reached this level before, set the new value if we beat the old one + if ((bestLevelMax == bestLevel) && (bestValueMax < bestValue)) { + bestValueMax = bestValue; + bestEntryMax = k; + } + myAssert (bestEntryMax > -1, "this should not happen"); + + // ** determine the PRIMARY level + int primaryLevel = 0; + if ( isPrimary && !psfQFperfAboveLimit) primaryLevel = 1; + if ( isPrimary && psfQFperfAboveLimit) primaryLevel = 2; + + // here is the rule for choosing the PRIMARY value: + float primaryValue = 0.0; + if (primaryLevel == 1) { primaryValue = measure[k].psfQFperf; } + if (primaryLevel == 2) { primaryValue = SNvalue; } + + // if we have not reached this level before, set the new level + if (isPrimary && (primaryLevelMax < primaryLevel)) { + primaryValueMax = primaryValue; + primaryLevelMax = primaryLevel; + primaryEntryMax = k; + } + // if we have reached this level before, set the new value if we beat the old one + if (isPrimary && (primaryLevelMax == primaryLevel) && (primaryValueMax < primaryValue)) { + primaryValueMax = primaryValue; + primaryEntryMax = k; + } + } + + // if we do not have a stack measurement for this photcode, skip everything below + if (!haveStack) continue; + + // now we have a BEST and a PRIMARY entry (may be the same entry) + + off_t meas = bestEntryMax; + off_t measSeq = meas + measureOffset; + + myAssert (meas >= 0, "this should not happen"); + + // measurements without an image are either external reference photometry or + // data for which the associated image has not been loaded (probably because of + // overlaps). we only want measurements associated with stack images in this loop + + // match measurement to its image (this is just a check, right?) + if (getImageEntry (measSeq, cat) < 0) { + // measurements without an image are either external reference photometry or + // data for which the associated image has not been loaded (probably because of + // overlaps). Msys + measure.Mcal is our best guess of the true magnitude + McalPSF = measure[meas].McalPSF; // check that this is zero for loaded REF value + McalAPER = McalPSF; // check that this is zero for loaded REF value + Mmos = 0.0; + Mgrid = 0.0; + } else { + McalPSF = getMcal (measSeq, cat, MAG_CLASS_PSF); + McalAPER = USE_MCAL_PSF_FOR_STACK_APER ? getMcal (measSeq, cat, MAG_CLASS_PSF) : getMcal (measSeq, cat, MAG_CLASS_KRON); + Mmos = getMmos (measSeq, cat); + Mgrid = getMgrid (measSeq, cat); + // XXX can Mmos and Mgrid exist for stacks? + } + + // ** Here is the math to relate mag,zp to flux + + // flux_cgs : erg sec^1 cm^-2 Hz^-1 + // mag_inst : -2.5 log (cts/sec) + // mag_inst : -2.5 log (flux_inst) + // flux_inst = ten(-0.4*mag_inst) + + // mag_AB = -2.5 log (flux_cgs) - 48.6 (~by definition) [~Vega flux in V-band] + // flux_cgs = ten(-0.4*(mag_AB + 48.6)) + + // flux_AB : ten(-0.4*mag_AB) + + // flux_cgs = ten(-0.4*48.6) * flux_AB + // flux_AB = ten(+0.4*48.6) * flux_cgs + + // flux_Jy : flux_cgs * 10^23 + + // flux_AB = ten(+0.4*48.6) * ten(-23) * flux_Jy + + // mag_AB = mag_inst + ZP + + // flux_inst = ten(-0.4*(mag_AB - ZP)) = ten(0.4*ZP) * flux_AB + + // flux_AB = flux_inst * ten(-0.4*ZP) + + // flux_inst * ten(-0.4*ZP) = ten(+0.4*48.6 - 23) * flux_Jy + + // flux_inst = flux_Jy * ten(0.4*ZP + 0.4*48.6 - 23) + // flux_inst = flux_Jy * ten(0.4*ZP - 3.56) + // flux_Jy = flux_inst * ten(-0.4*ZP + 3.56) + + // get the zero point for the selected image + // Use a different zero point for the PSF-like and APERTURE-like magnitudes + float zpPSF = PhotZeroPoint (&measure[meas], &average[0], &secfilt[0]) - (McalPSF + Mmos + Mgrid); + float zpAPER = PhotZeroPoint (&measure[meas], &average[0], &secfilt[0]) - (McalAPER + Mmos + Mgrid); + + // zpFactor to go from instrumental flux to Janskies + float zpFactorPSF = pow(10.0, -0.4*zpPSF + 3.56); + float zpFactorAPER = pow(10.0, -0.4*zpAPER + 3.56); + + // need to put in AB mag factor to get to Janskies (or uJy?) + secfilt[Nsec].FpsfStk = zpFactorPSF * measure[meas].FluxPSF; + secfilt[Nsec].dFpsfStk = zpFactorPSF * measure[meas].dFluxPSF; + secfilt[Nsec].FkronStk = zpFactorAPER * measure[meas].FluxKron; + secfilt[Nsec].dFkronStk = zpFactorAPER * measure[meas].dFluxKron; + secfilt[Nsec].FapStk = zpFactorAPER * measure[meas].FluxAp; + + // NOTE: for PV3, apFluxErr is broken (see pmSourcePhotometry.c:245). we are going to + // use PSF flux error instead here: + // secfilt[Nsec].dFapStk = zpFactorAPER * measure[meas].dFluxAp; + secfilt[Nsec].dFapStk = zpFactorAPER * measure[meas].dFluxPSF; + + // Jy to AB mags + secfilt[Nsec].MpsfStk = (measure[meas].FluxPSF > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FpsfStk) : NAN; + secfilt[Nsec].MkronStk = (measure[meas].FluxKron > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FkronStk) : NAN; + secfilt[Nsec].MapStk = (measure[meas].FluxAp > 0.0) ? 8.9 - 2.5*log10(secfilt[Nsec].FapStk) : NAN; + + // record the measurement which gave the best value + secfilt[Nsec].stackBestOff = (bestEntryMax == -1) ? -1 : bestEntryMax + measureOffset; + myAssert (secfilt[Nsec].stackBestOff <= catalog[0].Nmeasure, "stackBestOff out of range"); + + // record the selected primary measurement + secfilt[Nsec].stackPrmryOff = (primaryEntryMax == -1) ? -1 : primaryEntryMax + measureOffset; + myAssert (secfilt[Nsec].stackPrmryOff <= catalog[0].Nmeasure, "stackPrmryOff out of range"); + + // this is the selected measurement used by secfilt[] + measure[meas].dbFlags |= ID_MEAS_STACK_PHOT_SRC; + if (Nprimary) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMARY; + if (Nprimary > 1) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMARY_MULTIPLE; + } + } + + // stack measurement used for 'best' was a detection (not forced from the other bands) + if ((measure[bestEntryMax].photFlags2 & 0x00000004) == 0) { + secfilt[Nsec].flags |= ID_SECF_STACK_BESTDET; + } + + // stack measurement used for 'primary' was a detection (not forced from the other bands) + if ((primaryEntryMax >= 0) && ((measure[primaryEntryMax].photFlags2 & 0x00000004) == 0)) { + secfilt[Nsec].flags |= ID_SECF_STACK_PRIMDET; + } + + secfilt[Nsec].flags |= ID_SECF_HAS_PS1_STACK; + + // NOTE: negative and insignificant flux values are allowed, but not NAN flux values + // float Finst = PhotFluxInst (&measure[meas], MAG_CLASS_PSF); + + // data quality assessment + isBad |= (measure[meas].photFlags & photcodes[Ns][0].photomBadMask); + isBad |= (measure[meas].psfQF < 0.85); + isBad |= isnan(measure[meas].psfQF); + isBad |= measure[meas].dM > 0.2; // S/N < 5.0 + + isSuspect |= (measure[meas].photFlags & photcodes[Ns][0].photomPoorMask); + isSuspect |= (measure[meas].psfQFperf < 0.85); + + if (!isSuspect && !isBad) { + NstackGood ++; + } + if (isSuspect && !isBad) { + NstackSuspect ++; + } + + secfilt[Nsec].Nstack = NstackMeas; + secfilt[Nsec].NstackDet = NstackDet; + + } // Nsecfilt loop + + // in PSPS, there will be a stackObject row with the primary values. if all filter primary values which + // exist are also best values, then this row will be the best entry + + // the secfilt[Ns] entries are defined to be the Best values, so + + // conversely, if any secfilt[Ns] values are not PRIMARY, then the PRIMARY row is not the BEST row + + int PrimaryIsBest = TRUE; + for (int Ns = 0; Ns < Nphotcodes; Ns++) { + if (!(secfilt[Ns].flags & ID_SECF_HAS_PS1_STACK)) continue; // no stack detection in PS1, nothing is best + if (secfilt[Ns].flags & ID_SECF_STACK_PRIMARY) continue; // primary stack detection is best + PrimaryIsBest = FALSE; + } + + if (PrimaryIsBest) { + average[0].flags |= ID_OBJ_BEST_STACK; + } + + if (NstackGood >= 2) { + average[0].flags |= ID_OBJ_GOOD_STACK; + } else if (NstackGood + NstackSuspect >= 2) { + average[0].flags |= ID_OBJ_SUSPECT_STACK; + } else if (haveStackObject) { + average[0].flags |= ID_OBJ_BAD_STACK; + } + + return (TRUE); +} + +// set mean of forced-warp measurements (selected by photcode range for now): +// somewhat simplified relative to chip-photometry: +// * no grid, no mosaic, no 2MASS, no SYNTH, no Ubercal, no flatcorr +// analysis is done on flux, not mags (as the faintest objects will be nearly insignificant) + +// Although I calculate McalAPER for exposures, I am only using McalPSF for warps.. +// Note in StarOps.c:setMcalOutput I am setting measure->McalAPER to image->McalPSF for +// chips and warps, but not stacks + +int setMrelAverageForcedWarp (Catalog *catalog, int cat, off_t ave, int Nsecfilt, FlatCorrectionTable *flatcorr, SetMrelInfo *results) { + OHANA_UNUSED_PARAM(flatcorr); + + // we are guaranteed to have average, measure, secfilt + Average *average = &catalog[0].average[ave]; + off_t measureOffset = average->measureOffset; + + Measure *measure = &catalog[0].measure[measureOffset]; + SecFilt *secfilt = &catalog[0].secfilt[ave*Nsecfilt]; + char *measureRank = &catalog[0].measureRank[measureOffset]; + + off_t k, Nsec; + float Mcal = 0; + + SetMrelInfoResetObject (results); // reset the per-object arrays + + // option for a test print + if ((average[0].objID == TEST_OBJ_ID) && (average[0].catID == TEST_CAT_ID)) { + fprintf (stderr, "test obj\n"); + print_measure_set_alt (average, secfilt, measure); + } + + // for a diff db, we just want primary, if available or not. for warp we want to use the + // skycell which provided the stack best entry + int tessID, projID, skycellID; + if (IS_DIFF_DB) { + get_tess_ids(&tessID, &projID, &skycellID, average[0].R, average[0].D); + } + for (Nsec = 0; Nsec < Nsecfilt; Nsec++) { + if (!IS_DIFF_DB) { + myAssert (secfilt[Nsec].stackBestOff <= catalog->Nmeasure, "stackBestOff out of range"); + if (!FindImageSkycellID (secfilt[Nsec].stackBestOff, cat, &tessID, &projID, &skycellID)) continue; + } + results->tessID[Nsec] = tessID; + results->projID[Nsec] = projID; + results->skycellID[Nsec] = skycellID; + } + + average->NwarpOK = 0; // count the number of warps with psfQF > 0.0 + + // assign measurements to the photcode lists + for (k = 0; k < average->Nmeasure; k++) { + off_t meas = measureOffset + k; + + measure[k].dbFlags &= ~ID_MEAS_WARP_USED; // this should not be set unless we use it in this pass + + // skip measurements that do not have a valid photcode (raise exception?) + PhotCode *code = GetPhotcodebyCode (measure[k].photcode); + myAssert (code, "invalid photcode??"); + + // only examine gpc1 forced-warp data + if (!isGPC1warp(measure[k].photcode)) continue; + + // assign the Nsec value so we can assign to the right lists + int Nsec = GetPhotcodeNsec (code->equiv); + if (Nsec < 0) continue; // skip measurements which do not have an equiv average photcode + + results->Nmeas[Nsec] ++; + if (isfinite(measure[k].psfQFperf) && (measure[k].psfQFperf > 0.85)) results->NmeasGood[Nsec] ++; + if (isfinite(measure[k].psfQF) && (measure[k].psfQF > 0.00)) average->NwarpOK ++; + + // use primary skycell for DIFF, stack skycell for WARP + if (!MatchImageSkycellID (meas, cat, results->tessID[Nsec], results->projID[Nsec], results->skycellID[Nsec])) { + continue; + } + + // assign the max psfQF values for each Nsec value + if (IS_DIFF_DB) { + measure[k].dbFlags |= ID_MEAS_STACK_PRIMARY; + MAX_NOT_NAN (results->psfQfMax[Nsec], measure[k].psfQF); + MAX_NOT_NAN (results->psfQfPerfMax[Nsec], measure[k].psfQFperf); + } + + if (getImageEntry (meas, cat) < 0) { + // measurements without an image are either external reference photometry or + // data for which the associated image has not been loaded (probably because of + // overlaps). Msys + measure.Mcal is our best guess of the true magnitude + Mcal = measure[k].McalPSF; // check that this is zero for loaded REF value + } else { + Mcal = getMcal (meas, cat, MAG_CLASS_PSF); + if (isnan(Mcal)) SKIP_THIS_MEAS(Ncal); + } + float Fcal = MagToFlux(-Mcal); + + // NOTE : I am using PhotCat not PhotSys for now since GPC1 chip-to-chip color terms + // are small (and not measured) + float Fpsf = PhotFluxCat (&measure[k], MAG_CLASS_PSF); + float dFpsf = PhotFluxCatErr (&measure[k], MAG_CLASS_PSF); + if (CHECK_VALID_FLUX(Fpsf, dFpsf)) { + dFpsf = MAX (hypot(dFpsf, code->photomErrSys*Fpsf), MIN_ERROR*Fpsf); // bump up the error by a systematic floor + int Npsf = results->psfData[Nsec].Nlist; + results->psfData[Nsec].flxlist[Npsf] = Fpsf * Fcal; + results->psfData[Nsec].errlist[Npsf] = dFpsf * Fcal; + results->psfData[Nsec].wgtlist[Npsf] = 1.0; + results->psfData[Nsec].ranking[Npsf] = measureRank ? measureRank[k] : 0; + results->psfData[Nsec].measSeq[Npsf] = k; + results->psfData[Nsec].msklist[Npsf] = 0; + results->psfData[Nsec].Nlist ++; + } + + float Fap = PhotFluxCat (&measure[k], MAG_CLASS_APER); + + // NOTE: for PV3, apFluxErr is broken (see pmSourcePhotometry.c:245). we are going to + // use PSF flux error instead here: + // float dFap = PhotFluxCatErr (&measure[k], MAG_CLASS_APER); + float dFap = PhotFluxCatErr (&measure[k], MAG_CLASS_PSF); + + if (CHECK_VALID_FLUX(Fap, dFap)) { + dFap = MAX (hypot(dFap, code->photomErrSys*Fap), MIN_ERROR*Fap); // bump up the error by a systematic floor + int Naper = results->aperData[Nsec].Nlist; + results->aperData[Nsec].flxlist[Naper] = Fap * Fcal; + results->aperData[Nsec].errlist[Naper] = dFap * Fcal; + results->aperData[Nsec].wgtlist[Naper] = 1.0; + results->aperData[Nsec].ranking[Naper] = measureRank ? measureRank[k] : 0; + results->aperData[Nsec].measSeq[Naper] = k; + results->aperData[Nsec].msklist[Naper] = 0; + results->aperData[Nsec].Nlist ++; + } + + float Fkron = PhotFluxCat (&measure[k], MAG_CLASS_KRON); + float dFkron = PhotFluxCatErr (&measure[k], MAG_CLASS_KRON); + if (CHECK_VALID_FLUX(Fkron, dFkron)) { + dFkron = MAX (hypot(dFkron, code->photomErrSys*Fkron), MIN_ERROR*Fkron); // bump up the error by a systematic floor + int Nkron = results->kronData[Nsec].Nlist; + results->kronData[Nsec].flxlist[Nkron] = Fkron * Fcal; + results->kronData[Nsec].errlist[Nkron] = dFkron * Fcal; + results->kronData[Nsec].wgtlist[Nkron] = 1.0; + results->kronData[Nsec].ranking[Nkron] = measureRank ? measureRank[k] : 0; + results->kronData[Nsec].measSeq[Nkron] = k; + results->kronData[Nsec].msklist[Nkron] = 0; + results->kronData[Nsec].Nlist ++; + } + } + + // find max values across filters + float psfQfMax = 0.0; + float psfQfPerfMax = 0.0; + + // now calculate the mean stats for the Nsec bands. + for (Nsec = 0; Nsec < Nsecfilt; Nsec++) { + dvo_secfilt_init (&secfilt[Nsec], SECFILT_RESET_WARP); + + if (IS_DIFF_DB) { + // for non DIFF_DB, these are set in setMrelAverageExposure + secfilt[Nsec].psfQfMax = results->psfQfMax[Nsec]; + secfilt[Nsec].psfQfPerfMax = results->psfQfPerfMax[Nsec]; + + MAX_NOT_NAN (psfQfMax, secfilt[Nsec].psfQfMax); + MAX_NOT_NAN (psfQfPerfMax, secfilt[Nsec].psfQfPerfMax); + } + + int Nranking, minRank; + + // if too few valid measurements meet the minimum criteria, go to the next entry + StatType *psfstats = &results->psfstats; + // Nranking = MAG_STATS_BY_RANKING (&results->psfData[Nsec], psfstats); + Nranking = useOLS ? magStatsByRanking (&results->psfData[Nsec], psfstats) : magStatsByRankingIRLS (&results->psfData[Nsec], psfstats); + if (Nranking) { + secfilt[Nsec].FpsfWrp = psfstats->mean; + secfilt[Nsec].dFpsfWrp = psfstats->error; + secfilt[Nsec].sFpsfWrp = (psfstats->Nmeas > 1) ? psfstats->chisq : NAN; + secfilt[Nsec].NusedWrp = psfstats->Nmeas; + secfilt[Nsec].MpsfWrp = isnan(secfilt[Nsec].FpsfWrp) ? NAN : 8.9 - 2.5*log10(secfilt[Nsec].FpsfWrp); // 8.9 since flux is in Jy + } + minRank = (Nranking > 0) ? results->psfData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->psfData[Nsec], measure, minRank, ID_MEAS_WARP_USED | ID_MEAS_PHOTOM_PSF, ID_MEAS_MASKED_PSF); + + // if too few valid measurements meet the minimum criteria, go to the next entry + StatType *apstats = &results->apstats; + // Nranking = MAG_STATS_BY_RANKING (&results->aperData[Nsec], apstats); + Nranking = useOLS ? magStatsByRanking (&results->aperData[Nsec], apstats) : magStatsByRankingIRLS (&results->aperData[Nsec], apstats); + if (Nranking) { + secfilt[Nsec].FapWrp = apstats->mean; + secfilt[Nsec].dFapWrp = apstats->error; + secfilt[Nsec].sFapWrp = (apstats->Nmeas > 1) ? apstats->chisq : NAN; + secfilt[Nsec].NusedApWrp = apstats->Nmeas; + secfilt[Nsec].MapWrp = isnan(secfilt[Nsec].FapWrp) ? NAN : 8.9 - 2.5*log10(secfilt[Nsec].FapWrp); // 8.9 since flux is in Jy + } + minRank = (Nranking > 0) ? results->aperData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->aperData[Nsec], measure, minRank, ID_MEAS_WARP_USED | ID_MEAS_PHOTOM_APER, ID_MEAS_MASKED_APER); + + // if too few valid measurements meet the minimum criteria, go to the next entry + StatType *kronstats = &results->kronstats; + // Nranking = MAG_STATS_BY_RANKING (&results->kronData[Nsec], kronstats); + Nranking = useOLS ? magStatsByRanking (&results->kronData[Nsec], kronstats) : magStatsByRankingIRLS (&results->kronData[Nsec], kronstats); + if (Nranking) { + secfilt[Nsec].FkronWrp = kronstats->mean; + secfilt[Nsec].dFkronWrp = kronstats->error; + secfilt[Nsec].sFkronWrp = (kronstats->Nmeas > 1) ? kronstats->chisq : NAN; + secfilt[Nsec].NusedKronWrp = kronstats->Nmeas; + secfilt[Nsec].MkronWrp = isnan(secfilt[Nsec].FkronWrp) ? NAN : 8.9 - 2.5*log10(secfilt[Nsec].FkronWrp); // 8.9 since flux is in Jy + } + minRank = (Nranking > 0) ? results->kronData[Nsec].ranking[0] : 10; + markMeasureByRanking (&results->kronData[Nsec], measure, minRank, ID_MEAS_WARP_USED | ID_MEAS_PHOTOM_KRON, ID_MEAS_MASKED_KRON); + + secfilt[Nsec].Nwarp = results->Nmeas[Nsec]; + secfilt[Nsec].NwarpGood = results->NmeasGood[Nsec]; + } + return (TRUE); +} + +int magStatsByRanking (StatDataSet *dataset, StatType *stats) { + + liststats_init (stats); + + if (dataset->Nlist == 0) return 0; + + // we have a list of measurements for this Nsec value, along with errors, weights, and their rank + // first, sort by the rank (increasing) + sort_StatDataSet (dataset); + + // find the values with the same minimum rank: + int Nranking = 0; + int minRank = dataset->ranking[0]; // MIN (5, result->psfData[Nsec].ranking[0]); -- only allow rank 5 or < + + int i; + for (i = 0; (i < dataset->Nlist) && (dataset->ranking[i] == minRank); i++, Nranking++); + + liststats (dataset->flxlist, dataset->errlist, dataset->wgtlist, Nranking, stats); + return (Nranking); +} + +int magStatsByRankingIRLS (StatDataSet *dataset, StatType *stats) { + + liststats_init (stats); + + if (dataset->Nlist == 0) return 0; + + // we have a list of measurements for this Nsec value, along with errors, weights, and their rank + // first, sort by the rank (increasing) + sort_StatDataSet (dataset); + + // find the values with the same minimum rank: + int Nranking = 0; + int minRank = dataset->ranking[0]; // MIN (5, result->psfData[Nsec].ranking[0]); -- only allow rank 5 or < + + int i; + for (i = 0; (i < dataset->Nlist) && (dataset->ranking[i] == minRank); i++, Nranking++); + + liststats_irls (dataset, Nranking, stats); + return (Nranking); +} + +// outlier warp measurements are driving bad mean values. +int magStatsByRankingClipped (StatDataSet *dataset, StatType *stats) { + + liststats_init (stats); + + if (dataset->Nlist == 0) return 0; + + // we have a list of measurements for this Nsec value, along with errors, weights, and their rank + // first, sort by the rank (increasing) + sort_StatDataSet (dataset); + + // find the values with the same minimum rank: + int Nranking = 0; + int minRank = dataset->ranking[0]; // MIN (5, result->psfData[Nsec].ranking[0]); -- only allow rank 5 or < + + int i; + for (i = 0; (i < dataset->Nlist) && (dataset->ranking[i] == minRank); i++, Nranking++); + + // we want to measure the weighted average, but first we want to clip points which are > 3 sigma from the median + // since we are only allowed to clip at most 25% of the points, we need to have at least 4 points to do this clipping + if (Nranking > 3) { + + double *fluxes = NULL; + double *offset = NULL; + int *entry = NULL; + + // now find the distances + ALLOCATE (entry, int, Nranking); + ALLOCATE (offset, double, Nranking); + ALLOCATE (fluxes, double, Nranking); + + // first, measure the median of the flxlist: + for (i = 0; i < Nranking; i++) { + fluxes[i] = dataset->flxlist[i]; + } + dsort (fluxes, Nranking); + int Nmidpoint = (int)(0.5*Nranking); // if Nranking is odd (e.g., 5), Nmidpoint is central bin (e.g., 2); if Nranking is even (e.g., 6), Nmidpoint is central bin + 1 (e.g., 3); + double medianFlux = (Nranking % 2) ? fluxes[Nmidpoint] : 0.5*(fluxes[Nmidpoint] + fluxes[Nmidpoint - 1]); + + for (i = 0; i < Nranking; i++) { + offset[i] = fabs((dataset->flxlist[i] - medianFlux) / dataset->errlist[i]); + entry[i] = i; + } + + sort_entry_by_offset (offset, entry, Nranking); + + // we are only going to clip at most 25% of the points. Start at the 75% point and check for outliers + int Nreject = 0; + for (i = 0.75*Nranking; i < Nranking; i++) { + if (offset[i] > 4.0) { + // this is an outlier -- add 10 to the ranking + dataset->ranking[entry[i]] += 10; + Nreject ++; + } + } + Nranking -= Nreject; // the excluded points are now removed from the count + + // re-sort by the rank so liststats below only operates on the unrejected points + sort_StatDataSet (dataset); + + free (fluxes); + free (offset); + free (entry); + } + + // get stats on only the first Nranking entries + liststats (dataset->flxlist, dataset->errlist, dataset->wgtlist, Nranking, stats); + + return (Nranking); +} + +int markMeasureByRanking (StatDataSet *dataset, Measure *measure, int minrank, DVOMeasureFlags keepflag, DVOMeasureFlags maskflag) { + int i; + + for (i = 0; i < dataset->Nlist; i++) { + int k = dataset->measSeq[i]; + if (dataset->ranking[i] > minrank) { + measure[k].dbFlags &= ~keepflag; + } else { + measure[k].dbFlags |= keepflag; + } + if (dataset->msklist[i]) measure[k].dbFlags |= maskflag; + } + return TRUE; +} + +/* sort a coordinate pair (X,Y) and the associated index (S) */ +void sort_StatDataSet (StatDataSet *dataset) { + +# define SWAPFUNC(A,B){ double dtmp; int itmp; \ + dtmp = dataset->flxlist[A]; dataset->flxlist[A] = dataset->flxlist[B]; dataset->flxlist[B] = dtmp; \ + dtmp = dataset->errlist[A]; dataset->errlist[A] = dataset->errlist[B]; dataset->errlist[B] = dtmp; \ + dtmp = dataset->wgtlist[A]; dataset->wgtlist[A] = dataset->wgtlist[B]; dataset->wgtlist[B] = dtmp; \ + itmp = dataset->ranking[A]; dataset->ranking[A] = dataset->ranking[B]; dataset->ranking[B] = itmp; \ + itmp = dataset->measSeq[A]; dataset->measSeq[A] = dataset->measSeq[B]; dataset->measSeq[B] = itmp; \ + } +# define COMPARE(A,B)(dataset->ranking[A] < dataset->ranking[B]) + + OHANA_SORT (dataset->Nlist, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +void sort_entry_by_offset (double *offset, int *entry, int N) { + +# define SWAPFUNC(A,B){ double dtmp; int itmp; \ + dtmp = offset[A]; offset[A] = offset[B]; offset[B] = dtmp; \ + itmp = entry[A]; entry[A] = entry[B]; entry[B] = itmp; \ + } +# define COMPARE(A,B)(offset[A] < offset[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE +} + +void GetPhotFlagStats (uint32_t *photFlagUpper, uint32_t *photFlagLower, uint32_t *photflag_list, int Nphotflag) { + + int i, bit; + + uint32_t outputUpper = 0; + uint32_t outputLower = 0; + + uint32_t flag = 0x1; // start with the flag on the 0 bit + for (bit = 0; bit < 32; bit++) { + + int Nraised = 0; + for (i = 0; i < Nphotflag; i++) { + if (photflag_list[i] & flag) Nraised ++; + } + + float fraction = Nraised / (float) Nphotflag; + + if (fraction < 0.25) { + // do nothing (no bits raised) + } + if ((fraction >= 0.25) && (fraction < 0.50)) { + outputLower |= flag; + } + if ((fraction >= 0.50) && (fraction < 0.75)) { + outputUpper |= flag; + } + if (fraction >= 0.75) { + outputLower |= flag; + outputUpper |= flag; + } + flag = (flag << 1); + } + *photFlagUpper = outputUpper; + *photFlagLower = outputLower; +} Index: /branches/eam_branches/relphot.20210521/src/setMrelFinal.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/setMrelFinal.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/setMrelFinal.c (revision 41602) @@ -0,0 +1,349 @@ +# include "relphot.h" + +// we've just reloaded the data from disk; we now need to apply the Image/Mosaic/Grid +// calibrations determined by the rest of the program. We also need to set the final +// output dbFlags values + +void setMrelFinal (Catalog *catalog, FlatCorrectionTable *flatcorr, int simpleAverage) { + + off_t i; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + /*** RESET photometry flags and Mcal values ***/ + if (RESET) { + // flags used by the photometry analysis (excluding UBERCAL) + unsigned int PHOTOM_FLAGS = + ID_MEAS_NOCAL | // detection ignored for this analysis (photcode, time range) + ID_MEAS_POOR_PHOTOM | // detection is photometry outlier + ID_MEAS_SKIP_PHOTOM | // detection was ignored for photometry measurement + ID_MEAS_AREA | // detetion was outside acceptable area of device + ID_MEAS_SYNTH_MAG | // magnitude is synthetic + ID_MEAS_STACK_PRIMARY | // this stack measurement is in the primary skycell + ID_MEAS_STACK_PHOT_SRC; // this measurement supplied the stack photometry + + // ID_MEAS_PHOTOM_UBERCAL -- externally-supplied zero point from ubercal analysis + // this is set by 'setphot', do not reset here + + // flags used by the photometry analysis (excluding UBERCAL) + // unsigned int secfiltFlags = + // ID_PHOTOM_PASS_0 | // average measured at pass 0 + // ID_PHOTOM_PASS_1 | // average measured at pass 1 + // ID_PHOTOM_PASS_2 | // average measured at pass 2 + // ID_PHOTOM_PASS_3 | // average measured at pass 3 + // ID_PHOTOM_PASS_4 | // average measured at pass 3 + // ID_SECF_USE_SYNTH | // average measured at pass 3 + // ID_SECF_USE_UBERCAL | // average measured at pass 3 + // ID_SECF_OBJ_EXT; // average measured at pass 3 + + // XXX I should really deprecate the concept of applying the average + // calculation to a limited set of photcodes. + // for now, just do all photcodes here ( + // as it stands, only stacks are limited by photcode; mean exp and forced warp + // are applied to all Nsecfilt + + for (i = 0; i < catalog[0].Naverage; i++) { + int Ns; + for (Ns = 0; Ns < Nsecfilt; Ns++) { + + off_t N = Nsecfilt*i+Ns; + dvo_secfilt_init (&catalog[0].secfilt[N], SECFILT_RESET_ALL); + + off_t m = catalog[0].average[i].measureOffset; + off_t j; + for (j = 0; j < catalog[0].average[i].Nmeasure; j++, m++) { + + /* select measurements by time */ + if (TimeSelect) { + if (catalog[0].measure[m].t < TSTART) continue; + if (catalog[0].measure[m].t > TSTOP) continue; + } + + // only reset Mcal for measures with a matching image + // do not reset Mcal for ubercal images unless explicitly requested + + if (catalog[0].measure[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) { + if (!KEEP_UBERCAL) { + catalog[0].measure[m].McalPSF = 0.0; + catalog[0].measure[m].McalAPER = 0.0; + catalog[0].measure[m].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL; + } + } else { + if (RESET_ZEROPTS && (getImageEntry (m, 0) >= 0)) { + catalog[0].measure[m].McalPSF = 0.0; + catalog[0].measure[m].McalAPER = 0.0; + } + } + catalog[0].measure[m].dbFlags &= ~PHOTOM_FLAGS; + } + } + } + } + + // this sets flags in the measureT element, not the measure element + setExclusions (catalog, 1, VERBOSE); /* mark by area */ + + /* set catalog[0].found[i] = FALSE */ + ALLOCATE (catalog[0].found_t, off_t, MAX (1, Nsecfilt*catalog[0].Naverage)); + ALLOCATE (catalog[0].foundWarp_t, off_t, MAX (1, Nsecfilt*catalog[0].Naverage)); + for (i = 0; i < Nsecfilt*catalog[0].Naverage; i++) { + catalog[0].found_t[i] = FALSE; + catalog[0].foundWarp_t[i] = FALSE; + } + + for (i = 0; i < catalog[0].Naverage; i++) { + catalog[0].average[i].psfQF = NAN; // force recalculation below + catalog[0].average[i].psfQFperf = NAN; // force recalculation below + catalog[0].average[i].stargal = NAN; // force recalculation below + catalog[0].average[i].photFlagsUpper = 0; // reset (will be re-calculated here) + catalog[0].average[i].photFlagsLower = 0; // reset (will be re-calculated here) + catalog[0].average[i].NwarpOK = 0; // reset (will be re-calculated here) + } + + // XXX make this optional? (do not clean for -averages?) + // XXX : for the moment, disable clean_measures + if (!simpleAverage && FALSE) clean_measures (catalog, 1, TRUE, flatcorr); /* mark outliers ID_MEAS_POOR_PHOTOM */ + + ALLOCATE (catalog[0].measureRank, char, catalog[0].Nmeasure); + setMeasureRank (catalog); + setMrelOutput (catalog, 1, flatcorr); // sets the values secfilt.MpsfChp = + setMcalOutput (catalog, 1, flatcorr); // sets measure.Mcal = image.Mcal + + /* clear ID_OBJ_POOR, ID_OBJ_FEW values before writing ??? */ + /* ID_MEAS_NOCAL is an internal bit, so it should be cleared */ + // XXX eventually: move these internal flags to an internal-only catalog->FOO element (like found) + for (i = 0; i < catalog[0].Naverage; i++) { + catalog[0].average[i].flags &= ~ID_OBJ_FEW; + catalog[0].average[i].flags &= ~ID_OBJ_POOR; + off_t j; + for (j = 0; j < Nsecfilt; j++) { + catalog[0].secfilt[i*Nsecfilt+j].flags &= ~ID_OBJ_FEW; + catalog[0].secfilt[i*Nsecfilt+j].flags &= ~ID_OBJ_POOR; + } + off_t m = catalog[0].average[i].measureOffset; + for (j = 0; j < catalog[0].average[i].Nmeasure; j++, m++) { + catalog[0].measure[m].dbFlags &= ~ID_MEAS_NOCAL; + } + } +} + +/* ID_MEAS_SKIP marks measurements which were not used to calculate Mrel */ +// For each star & photcode, we are selecting the measurements to use in the average +// photometry value. This function is called several times, lowering the bar to +// acceptance on each pass. The rejections at the different passes are: + +// 0 : only use measurements thought to be GOOD (photflags not POOR or BAD) +// 1 : accept measurements thought to be POOR based on photflags +// 2 : accept the measurements marked as outliers; accept images marked as outliers +// 3 : accept measurements thought to be BAD based on photflags +// 4 : accept the measurements outside of the instrumental magnitude limits (eg, SAT) + +// XXX if I'm reading this correctly, passes 0, 1, 2 are identical & accept all +// non-outliers with Imag range + +// int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure) { +// +// off_t k; +// +// int Nsecfilt = GetPhotcodeNsecfilt (); +// +// off_t m = average[0].measureOffset; +// +// for (k = 0; k < average[0].Nmeasure; k++, m++) { +// fprintf (stderr, "meas: %08x\n", measure[m].dbFlags); +// } +// +// int Ns; +// for (Ns = 0; Ns < Nsecfilt; Ns++) { +// fprintf (stderr, "secf: %08x\n", secfilt[Ns].flags); +// } +// return 1; +// } + +int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure); + +void skip_measurements (Catalog *catalog, int pass, FlatCorrectionTable *flatcorr) { + + off_t i, k, m; + off_t Ntot, Ntry, Nkeep, Nskip; + float mag; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + Ntot = Ntry = Nskip = Nkeep = 0; + + /* allow measures from images marked POOR and FEW */ + if (pass >= 3) IMAGE_BAD = ID_IMAGE_PHOTOM_NOCAL; + + /* allow measures marked as outliers (POOR) and off image region (AREA) */ + if (pass >= 3) { + MEAS_BAD = ID_MEAS_NOCAL | ID_MEAS_SKIP_PHOTOM; + } else { + MEAS_BAD = ID_MEAS_NOCAL | ID_MEAS_POOR_PHOTOM | ID_MEAS_SKIP_PHOTOM | ID_MEAS_AREA; + } + + /* mark measures which should be ignored on this pass */ + for (i = 0; i < catalog[0].Naverage; i++) { + Ntot += catalog[0].average[i].Nmeasure; + + if (FALSE && (catalog[0].average[i].objID == 0x46a4) && (catalog[0].average[i].catID == 0xf40e)) { + fprintf (stderr, "test obj\n"); + print_measure_set (&catalog[0].average[i], &catalog[0].secfilt[i*Nsecfilt], catalog[0].measure); + } + + // mark measurements for each secfilt separately + int Ns; + for (Ns = 0; Ns < Nphotcodes; Ns++) { + + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + + /* star/photcodes already calibrated */ + if (catalog[0].found_t[Nsecfilt*i+Nsec]) continue; + + m = catalog[0].average[i].measureOffset; + for (k = 0; k < catalog[0].average[i].Nmeasure; k++, m++) { + Ntry++; + + // skip measurements not related to this photcode + PhotCode *code = GetPhotcodebyCode (catalog[0].measure[m].photcode); + if (!code) continue; + if (code->equiv != thisCode) continue; + + // skip measurements by time range (mark as skipped for this secfilt) + // XXX note that this is a bit dangerous : some objects may never get calibrated + if (TimeSelect) { + if (catalog[0].measure[m].t < TSTART) goto skip; + if (catalog[0].measure[m].t > TSTOP) goto skip; + } + + // we now have a measurement relavant to this photcode (and time range) + + // clear SKIP for all measures at first + catalog[0].measureT[m].dbFlags &= ~ID_MEAS_SKIP_PHOTOM; + catalog[0].measure [m].dbFlags &= ~ID_MEAS_SKIP_PHOTOM; + + // skip measurements from BAD images and mosaics (not REF mags) + // do NOT skip measurements without a matching image (REF mags) + off_t Nim = getImageEntry (m, 0); + if (Nim > -1) { + if (isnan(getMcal (m, 0, MAG_CLASS_PSF))) goto skip; + if (isnan(getMmos (m, 0))) goto skip; + } + + // PASS 4 : skip measurements by inst mag limit (not REF mags) + if ((pass < 4) && ImagSelect) { + if (Nim > -1) { + mag = PhotInst (&catalog[0].measure[m], MAG_CLASS_PSF); + if (mag < ImagMin) goto skip; + if (mag > ImagMax) goto skip; + } + } + + // PASS 3 : accept bad measurements (eg, SAT, CR), internal outliers + if (pass < 3) { + if (catalog[0].measure[m].photFlags & code->photomBadMask) goto skip; + if ((catalog[0].measure[m].photcode > 10000) && (catalog[0].measure[m].photcode < 10500)) { + if (catalog[0].measure[m].psfQF < 0.85) goto skip; + } + } + + // PASS 2 : (no additional cut) + + // PASS 1 : accept poor measurements as well (eg, POOR FIT, etc) + if ((pass < 1) && (catalog[0].measure[m].photFlags & code->photomPoorMask)) goto skip; + + // PASS 0 : skip poor, outlier, bad measurements as well (eg, POOR FIT, etc) + + Nkeep ++; + + continue; + + skip: + catalog[0].measure [m].dbFlags |= ID_MEAS_SKIP_PHOTOM; + catalog[0].measureT[m].dbFlags |= ID_MEAS_SKIP_PHOTOM; + Nskip ++; + } + } + } + if (VERBOSE) fprintf (stderr, "pass %d, Ntot: "OFF_T_FMT", Ntry: "OFF_T_FMT", Nskip: "OFF_T_FMT", Nkeep: "OFF_T_FMT"\n", + pass, Ntot, Ntry, Nskip, Nkeep); +} + +void setMeasureRank (Catalog *catalog) { + + int i; + + // make these global + int IMAGE_BAD = ID_IMAGE_PHOTOM_NOCAL; + int IMAGE_POOR = ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_SKIP; + int MEAS_BAD = ID_MEAS_NOCAL | ID_MEAS_SKIP_PHOTOM; + int MEAS_POOR = ID_MEAS_POOR_PHOTOM | ID_MEAS_AREA; + + Measure *measure = catalog[0].measure; + MeasureTiny *measureT = catalog[0].measureT; + char *measureRank = catalog[0].measureRank; + + /* set measureRank[] based on various quality measurements */ + for (i = 0; i < catalog[0].Nmeasure; i++) { + measureRank[i] = 11; // start at a low rank + + // clear SKIP for all measures at first + measureT[i].dbFlags &= ~ID_MEAS_SKIP_PHOTOM; + measure [i].dbFlags &= ~ID_MEAS_SKIP_PHOTOM; + + // skip measurements without a valid photcode + PhotCode *code = GetPhotcodebyCode (measure[i].photcode); + if (!code) continue; + + // measurements outside time range have poor rank + if (TimeSelect) { + if (measure[i].t < TSTART) { measureRank[i] = 10; continue; } + if (measure[i].t > TSTOP) { measureRank[i] = 10; continue; } + } + + // measurements from BAD images and mosaics (not REF mags) + off_t Nim = getImageEntry (i, 0); + int imageFlags = getImageFlags (i, 0); + int mosaicFlags = getMosaicFlags (i, 0); + + if (Nim > -1) { + if (imageFlags & IMAGE_BAD) { measureRank[i] = 9; continue; } + + if (mosaicFlags & IMAGE_BAD) { measureRank[i] = 9; continue; } + + // measurements ranked by inst mag limit (not REF mags) + if (ImagSelect) { + float mag = PhotInst (&measure[i], MAG_CLASS_PSF); + if (mag < ImagMin) { measureRank[i] = 8; continue; } + if (mag > ImagMax) { measureRank[i] = 8; continue; } + } + } + + // RANK 1 : BAD photFlags + if (measure[i].dbFlags & MEAS_BAD) { measureRank[i] = 7; continue; } + + // RANK 3 : BAD photFlags (eg, SAT, CR), internal outliers + if (measure[i].photFlags & code->photomBadMask) { measureRank[i] = 6; continue; } + + if (Nim > -1) { + if (imageFlags & IMAGE_POOR) { measureRank[i] = 5; continue; } + if (mosaicFlags & IMAGE_POOR) { measureRank[i] = 5; continue; } + } + + // RANK 2 : psfQF value + if (!isfinite(measure[i].psfQF) || measure[i].psfQF < 0.85) { measureRank[i] = 4; continue; } + + // RANK 1 : POOR photFlags + if (measure[i].dbFlags & MEAS_POOR) { measureRank[i] = 3; continue; } + + // RANK 1 : POOR photFlags + if (measure[i].photFlags & code->photomPoorMask) { measureRank[i] = 2; continue; } + + // RANK 1 : psfQFperfect value + if (!isfinite(measure[i].psfQFperf) || measure[i].psfQFperf < 0.85) { measureRank[i] = 1; continue; } + + // RANK 0 : perfect measurement: + measureRank[i] = 0; + } +} Index: /branches/eam_branches/relphot.20210521/src/share_image_mags.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/share_image_mags.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/share_image_mags.c (revision 41602) @@ -0,0 +1,122 @@ +# include "relphot.h" + +// we are sharing image calibrations mags for all images which (a) I own and (b) which have unowned detections + +# define D_NIMAGEMAGS 1000 +int share_image_mags (RegionHostTable *regionHosts, int nloop) { + + off_t i, Nimages; + Image *images = getimages (&Nimages, NULL); + + off_t Nimage_mags = 0; + off_t NIMAGE_MAGS = D_NIMAGEMAGS; + + ImageMag *image_mags = NULL; + ALLOCATE (image_mags, ImageMag, NIMAGE_MAGS); + + for (i = 0; i < Nimages; i++) { + // XXX does this image have missing detections (does someone else need it?) + // XXX : NOTE NEED TO FIX THIS: if (imageExtra[i].Nmiss == 0) continue; + + set_image_mags (&image_mags[Nimage_mags], &images[i]); + Nimage_mags ++; + + CHECK_REALLOCATE (image_mags, ImageMag, NIMAGE_MAGS, Nimage_mags, D_NIMAGEMAGS); + } + + // write out the image_mag fits table AND write state in some file + int myHost = regionHosts->index[REGION_HOST_ID]; + char *hostname = regionHosts->hosts[myHost].hostname; + + char *imagfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "imagemags.fits"); + ImageMagSave (imagfile, image_mags, Nimage_mags); + free (image_mags); + free (imagfile); + + char *syncfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "imagemags.sync"); + update_sync_file (syncfile, nloop); + free (syncfile); + + return TRUE; +} + +int slurp_image_mags (RegionHostTable *regionHosts, int nloop) { + + off_t Nimage, i; + Image *images = getimages (&Nimage, NULL); + + int Nimage_mags = 0; + ImageMag *image_mags = NULL; + ALLOCATE (image_mags, ImageMag, 1); + + fprintf (stderr, "grabbing image mags from other hosts...\n"); + + for (i = 0; i < regionHosts->Nhosts; i++) { + if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue; + if (REGION_HOST_ID && !regionHosts->hosts[i].isNeighbor) continue; + + char *syncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.sync"); + check_sync_file (syncfile, nloop); + free (syncfile); + + off_t Nsubset; + char *imagfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "imagemags.fits"); + ImageMag *image_mags_subset = ImageMagLoad (imagfile, &Nsubset); + free (imagfile); + + image_mags = merge_image_mags (image_mags, &Nimage_mags, image_mags_subset, Nsubset); + } + + for (i = 0; i < Nimage_mags; i++) { + off_t seq = getImageByID (image_mags[i].imageID); + if (seq < 0) { + // XXX is this a problem? (no, other hosts don't know which images I own) + continue; + } + images[seq].McalPSF = image_mags[i].McalPSF; + images[seq].McalAPER = image_mags[i].McalAPER; + images[seq].dMcal = image_mags[i].dMcal; + images[seq].dMagSys = image_mags[i].dMagSys; + images[seq].McalChiSq = image_mags[i].McalChiSq; + images[seq].nFitPhotom = image_mags[i].nFitPhotom; + images[seq].flags = image_mags[i].flags; + images[seq].ubercalDist = image_mags[i].ubercalDist; + } + free (image_mags); + + fprintf (stderr, "DONE grabbing image mags from other hosts\n"); + + return TRUE; +} + +int set_image_mags (ImageMag *image_mags, Image *image) { + + image_mags->McalPSF = image->McalPSF; + image_mags->McalAPER = image->McalAPER; + image_mags->dMcal = image->dMcal; + image_mags->dMagSys = image->dMagSys; + image_mags->McalChiSq = image->McalChiSq; + image_mags->nFitPhotom = image->nFitPhotom; + image_mags->flags = image->flags; + image_mags->ubercalDist = image->ubercalDist; + image_mags->imageID = image->imageID; + + return TRUE; +} + +ImageMag *merge_image_mags (ImageMag *target, int *ntarget, ImageMag *source, int Nsource) { + + off_t i; + + REALLOCATE (target, ImageMag, *ntarget + Nsource); + for (i = 0; i < Nsource; i++) { + off_t n = i + *ntarget; + target[n] = source[i]; + } + + free (source); + + *ntarget += Nsource; + return (target); +} + Index: /branches/eam_branches/relphot.20210521/src/share_mean_mags.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/share_mean_mags.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/share_mean_mags.c (revision 41602) @@ -0,0 +1,150 @@ +# include "relphot.h" +// we are sharing mean mags for all objects which (a) I own and (b) which have unowned detections + +# define D_NMEANMAGS 10000 +int share_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) { + + int i; + off_t j; + + off_t Nmeanmags = 0; + off_t NMEANMAGS = D_NMEANMAGS; + + MeanMag *meanmags = NULL; + ALLOCATE (meanmags, MeanMag, NMEANMAGS); + + int Ns; + int Nsecfilt = GetPhotcodeNsecfilt(); + + int myHost = regionHosts->index[REGION_HOST_ID]; + double Rmin = regionHosts->hosts[myHost].Rmin; + double Rmax = regionHosts->hosts[myHost].Rmax; + double Dmin = regionHosts->hosts[myHost].Dmin; + double Dmax = regionHosts->hosts[myHost].Dmax; + + // XXX skip some catalogs based on UserPatch? + for (i = 0; i < Ncatalog; i++) { + for (j = 0; j < catalog[i].Naverage; j++) { + + // do I own this object? (in region range?) + if (catalog[i].averageT[j].R < Rmin) continue; + if (catalog[i].averageT[j].R >= Rmax) continue; + if (catalog[i].averageT[j].D < Dmin) continue; + if (catalog[i].averageT[j].D >= Dmax) continue; + + // does this object have missing detections (does someone else need it?) + // XXX : sky objects without missing detections + // XXX watch out for detections which are not associated with an image (REF) + if (catalog[i].averageT[j].nOwn == catalog[i].averageT[j].Nmeasure) continue; + + for (Ns = 0; Ns < Nphotcodes; Ns++) { + int thisCode = photcodes[Ns][0].code; + int Nsec = GetPhotcodeNsec(thisCode); + set_mean_mags (&meanmags[Nmeanmags], &catalog[i].averageT[j], &catalog[i].secfilt[Nsecfilt*j + Nsec], Nsec); + Nmeanmags ++; + CHECK_REALLOCATE (meanmags, MeanMag, NMEANMAGS, Nmeanmags, D_NMEANMAGS); + } + } + } + + // write out the meanmag fits table AND write state in some file + char *hostname = regionHosts->hosts[myHost].hostname; + + char *magsfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "meanmags.fits"); + MeanMagSave (magsfile, meanmags, Nmeanmags); + free (meanmags); + free (magsfile); + + char *syncfile = make_filename (CATDIR, hostname, REGION_HOST_ID, "meanmags.sync"); + update_sync_file (syncfile, nloop); + free (syncfile); + + return TRUE; +} + +int slurp_mean_mags (Catalog *catalog, int Ncatalog, RegionHostTable *regionHosts, int nloop) { + OHANA_UNUSED_PARAM(Ncatalog); + + off_t i; + + int Nmeanmags = 0; + MeanMag *meanmags = NULL; + ALLOCATE (meanmags, MeanMag, 1); + + fprintf (stderr, "grabbing mean object mags from other hosts...\n"); + + int Nsecfilt = GetPhotcodeNsecfilt(); + + for (i = 0; i < regionHosts->Nhosts; i++) { + // if (not_neighbor(host[i])) continue; + if (regionHosts->hosts[i].hostID == REGION_HOST_ID) continue; + if (REGION_HOST_ID && !regionHosts->hosts[i].isNeighbor) continue; + + char *syncfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.sync"); + check_sync_file (syncfile, nloop); + free (syncfile); + + off_t Nsubset = 0; + char *magsfile = make_filename (CATDIR, regionHosts->hosts[i].hostname, regionHosts->hosts[i].hostID, "meanmags.fits"); + MeanMag *meanmagsSubset = MeanMagLoad (magsfile, &Nsubset); + free (magsfile); + + // merge_mean_mags reallocs meanmags and frees the input meanmagsSubset + meanmags = merge_mean_mags (meanmags, &Nmeanmags, meanmagsSubset, Nsubset); + } + + for (i = 0; i < Nmeanmags; i++) { + int objID = meanmags[i].objID; + int catID = meanmags[i].catID; + + // set the mean mag + int catSeq; + off_t objSeq; + if (!catID_and_objID_to_seq (catID, objID, &catSeq, &objSeq)) { + // XXX what should I do if this does not match? + continue; + } + + int ecode = GetPhotcodeEquivCodebyCode (meanmags[i].photcode); + if (ecode < 0) continue; + int Nsec = GetPhotcodeNsec (ecode); + if (Nsec < 0) continue; + + catalog[catSeq].secfilt[objSeq*Nsecfilt + Nsec].MpsfChp = meanmags[i].M; + } + free (meanmags); + + fprintf (stderr, "DONE grabbing mean object mags from other hosts...\n"); + + return TRUE; +} + +int set_mean_mags (MeanMag *meanmags, AverageTiny *average, SecFilt *secfilt, int Nsec) { + + meanmags->M = secfilt->MpsfChp; + meanmags->dM = secfilt->dMpsfChp; + meanmags->Mchisq = secfilt->Mchisq; + meanmags->Nsec = Nsec; // key to secfilt entry + + meanmags->objID = average->objID; + meanmags->catID = average->catID; + + return TRUE; +} + +MeanMag *merge_mean_mags (MeanMag *target, int *ntarget, MeanMag *source, int Nsource) { + + off_t i; + + REALLOCATE (target, MeanMag, *ntarget + Nsource); + for (i = 0; i < Nsource; i++) { + off_t n = i + *ntarget; + target[n] = source[i]; + } + + free (source); + + *ntarget += Nsource; + return (target); +} + Index: /branches/eam_branches/relphot.20210521/src/syncfile.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/syncfile.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/syncfile.c (revision 41602) @@ -0,0 +1,77 @@ +# include "relphot.h" +# define MSG_LENGTH 12 + +char *make_filename (char *dirname, char *hostname, int hostID, char *tailname) { + + char tmp[10], *line; + int Nchar = snprintf (tmp, 0, "%s/%s.%03d.%s", dirname, hostname, hostID, tailname); + + ALLOCATE (line, char, Nchar + 1); + snprintf (line, Nchar + 1, "%s/%s.%03d.%s", dirname, hostname, hostID, tailname); + + return line; +} + +int check_sync_file (char *filename, int nloop) { + + char message[MSG_LENGTH]; + + FILE *f = NULL; + + while (TRUE) { + + f = fopen (filename, "r"); + if (!f) { + usleep (2000000); + continue; + } + + // XXX MSG_LENGTH : 0 EOL byte? + int Nread = fread (message, 1, MSG_LENGTH, f); + if (Nread < MSG_LENGTH) { + fclose (f); + usleep (2000000); + continue; + } + fclose (f); + + // message is of the form: NLOOP: %03d + int loop; + sscanf (message, "%*s %d", &loop); + if (loop != nloop) { + usleep (2000000); + continue; + } + return TRUE; + } + return FALSE; +} + +int clear_sync_file (char *filename) { + // delete file contents + if (truncate (filename, 0)) fprintf (stderr, "trouble clearing file %s\n", filename); + + return TRUE; +} + +int update_sync_file (char *filename, int nloop) { + + char message[MSG_LENGTH]; + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "failure to open sync file for write\n"); + exit (4); + } + + snprintf (message, MSG_LENGTH, "NLOOP: %03d\n", nloop); + + int Nwrite = fwrite (message, 1, MSG_LENGTH, f); + if (Nwrite != MSG_LENGTH) { + fprintf (stderr, "failure to write sync message\n"); + exit (3); + } + + fclose (f); + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/synthetic_mags.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/synthetic_mags.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/synthetic_mags.c (revision 41602) @@ -0,0 +1,72 @@ +# include "relphot.h" + +static int NSr = -1; +static int NSi = -1; +static int W_SYNTH = -1; + +int init_synthetic_mags () { + int Pr = GetPhotcodeCodebyName ("r"); + int Pi = GetPhotcodeCodebyName ("i"); + + NSr = GetPhotcodeNsec (Pr); + NSi = GetPhotcodeNsec (Pi); + + W_SYNTH = GetPhotcodeCodebyName ("SYNTH.w"); + + return TRUE; +} + +int add_synthetic_mags (AverageTiny *average, SecFilt *secfilt, MeasureTiny *measure, off_t *Nmeasure, off_t *Nm) { + OHANA_UNUSED_PARAM(average); + + if (!SyntheticPhotometry) return TRUE; + + myAssert (NSr >= 0, "failed to call init_synthetic_mags"); + myAssert (NSi >= 0, "failed to call init_synthetic_mags"); + myAssert (W_SYNTH >= 0, "failed to call init_synthetic_mags"); + + // XXX for now, hard-wire the photometric relationship + + // we want to create a synthetic w-band measurement based on the observed mean r & i photometry + + // from Tonry + // (w-r)_obs = 0.042 + 0.166 (r-i)_obs - 0.398 (r-i)_obs^2, (r-i)_obs < 0.5 + // (w-r)_obs = 0.268 - 0.435 (r-i)_obs - 0.078 (r-i)_obs^2, (r-i)_obs > 0.5 + // thus, for (r-i < 0.5): + // (r - i < 0.5) : w = r + 0.042 + 0.166*(r-i) - 0.398(r-i)^2 + // (r - i > 0.5) : w = r + 0.268 - 0.435 (r-i) - 0.078(r-i)^2 + + float Mr = secfilt[NSr].MpsfChp; + float Mi = secfilt[NSi].MpsfChp; + + if (!isfinite(Mr)) return FALSE; + if (!isfinite(Mi)) return FALSE; + + // apply a color correction + // XXX this is very GPC1 specific and hard-wired -- be very afraid! + // select the filter; default to fixed photcode and mag limit otherwise + + float Mri = Mr - Mi; + + // saturate at some valid range limits + Mri = MAX (MIN(Mri, 2.0), -0.2); + + float Mw = NAN; + if (Mri < 0.5) { + Mw = Mr + 0.042 + 0.166*(Mri) - 0.398*SQ(Mri); + } else { + Mw = Mr + 0.268 - 0.435*(Mri) - 0.078*SQ(Mri); + } + + dvo_measureT_init (measure); + + measure->M = Mw; + measure->dM = 0.001; + measure->photcode = W_SYNTH; + measure->dbFlags |= ID_MEAS_SYNTH_MAG; + + (*Nmeasure) ++; + (*Nm) ++; + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/synthetic_zpts.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/synthetic_zpts.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/synthetic_zpts.c (revision 41602) @@ -0,0 +1,57 @@ +# include "relphot.h" + +// the PS1 Synthetic magnitudes (Pitts & Magnier) have known systematic errors due to the +// POSS plate photometry. This file contains support code to correct that error based on +// sky maps of the mean offsets. + +static char *extname[5] = {"map_g", "map_r", "map_i", "map_z", "map_y"}; + +static SynthZeroPoints *zpts = NULL; + +int SynthZeroPointsLoad (char *filename) { + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open file %s\n", filename); + return FALSE; + } + + // SynthZeroPoint file contains: + // PHU header (including WCS) + // map_g + // map_r + // map_i + // map_z + // map_y + + ALLOCATE (zpts, SynthZeroPoints, 1); + + if (!gfits_fread_header (f, &zpts->PHU)) { + if (VERBOSE) fprintf (stderr, "can't read header\n"); + fclose (f); + return FALSE; + } + + int i; + for (i = 0; i < 5; i++) { + if (!gfits_find_Xheader(f, &zpts->header[i], extname[i])) { + fprintf (stderr, "ERROR: file %s is missing extension %s\n", filename, extname[i]); + exit (2); + } + + if (!gfits_fread_matrix (f, &zpts->matrix[i], &zpts->header[i])) { + if (VERBOSE) fprintf (stderr, "can't read image for %s\n", extname[i]); + exit (2); + } + } + + GetCoords (&zpts->coords, &zpts->PHU); + zpts->Nx = zpts->matrix[0].Naxis[0]; + zpts->Ny = zpts->matrix[0].Naxis[1]; + + return TRUE; +} + +SynthZeroPoints *SynthZeroPointsGet () { + return zpts; +} Index: /branches/eam_branches/relphot.20210521/src/test_fitirls.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/test_fitirls.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/test_fitirls.c (revision 41602) @@ -0,0 +1,182 @@ +# include "relphot.h" + +int mkdata (FitDataSet *dataset, int fitSlope, double valueTru, double slopeTru, int Npoints, int Nbad, int GaussianOutliers); + +int main (int argc, char **argv) { + + int Ntests = 100; + int Npoints = 100; + int Noutliers = 10; + int Nbootstrap = 100; + int Niterations = 10; + + int N; + if ((N = get_argument (argc, argv, "-Ntests"))) { + remove_argument (N, &argc, argv); + Ntests = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Npoints"))) { + remove_argument (N, &argc, argv); + Npoints = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Noutliers"))) { + remove_argument (N, &argc, argv); + Noutliers = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Nbootstrap"))) { + remove_argument (N, &argc, argv); + Nbootstrap = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Niterations"))) { + remove_argument (N, &argc, argv); + Niterations = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + + int GaussianOutliers = TRUE; + if ((N = get_argument (argc, argv, "-flat-outliers"))) { + remove_argument (N, &argc, argv); + GaussianOutliers = FALSE; + } + + int FitSlope = FALSE; + if ((N = get_argument (argc, argv, "-fit-slope"))) { + remove_argument (N, &argc, argv); + FitSlope = TRUE; + } + + double ValuePrior = NAN; + double ValueSigma = NAN; + if ((N = get_argument (argc, argv, "-value-prior"))) { + remove_argument (N, &argc, argv); + ValuePrior = atof (argv[N]); + remove_argument (N, &argc, argv); + ValueSigma = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + double SlopePrior = NAN; + double SlopeSigma = NAN; + if ((N = get_argument (argc, argv, "-slope-prior"))) { + if (!FitSlope) { fprintf (stderr, "-slope-prior only valid if -fit-slope is selected\n"); exit (2); } + remove_argument (N, &argc, argv); + SlopePrior = atof (argv[N]); + remove_argument (N, &argc, argv); + SlopeSigma = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 1) { + fprintf (stderr, "USAGE: %s\n", argv[0]); + fprintf (stderr, " some options: -Ntests [3000], -Npoints [100], -Noutliers [10], -flat-outliers\n"); + fprintf (stderr, " more options: -fit-slope, -value-prior (value) (sigma), -slope-prior (slope) (sigma)\n"); + exit (2); + } + + // plan_tests (14); + // diag ("relastro fitpm tests"); + + // init the random seed + { + struct timeval now; + gettimeofday (&now, NULL); + long A = now.tv_sec + now.tv_usec * 1000000; + srand48(A); + // srand48(1); + } + + ohana_gaussdev_init (); + + FitDataSet dataset; + FitDataSetAlloc (&dataset, Npoints + Noutliers, FitSlope, Nbootstrap); + dataset.MaxIterations = Niterations; + + if (isfinite(ValuePrior) || isfinite(SlopePrior)) { + FitDataSetAddPriors (&dataset); + if (isfinite(ValuePrior)) { + dataset.bPriorValue[0] = ValuePrior; + dataset.bPriorSigma[0] = ValueSigma; + } + if (isfinite(SlopePrior)) { + dataset.bPriorValue[1] = SlopePrior; + dataset.bPriorSigma[1] = SlopeSigma; + } + } + + for (int i = 0; i < Ntests; i++) { + + double valueTru = 25.0; + double slopeTru = 0.05; + + // generate a single fake star with Npoints real points and Noutliers bad points + mkdata (&dataset, FitSlope, valueTru, slopeTru, Npoints, Noutliers, GaussianOutliers); + + fit1d_irls (&dataset, Npoints + Noutliers); + + if (FitSlope) { + fprintf (stdout, "%8.3f %8.5f : %6.3f %6.3f : %8.5f %8.3f %6.3f : %6.3f %6.3f : %6.3f : %4d %4d %4d\n", valueTru, dataset.bSaveArray[0][0], dataset.bSigma[0], slopeTru, dataset.bSaveArray[1][0], dataset.bSigma[1], dataset.sigma, dataset.min, dataset.max, dataset.chisq, Npoints, Noutliers, dataset.Nmeas); + } else { + fprintf (stdout, "%8.3f %8.5f : %6.3f %6.3f : %6.3f %6.3f : %6.3f : %4d %4d %4d\n", valueTru, dataset.bSaveArray[0][0], dataset.bSigma[0], dataset.sigma, dataset.min, dataset.max, dataset.chisq, Npoints, Noutliers, dataset.Nmeas); + } + } + + ohana_gaussdev_free(); + + FitDataSetFree (&dataset); + + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + + exit (0); +} + +// we are using IRLS 1D fitting to solve for zero points and airmass slope +// I will generate data samples with a nominal zero point and an airmass slope + +// valueObs = valueTru + slopeTru*trend + +// choose a random value for 'trend' between MIN_TREND and MAX_TREND +// set the raw value to valueTru + slopeTru*trend +// vary by Gaussian deviate using sigma of VAL_ERROR + +# define VAL_ERROR 0.03 +# define MIN_TREND 0.0 +# define MAX_TREND 1.0 +int mkdata (FitDataSet *dataset, int FitSlope, double valueTru, double slopeTru, int Npoints, int Nbad, int GaussianOutliers) { + + for (int i = 0; i < Npoints; i++) { + double dF = ohana_gaussdev_rnd(0.0, VAL_ERROR); + + double trend = MIN_TREND + (MAX_TREND - MIN_TREND)*drand48(); + + double valueRaw = FitSlope ? valueTru + slopeTru*trend : valueTru; + double valueObs = valueRaw + dF; + + dataset->alldata->yVector[i] = valueObs; + dataset->alldata->dyVector[i] = VAL_ERROR; + if (FitSlope) { + dataset->alldata->xVector[i] = trend; + } + } + + for (int i = 0; i < Nbad; i++) { + double dF = GaussianOutliers ? ohana_gaussdev_rnd(10.0*VAL_ERROR, 10.0*VAL_ERROR) : 10.0*VAL_ERROR*drand48(); + + double trend = MIN_TREND + (MAX_TREND - MIN_TREND)*drand48(); + + double valueRaw = FitSlope ? valueTru + slopeTru*trend : valueTru; + double valueObs = valueRaw + dF; + + dataset->alldata->yVector[i + Npoints] = valueObs; + dataset->alldata->dyVector[i + Npoints] = VAL_ERROR; + if (FitSlope) { + dataset->alldata->xVector[i + Npoints] = trend; + } + } + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/test_liststats.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/test_liststats.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/test_liststats.c (revision 41602) @@ -0,0 +1,155 @@ +# include "relphot.h" + +int mkstar (StatDataSet *dataset, double flux, int Npoints, int Nbad, int GaussianOutliers); + +static int Nrand = 0; +double drand48_cnt () { + double value = drand48(); + Nrand ++; + return value; +} + +typedef enum { + FIT_FLUX_NONE, + FIT_FLUX_MEAN, + FIT_FLUX_IRLS, +} TestModeType; + +int main (int argc, char **argv) { + + int N_STARS = 3000; + int N_POINTS = 100; + int N_OUTLIERS = 10; + // int N_BOOTSTRAP = 100; + + int N; + if ((N = get_argument (argc, argv, "-Nstars"))) { + remove_argument (N, &argc, argv); + N_STARS = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Npoints"))) { + remove_argument (N, &argc, argv); + N_POINTS = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-Noutliers"))) { + remove_argument (N, &argc, argv); + N_OUTLIERS = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + int GaussianOutliers = TRUE; + if ((N = get_argument (argc, argv, "-flat-outliers"))) { + remove_argument (N, &argc, argv); + GaussianOutliers = FALSE; + } + + if (argc != 2) { + fprintf (stderr, "USAGE: %s (mode)\n", argv[0]); + fprintf (stderr, " options: -Nstars [3000], -Npoints [100], -Noutliers [10], -flat-outliers\n"); + exit (2); + } + + TestModeType mode = FIT_FLUX_NONE; + if (!strcasecmp (argv[1], "irls")) mode = FIT_FLUX_IRLS; + if (!strcasecmp (argv[1], "mean")) mode = FIT_FLUX_MEAN; + if (!mode) { + fprintf (stderr, " mode options: mean, irls\n"); + exit (2); + } + + // plan_tests (14); + + // diag ("relastro fitpm tests"); + + // init the random seed + { + struct timeval now; + gettimeofday (&now, NULL); + long A = now.tv_sec + now.tv_usec * 1000000; + srand48(A); + // srand48(1); + } + + ohana_gaussdev_init (); + + // allow for 5 filters and 1-to-1 outliers + StatDataSet *dataset = StatDataSetAlloc (1, N_POINTS + N_OUTLIERS); + + StatType stats; + liststats_init (&stats); + + int Nstars = N_STARS; + for (int i = 0; i < Nstars; i++) { + + double lflux = 2.0 + 3.0*drand48_cnt(); // uniform in log-flux + double flux = pow(10.0, lflux); + + int Npoints = N_POINTS; + int Noutliers = N_OUTLIERS; + + // generate a single fake star with N_POINTS real points and N_OUTLIERS bad points + mkstar (dataset, flux, Npoints, Noutliers, GaussianOutliers); + + switch (mode) { + case FIT_FLUX_MEAN: + liststats_setmode (&stats, "MEAN"); + liststats (dataset->flxlist, dataset->errlist, dataset->wgtlist, Npoints + Noutliers, &stats); + break; + + case FIT_FLUX_IRLS: + liststats_irls (dataset, Npoints + Noutliers, &stats); + break; + + default: myAbort ("programming error"); + } + + fprintf (stdout, "%8.2f %8.2f : %6.2f %6.2f : %6.3f : %4d %4d %4d\n", + flux, stats.mean, stats.sigma, stats.error, stats.chisq, Npoints, Noutliers, stats.Nmeas); + } + // return exit_status(); + ohana_gaussdev_free(); + + StatDataSetFree (dataset, 1); + + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + + exit (0); +} + +# define READ_NOISE_SQR 25.0 +int mkstar (StatDataSet *dataset, double flux, int Npoints, int Nbad, int GaussianOutliers) { + + double FluxError = sqrt(flux); + + for (int i = 0; i < Npoints; i++) { + double dF = ohana_gaussdev_rnd(0.0, FluxError); + + double fluxObs = flux + dF; + double dFluxObs = sqrt(fluxObs + READ_NOISE_SQR); + + dataset->flxlist[i] = fluxObs; + dataset->errlist[i] = dFluxObs; + dataset->wgtlist[i] = 1.0; + dataset->ranking[i] = 1; + dataset->measSeq[i] = i; + } + + for (int i = 0; i < Nbad; i++) { + + // double dF = GaussianOutliers ? ohana_gaussdev_rnd(0.0, 10.0*FluxError) : 10.0*FluxError*(drand48_cnt() - 0.5); + double dF = GaussianOutliers ? ohana_gaussdev_rnd(10.0*FluxError, 10.0*FluxError) : 10.0*FluxError*drand48_cnt(); + + double fluxObs = flux + dF; + double dFluxObs = sqrt(fluxObs + READ_NOISE_SQR); + + dataset->flxlist[Npoints + i] = fluxObs; + dataset->errlist[Npoints + i] = dFluxObs; + dataset->wgtlist[Npoints + i] = 1.0; + dataset->ranking[Npoints + i] = 1; + dataset->measSeq[Npoints + i] = i; + } + + return TRUE; +} Index: /branches/eam_branches/relphot.20210521/src/write_coords.c =================================================================== --- /branches/eam_branches/relphot.20210521/src/write_coords.c (revision 41602) +++ /branches/eam_branches/relphot.20210521/src/write_coords.c (revision 41602) @@ -0,0 +1,29 @@ +# include "relphot.h" + +# define CD_COORDS 1 + +void write_coords (Header *header, Coords *coords) { + + gfits_modify (header, "CTYPE1", "%s", 1, "RA---TAN"); + gfits_modify (header, "CTYPE2", "%s", 1, "DEC--TAN"); + + gfits_modify (header, "CRVAL1", "%lf", 1, coords[0].crval1); + gfits_modify (header, "CRVAL2", "%lf", 1, coords[0].crval2); + + gfits_modify (header, "CRPIX1", "%lf", 1, coords[0].crpix1); + gfits_modify (header, "CRPIX2", "%lf", 1, coords[0].crpix2); + +# if (CD_COORDS) + gfits_modify (header, "CD1_1", "%le", 1, coords[0].pc1_1 * coords[0].cdelt1); + gfits_modify (header, "CD2_1", "%le", 1, coords[0].pc2_1 * coords[0].cdelt1); + gfits_modify (header, "CD1_2", "%le", 1, coords[0].pc1_2 * coords[0].cdelt2); + gfits_modify (header, "CD2_2", "%le", 1, coords[0].pc2_2 * coords[0].cdelt2); +# else + gfits_modify (header, "CDELT1", "%le", 1, coords[0].cdelt1); + gfits_modify (header, "CDELT2", "%le", 1, coords[0].cdelt2); + gfits_modify (header, "PC001001", "%le", 1, coords[0].pc1_1); + gfits_modify (header, "PC001002", "%le", 1, coords[0].pc1_2); + gfits_modify (header, "PC002001", "%le", 1, coords[0].pc2_1); + gfits_modify (header, "PC002002", "%le", 1, coords[0].pc2_2); +# endif +} Index: /branches/eam_branches/relphot.20210521/test/dvo.config =================================================================== --- /branches/eam_branches/relphot.20210521/test/dvo.config (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/dvo.config (revision 41602) @@ -0,0 +1,17 @@ +IMAGE_SCATTER 0.075 # mark images POOR if stdev(Mcal) > IMAGE_SCATTER +STAR_SCATTER 0.005 # mark stars POOR if stdev(Mrel) > STAR_SCATTER +IMAGE_OFFSET 0.100 # mark images POOR if abs(delta(Mcal)) > IMAGE_OFFSET +STAR_CHISQ 10.000 # mark stars POOR if Xm > STAR_CHISQ +STAR_TOOFEW 2 # mark star FEW if N(good) < STAR_TOOFEW +GRID_TOOFEW 10 # keep grid FROZEN if N(good) < GRID_TOOFEW +IMAGE_TOOFEW 10 # mark image FEW if N(good) < IMAGE_TOOFEW +IMAGE_GOOD_FRACTION 0.050 # mark image FEW if N(good) < IMAGE_GOOD_FRACTION * Nstars + +SCATTER_LIM 15.000 +MAG_LIM 20.000 # select stars brighter than this for relphot analysis +SIGMA_LIM 0.015 # select measurements with dM < SIGMA_LIM for relphot analysis + +RELPHOT_GRID_BINNING 256 +RELPHOT_GRID_X 4 +RELPHOT_GRID_Y 4 + Index: /branches/eam_branches/relphot.20210521/test/mana.sh =================================================================== --- /branches/eam_branches/relphot.20210521/test/mana.sh (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/mana.sh (revision 41602) @@ -0,0 +1,63 @@ + +macro go + if ($0 != 2) + echo "USAGE: go (datafile)" + break + end + + data $1 + read Fo 1 Fx 2 dFo 4 dFx 5 chisq 7 Nstar 9 Nbad 10 Nfit 11 + set dF = Fo - Fx + set n = ramp(Fo) + + # predicted errorbar compared to measured errorbar + set dfo = dFo / sqrt(Nstar) + lim -n 1 dfo dFx; clear; box; plot dfo dFx + line -c red 0 0 to 50 50 + + # number fitted + set fFit = Nfit / Nstar + lim -n 2 n fFit; clear; box; plot n fFit + + # fitted mean vs truth scaled by sigma + set dNsigma = dF / dFx + lim -n 0 n dNsigma; clear; box; plot n dNsigma + + histogram dNsigma NdNsigma -5.0 5.0 0.1 -range dx + $C0 = 0 + $C1 = 1.0 + $C2 = 100 + $C3 = 0 + vgauss -q dx NdNsigma con NdNsigmaF + vstat -q dNsigma + + lim -n 3 dx NdNsigma; clear; box; plot -x 1 dx NdNsigma; plot -x 0 dx NdNsigmaF -c red + + echo $C0 $C1 $MEAN $SIGMA +end + +macro testset + if ($0 != 2) + echo "USAGE: testset (mode)" + break + end + + foreach Nout 0 1 2 4 8 16 + echo --- $Nout --- + exec bin/test_liststats.lin64 $1 -Noutliers $Nout > test.$1.dat; go test.$1.dat + echo ------------- + end +end + +macro testset.flat + if ($0 != 2) + echo "USAGE: testset (mode)" + break + end + + foreach Nout 0 1 2 4 8 16 + echo --- $Nout --- + exec bin/test_liststats.lin64 $1 -Noutliers $Nout -flat-outliers > test.$1.dat; go test.$1.dat + echo ------------- + end +end Index: /branches/eam_branches/relphot.20210521/test/ptolemy.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/ptolemy.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/ptolemy.dvo (revision 41602) @@ -0,0 +1,63 @@ + +macro mkptolemy.rc + # generate a fake ptolemy.rc (to make this a contained test) + exec rm -f ptolemy.rc + output ptolemy.rc + echo "\#" + echo PHOTCODE_FILE dvo.photcodes + echo GSCFILE GSCregions.tbl + echo ZERO_PT 25.0 + echo ADDSTAR_RADIUS 1.0 + echo CATMODE SPLIT + echo CATFORMAT PS1_V5 + echo SKY_DEPTH 4 + echo CAMERA gpc1 + echo MOSAICNAME GPC1 + echo CAMERA_CONFIG dvo.layout + echo JD-KEYWORD NONE + echo MJD-KEYWORD MJD-OBS + echo CCDNUM-KEYWORD EXTNAME + echo EXPTIME-KEYWORD EXPTIME + echo AIRMASS-KEYWORD AIRMASS + echo "input dvo.config" + output stdout + + # we need to define a photcode table consistent with the fake construction + $zpt_nominal:g = 24.0 + $zpt_nominal:r = 24.5 + + # klam is negative, so klam*(airmass - 1) increases the zero point + $klam_nominal:g = -0.15 + $klam_nominal:r = -0.10 + + local found + file GSCregions.tbl found + if (not($found)) + file GSCregions.tbl.gz found + if (not($found)) + echo "missing GSCregions.tbl file or gzipped version" + break + end + exec gunzip -c GSCregions.tbl.gz > GSCregions.tbl + end + + # generate a fake dvo.photcode + exec rm -f dvo.photcodes + output dvo.photcodes + echo "\# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo "1 g sec 0.000 0.000 0.000 1 3 0.0000 0 1051 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo "2 r sec 0.000 0.000 0.000 2 3 0.0000 0 1052 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo "10000 GPC1.g.XY00 dep $zpt_nominal:g $klam_nominal:g 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10001 GPC1.g.XY01 dep $zpt_nominal:g $klam_nominal:g 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10010 GPC1.g.XY10 dep $zpt_nominal:g $klam_nominal:g 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10011 GPC1.g.XY11 dep $zpt_nominal:g $klam_nominal:g 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10100 GPC1.r.XY00 dep $zpt_nominal:r $klam_nominal:r 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10101 GPC1.r.XY01 dep $zpt_nominal:r $klam_nominal:r 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10110 GPC1.r.XY10 dep $zpt_nominal:r $klam_nominal:r 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "10111 GPC1.r.XY11 dep $zpt_nominal:r $klam_nominal:r 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88" + echo "2011 2MASS_J ref 0.000 0.000 0.000 - - 0.0000 0 6 0.080 0.000 2.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo "3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 nan 0.000 2.000 100.0 0x0000 0x0000 0x0000 0x0000" + echo "3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 1 nan 0.000 2.000 50.0 0x0000 0x0000 0x0000 0x0000" + echo "2020 TYCHO_B ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end Index: /branches/eam_branches/relphot.20210521/test/relphot.basic.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/relphot.basic.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/relphot.basic.dvo (revision 41602) @@ -0,0 +1,448 @@ +# -*-sh-*- + +$TAP_BREAK = 0 +$TAP_VERBOSE = 1 +input tap.dvo +input ptolemy.dvo + +# set globals +if (not($?PLOT)) set PLOT = 0 + +$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets +$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact) +$DO_2MASS = 1 +$DO_SYNTH = 1 +$DO_TYCHO = 1 +$NC_CLOUDS = 0.20 +$UC_CLOUDS = 0.02 + +## full test set: generate a set of test files, insert into a catdir, check raw catalog, run relphot, check results +macro go + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + dirname $1 -var rootdir + mkdir $rootdir + + break -auto off + exec rm -rf $catdir + exec rm -rf $fileroot* + break -auto on + + # define a set of photometric (ubercal) and non-photometric exposure times + init + + # generate the common list of stars (coords & mags) for all exposures + mkstars 1000 + + catdir $catdir + + tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3} + + # generate the basic images and check they were correctly ingested by dvo + for i 0 mjd_uc[] + mkexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i $fileroot $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw + end + for i 0 mjd_nc[] + mkexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i $fileroot $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i raw + end + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + # reset the per-filter ZPT_OFF:$filter values + for i 0 $filt_uc:n + $filter = $filt_uc:$i + $ZPT_OFF:$filter = NAN + end + for i 0 $filt_nc:n + $filter = $filt_nc:$i + $ZPT_OFF:$filter = NAN + end + + for i 0 mjd_uc[] + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc + end + for i 0 mjd_nc[] + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc + end + tapDONE +end + +# set various global variables +macro init + + # generates ptolemy.rc, dvo.photcodes, $zpt_nominal:g,r $klam_nominal:g,r + mkptolemy.rc + + $RA_CENTER = 10.0 + $DEC_CENTER = 20.0 + $PLATE_SCALE = 0.25 + $NSEASON = 3 + $NFILTER = 3 + $NCHIP_X = 2 + $NCHIP_Y = 2 + $NCELL_X = 2 + $NCELL_Y = 2 + $CHIP_DX = 1000 + $CHIP_DY = 1000 + $DEFECT_FRAC = 0.05 + $OFFSET_FRAC_UC = 0.0 + $OFFSET_FRAC_NC = 0.5 + + # we have two sets of images: ubercal (photometric) and not-ubercaled + + # ********* UBERCAL IMAGES *********** + + # the number of exposures is defined by filt_uc:n + list filt_uc -split g g g g r r r r + + # tmpseq is used to generate vectors in this function + create tmpseq 0 $filt_uc:n + + # exptime and airmasses for uc images + set exptime_uc = 10.0 + zero(tmpseq) + +# XXX range of airmass + set airmass_uc = 1.3 + zero(tmpseq) + set klam_uc = zero(tmpseq) + set zpt_uc = zero(tmpseq) + + # ubercal zero points are defined as ZP_nominal + 2.5log(exptime) + K*(airmass - 1.0) + # note that K is defined as a negative value (is this sensible?) + + # zero points and airmass slopes for these sequences + for i 0 $filt_uc:n + klam_uc[$i] = $klam_nominal:$filt_uc:$i + zpt_uc[$i] = $zpt_nominal:$filt_uc:$i + 2.5*log(exptime_uc[$i]) + klam_uc[$i]*(airmass_uc[$i] - 1.0) + $UC_CLOUDS*(rnd(klam_uc[$i]) - 0.5) + $ZPT_OFF:$filt_uc:$i = NAN + end + set mjd_uc = 55000.00 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3) + + # ********* NON-UBERCAL IMAGES *********** + + # mjd and zpt values for not-ubercal'ed images + list filt_nc -split g g g g r r r r + + # tmpseq is used to generate vectors in this function + create tmpseq 0 $filt_nc:n + + # place this within a valid season (55000.0 - 55010.0 - 55020.0 - 55030.0) + set exptime_nc = 10.0 + zero(tmpseq) + set airmass_nc = 1.6 + zero(tmpseq) + set klam_nc = zero(tmpseq) + set zpt_nc = zero(tmpseq) + + # airmass slopes for these sequnece + for i 0 $filt_nc:n + klam_nc[$i] = $klam_nominal:$filt_nc:$i + zpt_nc[$i] = $zpt_nominal:$filt_nc:$i + 2.5*log(exptime_nc[$i]) + klam_nc[$i]*(airmass_nc[$i] - 1.0) + $NC_CLOUDS*(rnd(klam_nc[$i]) - 0.5) + $ZPT_OFF:$filt_uc:$i = NAN + end + set mjd_nc = 55000.10 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3) + + delete tmpseq +end + +macro try.relphot + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + # reset the per-filter ZPT_OFF:$filter values + for i 0 $filt_uc:n + $filter = $filt_uc:$i + $ZPT_OFF:$filter = NAN + end + for i 0 $filt_nc:n + $filter = $filt_nc:$i + $ZPT_OFF:$filter = NAN + end + + for i 0 mjd_uc[] + # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc + end + for i 0 mjd_nc[] + # ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc + end +end + +macro ckexposure + if ($0 != 8) + echo "ckexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (mode)" + echo " mode == raw or corr" + break + end + + local CATDIR MJD_IMAGE ZPT_REAL ZPT_NOMINAL EXPTIME FILTER AIRMASS + + $CATDIR = $1 + $MJD_IMAGE = $2 + $ZPT_REAL = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $FUNC_MODE = $7 + + # XXX need a function to extract the nominal zpt for a given filter / photcode from the db + $ZPT_NOMINAL = $zpt_nominal:$FILTER + $KLAM_NOMINAL = $klam_nominal:$FILTER + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + catdir $CATDIR + + # assume we still have stars_ra, stars_dec, stars_mag in hand + if ($PLOT) + dev -n 0 + region $RA_CENTER $DEC_CENTER 0.2 + images + pmeasure -all -m 15 20 + end + + skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} + + ## calculate the average zero point offset + if (isnan($ZPT_OFF:$FILTER)) + avextract ra dec $FILTER + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + reindex stars_tru_m = stars_mag using index1 + set dm_tru = $FILTER - stars_tru_m + vstat -q dm_tru + $ZPT_OFF:$FILTER = $MEDIAN + end + + mextract ra dec mag xccd yccd dbflags photflags time mag:sys mag:rel mag:inst mag:ave where (abs(time - $MJD_IMAGE) < 0.0001) + + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + + reindex stars_ra_m = stars_ra using index1 + reindex stars_dec_m = stars_dec using index1 + reindex stars_mag_m = stars_mag using index1 + set dr = 3600*(ra - stars_ra_m) + set dd = 3600*(dec - stars_dec_m) + set dm = mag - stars_mag_m + + set good_photom = (not(photflags & 0x02)) + foreach field dm xccd yccd + subset $field\_good = $field if good_photom + end + + if ($PLOT) + dev -n 1 + lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002) + dev -n 2 + lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002) + dev -n 3 + lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002) + echo "red points are known bad photometry" + end + + $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0) + + if ("$FUNC_MODE" == "raw") + set dm_adjust = dm_good - $ZPT_NOMINAL + $ZPT_REAL_NORM + + vstat -q dm_adjust + sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f): %7.4f %7.4f : %6.3f" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + tapOK {abs($MEAN) < 0.005} "RAW : $line" + return + end + + if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc")) + + ## relphot does not set an absolute zero point, so we need to find that and subtract + set dm_adjust = dm_good - $ZPT_OFF:$FILTER + + vstat -q dm_adjust + sprintf line "relphot ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f) : %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + + if ("$FUNC_MODE" == "relphot_uc") + tapOK {abs($MEAN) < 0.001} "MEAN : $line" + tapOK {abs($SIGMA) < 0.001} "SIGMA: $line" + end + if ("$FUNC_MODE" == "relphot_nc") + # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias + tapOK {abs($MEAN) < 0.020} "MEAN : $line" + tapOK {abs($SIGMA) < 0.020} "SIGMA: $line" + end + + if ($PLOT) + cursor 1 + if ("$KEY" == "Escape") break + end + return + end + + echo "ERROR: unknown mode for ckexposure : $FUNC_MODE" +end + +macro mkexposure + if ($0 != 10) + echo "mkexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (fileroot) (ra) (dec)" + break + end + + local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd + + $CATDIR = $1 + $MJD = $2 + $ZPT = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $ROOT = $7 + $RAo = $8 + $DECo = $9 + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + # XX ctimes -ref $MJD -var datetime + # XX substr $datetime 0 10 date + # XX substr $datetime 11 8 time + + for ix 0 $NCHIP_X + for iy 0 $NCHIP_Y + $dx = $CHIP_DX * $PLATE_SCALE * ($ix - 0.5*$NCHIP_X + 0.5) + $dy = $CHIP_DY * $PLATE_SCALE * ($iy - 0.5*$NCHIP_Y + 0.5) + $ra = $RAo - $dx / 3600.0 / dcos($DECo) + $dec = $DECo - $dy / 3600.0 + # echo $ra $dec $dx $dy + mkinput test.in.txt $ra $dec $ZPT + + # ra,dec is the center of this chip + local options + $options = -mjd $MJD + $options = $options -radec $ra $dec + + #$options = $options -type PS1_V2 + #$options = $options -type PS1_V3 + #$options = $options -type PS1_V4 + $options = $options -type PS1_V5 + + $options = $options -coords + $options = $options -photcode GPC1.$FILTER.XY$ix\$iy + $options = $options -no-noise + $options = $options -flags + $options = $options -bad-psfqf-frac $DEFECT_FRAC + $options = $options -size $CHIP_DX $CHIP_DY + $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} + $options = $options -airmass $AIRMASS + $options = $options -exptime $EXPTIME + + tapEXEC mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options + + # the fake images have inconsistent ra,dec and airmass,sidtime values + tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass + end + end +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 5) + echo "mkinput (filename) (ra) (dec) (zpt)" + break + end + + # chip coordinate of the stars + # ra,dec is at the chip center, which corresponds to pixel (CHIP_DX,CHIP_DY)/2 + set stars_X = (stars_ra - $2) * 3600.0 * dcos($3) / $PLATE_SCALE + $CHIP_DX*0.5 + set stars_Y = (stars_dec - $3) * 3600.0 / $PLATE_SCALE + $CHIP_DY*0.5 + set stars_M = (stars_mag - $4) + + set valid = (stars_X > 0) && (stars_X < $CHIP_DX) && (stars_Y > 0) && (stars_Y < $CHIP_DX) + subset stars_x = stars_X if valid + subset stars_y = stars_Y if valid + subset stars_m = stars_M if valid + subset stars_r = stars_ra if valid + subset stars_d = stars_dec if valid + subset stars_mr = stars_mag if valid + + # if we want to modify the observed magnitude based on the camera locations, add code here: + set stars_mi = stars_m + + # create bad detections / detections to ignore or avoid, save to a file? + + # create 3 types of things to ignore / avoid + # * badPhot (deviate the photometry) + # * badFlag (set a bad flag bit) + # * poorFlag (set a bad flag bit) + set badPhot = rnd(stars_r) < $DEFECT_FRAC + set badFlag = rnd(stars_r) < $DEFECT_FRAC + set poorFlag = rnd(stars_r) < $DEFECT_FRAC + + # create deviant photometry (1%) + # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated. This is a simple way to pass the info + # note that outlier rejection only occurs for Nmeas > 5 + set stars_mi = stars_mi + 0.2*badPhot + set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot + + # bad flags that we sould respect: + # PM_SOURCE_MODE_FAIL = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero) + # PM_SOURCE_MODE_SATSTAR = 0x00000080, ///< Source model peak is above saturation + # PM_SOURCE_MODE_BADPSF = 0x00000400, ///< Failed to get good estimate of object's PSF + # PM_SOURCE_MODE_DEFECT = 0x00000800, ///< Source is thought to be a defect + # PM_SOURCE_MODE_SATURATED = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail) + # PM_SOURCE_MODE_CR_LIMIT = 0x00002000, ///< Source has crNsigma above limit + # PM_SOURCE_MODE_MOMENTS_FAILURE = 0x00008000, ///< could not measure the moments + # PM_SOURCE_MODE_SKY_FAILURE = 0x00010000, ///< could not measure the local sky + # PM_SOURCE_MODE_SKYVAR_FAILURE = 0x00020000, ///< could not measure the local sky variance + # PM_SOURCE_MODE_SIZE_SKIPPED = 0x10000000, ///< size could not be determined + + # poor flags that we should respect: + # PM_SOURCE_MODE_POOR = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?) + # PM_SOURCE_MODE_PAIR = 0x00000020, ///< Source fitted with a double psf + # PM_SOURCE_MODE_BLEND = 0x00000100, ///< Source is a blend with other sources + # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N + # PM_SOURCE_MODE_BLEND_FIT = 0x00400000, ///< source was fitted as a blend + # PM_SOURCE_MODE_ON_SPIKE = 0x20000000, ///< peak lands on diffraction spike + # PM_SOURCE_MODE_ON_GHOST = 0x40000000, ///< peak lands on ghost or glint + # PM_SOURCE_MODE_OFF_CHIP = 0x80000000, ///< peak lands off edge of chip + + write -f "%10.6f %10.6f %8.4f %6.1f %6.1f %8.4f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl +end + +# generate a set of stars with raw RA, DEC, MAG values to use for all of the output cmf files +macro mkstars + if ($0 != 2) + echo "USAGE: mkstars (Nstars)" + break + end + + local RA_RANGE DEC_RANGE + + # the images are oriented along N-S, E-W lines + + # size of region of interest in linear arcseconds + $RA_RANGE = 1.25 * $CHIP_DX * $NCHIP_X * $PLATE_SCALE + $DEC_RANGE = 1.25 * $CHIP_DY * $NCHIP_Y * $PLATE_SCALE + + create tmp 0 $1 + set stars_ra = $RA_CENTER + $RA_RANGE * (rnd(tmp) - 0.5) / 3600 / dcos ($DEC_CENTER) + set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) / 3600 + set stars_mag = 10.0 + 10.0 * rnd(tmp) + set stars_dmag = 0.02 + 0.1 * rnd(tmp) +end Index: /branches/eam_branches/relphot.20210521/test/relphot.mosaic.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/relphot.mosaic.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/relphot.mosaic.dvo (revision 41602) @@ -0,0 +1,485 @@ +# -*-sh-*- + +$TAP_BREAK = 0 +$TAP_VERBOSE = 1 +input tap.dvo +input ptolemy.dvo + +# set globals +if (not($?PLOT)) set PLOT = 0 + +$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets +$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact) +$DO_2MASS = 1 +$DO_SYNTH = 1 +$DO_TYCHO = 1 +$NC_CLOUDS = 0.20 +$UC_CLOUDS = 0.02 + +## full test set: generate a set of test files, insert into a catdir, check raw catalog, run relphot, check results +macro go + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + dirname $1 -var rootdir + mkdir $rootdir + + break -auto off + exec rm -rf $catdir + exec rm -rf $fileroot* + break -auto on + + # define a set of photometric (ubercal) and non-photometric exposure times + init + + # generate the common list of stars (coords & mags) for all exposures + mkstars 1000 + + catdir $catdir + + tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3} + + # XXX first, run the analysis with only ubercal data (photometric data) + # we should be able to recover the nightly zero points fitting only tgroups + + # generate the basic images and check they were correctly ingested by dvo + for i 0 mjd_uc[] + sprintf filerootout "%s.uc.%02d" $fileroot $i + mkexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER raw + end + + for i 0 mjd_nc[] + sprintf filerootout "%s.nc.%02d" $fileroot $i + mkexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER raw + end + + break + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -tgroup-fit-airmass -tgroups tgroups.dat -images g -v -region $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + break + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end + for i 0 mjd_nc[] + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $FILTER relphot_nc + end + tapDONE +end + +# set various global variables +macro init + + # generates ptolemy.rc, dvo.photcodes, $zpt_nominal:g,r $klam_nominal:g,r + mkptolemy.rc + + $RA_CENTER = 10.0 + $DEC_CENTER = 20.0 + $PLATE_SCALE = 0.25 + $NCHIP_X = 2 + $NCHIP_Y = 2 + $CHIP_DX = 1000 + $CHIP_DY = 1000 + $DEFECT_FRAC = 0.00 + + # we have two sets of images: ubercal (photometric) and not-ubercaled + + # ********* UBERCAL IMAGES *********** + + # ubercal nights each have a distinct zero point and slope: + vlist night_uc 55000.0 55010.0 55030.0 55060.0 + vlist dz_nt_uc +0.0000 +0.0300 -0.0200 -0.0400 +# vlist dk_nt_uc +0.0000 +0.0005 -0.0005 -0.0002 + vlist dk_nt_uc +0.0000 +0.0200 -0.0200 +0.0300 + + write tgroups.dat night_uc + + $EXPTIME = 10.0 + $FILTER = g + $ZPT_OFF:$FILTER = NAN + + delete -q mjd_uc seq_uc zpt_uc klam_uc secz_uc exptime_uc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_uc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = $klam_nominal:$FILTER + dk_nt_uc + set exp_tmp = zero(night_uc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + dz_nt_uc + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) + + set seq_tmp = ramp(night_uc) + set mjd_tmp = night_uc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_uc + concat secz_tmp secz_uc + concat zpt_tmp zpt_uc + concat seq_tmp seq_uc + concat mjd_tmp mjd_uc + + concat exp_tmp exptime_uc + end + + # ********* NON-UBERCAL IMAGES *********** + + # non-ubercal nights each have a random zero points and fixed slope (klam_nominal) + # non-ubercal nights must not be the same set as the ubercal nights above + vlist night_nc 55005.0 55015.0 55035.0 55065.0 + + delete -q mjd_nc seq_nc zpt_nc klam_nc secz_nc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_nc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = zero(night_nc) + $klam_nominal:$FILTER + set exp_tmp = zero(night_nc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) - $NC_CLOUDS*(0.1 + rnd(secz_tmp)) + # observed zero points for nc nights are 0.1 - 1.1 lower than the ubercal nights + + set seq_tmp = ramp(night_nc) + set mjd_tmp = night_nc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_nc + concat secz_tmp secz_nc + concat zpt_tmp zpt_nc + concat seq_tmp seq_nc + concat mjd_tmp mjd_nc + + concat exp_tmp exptime_nc + end +end + +macro try.relphot + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end +end + +macro ckexposure + if ($0 != 8) + echo "ckexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (mode)" + echo " mode == raw or corr" + break + end + + local CATDIR MJD_IMAGE ZPT_REAL ZPT_NOMINAL EXPTIME FILTER AIRMASS + + $CATDIR = $1 + $MJD_IMAGE = $2 + $ZPT_REAL = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $FUNC_MODE = $7 + + # XXX need a function to extract the nominal zpt for a given filter / photcode from the db + $ZPT_NOMINAL = $zpt_nominal:$FILTER + $KLAM_NOMINAL = $klam_nominal:$FILTER + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + catdir $CATDIR + + # assume we still have stars_ra, stars_dec, stars_mag in hand + if ($PLOT) + dev -n 0 + region $RA_CENTER $DEC_CENTER 0.2 + images + pmeasure -all -m 15 20 + end + + # RA_MIN, etc are defined by mkstars + skyregion $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX + + ## calculate the average zero point offset + if (isnan($ZPT_OFF:$FILTER)) + avextract ra dec $FILTER + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + reindex stars_tru_m = stars_mag using index1 + set dm_tru = $FILTER - stars_tru_m + vstat -q dm_tru + $ZPT_OFF:$FILTER = $MEDIAN + end + + mextract ra dec mag xccd yccd dbflags photflags time mag:sys mag:rel mag:inst mag:ave where (abs(time - $MJD_IMAGE) < 0.0001) + + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + + reindex stars_ra_m = stars_ra using index1 + reindex stars_dec_m = stars_dec using index1 + reindex stars_mag_m = stars_mag using index1 + set dr = 3600*(ra - stars_ra_m) + set dd = 3600*(dec - stars_dec_m) + set dm = mag - stars_mag_m + + set good_photom = (not(photflags & 0x02)) + foreach field dm xccd yccd + subset $field\_good = $field if good_photom + end + + if ($PLOT) + dev -n 1 + lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002) + dev -n 2 + lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002) + dev -n 3 + lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002) + echo "red points are known bad photometry" + end + + $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0) + + if ("$FUNC_MODE" == "raw") + set dm_adjust = dm_good - $ZPT_NOMINAL + $ZPT_REAL_NORM + + vstat -q dm_adjust + sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f): %7.4f %7.4f : %6.3f" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + tapOK {abs($MEAN) < 0.005} "RAW : $line" + return + end + + if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc")) + + ## relphot does not set an absolute zero point, so we need to find that and subtract + set dm_adjust = dm_good - $ZPT_OFF:$FILTER + + vstat -q dm_adjust + sprintf line "relphot ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f) : %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + + if ("$FUNC_MODE" == "relphot_uc") + tapOK {abs($MEAN) < 0.001} "MEAN : $line" + tapOK {abs($SIGMA) < 0.001} "SIGMA: $line" + end + if ("$FUNC_MODE" == "relphot_nc") + # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias + tapOK {abs($MEAN) < 0.020} "MEAN : $line" + tapOK {abs($SIGMA) < 0.020} "SIGMA: $line" + end + + if ($PLOT) + cursor 1 + if ("$KEY" == "Escape") break + end + return + end + + echo "ERROR: unknown mode for ckexposure : $FUNC_MODE" +end + +macro mkexposure + if ($0 != 10) + echo "mkexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (fileroot) (ra) (dec)" + break + end + + local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd + + $CATDIR = $1 + $MJD = $2 + $ZPT = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $ROOT = $7 + $RAo = $8 + $DECo = $9 + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + # XX ctimes -ref $MJD -var datetime + # XX substr $datetime 0 10 date + # XX substr $datetime 11 8 time + + for ix 0 $NCHIP_X + for iy 0 $NCHIP_Y + $dx = $CHIP_DX * $PLATE_SCALE * ($ix - 0.5*$NCHIP_X + 0.5) + $dy = $CHIP_DY * $PLATE_SCALE * ($iy - 0.5*$NCHIP_Y + 0.5) + $ra = $RAo - $dx / 3600.0 / dcos($DECo) + $dec = $DECo - $dy / 3600.0 + # echo $ra $dec $dx $dy + mkinput $ROOT.$ix.$iy.in.txt $ra $dec $ZPT + + # ra,dec is the center of this chip + local options + $options = -mjd $MJD + $options = $options -radec $ra $dec + + #$options = $options -type PS1_V2 + #$options = $options -type PS1_V3 + #$options = $options -type PS1_V4 + $options = $options -type PS1_V5 + + $options = $options -coords + $options = $options -photcode GPC1.$FILTER.XY$ix\$iy + $options = $options -no-noise + $options = $options -flags + $options = $options -bad-psfqf-frac $DEFECT_FRAC + $options = $options -size $CHIP_DX $CHIP_DY + $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} + $options = $options -airmass $AIRMASS + $options = $options -exptime $EXPTIME + + tapEXEC mkcmf $ROOT.$ix.$iy.in.txt $ROOT.$ix.$iy.cmf $options + + # the fake images have inconsistent ra,dec and airmass,sidtime values + tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass + end + end +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 5) + echo "mkinput (filename) (ra) (dec) (zpt)" + break + end + + # chip coordinate of the stars + # ra,dec is at the chip center, which corresponds to pixel (CHIP_DX,CHIP_DY)/2 + set stars_X = (stars_ra - $2) * 3600.0 * dcos($3) / $PLATE_SCALE + $CHIP_DX*0.5 + set stars_Y = (stars_dec - $3) * 3600.0 / $PLATE_SCALE + $CHIP_DY*0.5 + set stars_M = (stars_mag - $4) + + set valid = (stars_X > 0) && (stars_X < $CHIP_DX) && (stars_Y > 0) && (stars_Y < $CHIP_DX) + subset stars_x = stars_X if valid + subset stars_y = stars_Y if valid + subset stars_m = stars_M if valid + subset stars_r = stars_ra if valid + subset stars_d = stars_dec if valid + subset stars_mr = stars_mag if valid + + # if we want to modify the observed magnitude based on the camera locations, add code here: + set stars_mi = stars_m + + # create bad detections / detections to ignore or avoid, save to a file? + + # create 3 types of things to ignore / avoid + # * badPhot (deviate the photometry) + # * badFlag (set a bad flag bit) + # * poorFlag (set a bad flag bit) + set badPhot = rnd(stars_r) < $DEFECT_FRAC + set badFlag = rnd(stars_r) < $DEFECT_FRAC + set poorFlag = rnd(stars_r) < $DEFECT_FRAC + + # create deviant photometry (1%) + # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated. This is a simple way to pass the info + # note that outlier rejection only occurs for Nmeas > 5 + set stars_mi = stars_mi + 0.2*badPhot + set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot + + # bad flags that we sould respect: + # PM_SOURCE_MODE_FAIL = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero) + # PM_SOURCE_MODE_SATSTAR = 0x00000080, ///< Source model peak is above saturation + # PM_SOURCE_MODE_BADPSF = 0x00000400, ///< Failed to get good estimate of object's PSF + # PM_SOURCE_MODE_DEFECT = 0x00000800, ///< Source is thought to be a defect + # PM_SOURCE_MODE_SATURATED = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail) + # PM_SOURCE_MODE_CR_LIMIT = 0x00002000, ///< Source has crNsigma above limit + # PM_SOURCE_MODE_MOMENTS_FAILURE = 0x00008000, ///< could not measure the moments + # PM_SOURCE_MODE_SKY_FAILURE = 0x00010000, ///< could not measure the local sky + # PM_SOURCE_MODE_SKYVAR_FAILURE = 0x00020000, ///< could not measure the local sky variance + # PM_SOURCE_MODE_SIZE_SKIPPED = 0x10000000, ///< size could not be determined + + # poor flags that we should respect: + # PM_SOURCE_MODE_POOR = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?) + # PM_SOURCE_MODE_PAIR = 0x00000020, ///< Source fitted with a double psf + # PM_SOURCE_MODE_BLEND = 0x00000100, ///< Source is a blend with other sources + # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N + # PM_SOURCE_MODE_BLEND_FIT = 0x00400000, ///< source was fitted as a blend + # PM_SOURCE_MODE_ON_SPIKE = 0x20000000, ///< peak lands on diffraction spike + # PM_SOURCE_MODE_ON_GHOST = 0x40000000, ///< peak lands on ghost or glint + # PM_SOURCE_MODE_OFF_CHIP = 0x80000000, ///< peak lands off edge of chip + + write -f "%10.6f %10.6f %8.4f %6.1f %6.1f %8.4f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl +end + +# generate a set of stars with raw RA, DEC, MAG values to use for all of the output cmf files +macro mkstars + if ($0 != 2) + echo "USAGE: mkstars (Nstars)" + break + end + + local RA_RANGE DEC_RANGE + + # the images are oriented along N-S, E-W lines + + # size of region of interest in linear degrees + $RA_RANGE = 1.25 * $CHIP_DX * $NCHIP_X * $PLATE_SCALE / 3600 + $DEC_RANGE = 1.25 * $CHIP_DY * $NCHIP_Y * $PLATE_SCALE / 3600 + + $RA_MIN = $RA_CENTER - 0.5*$RA_RANGE/dcos($DEC_CENTER) + $RA_MAX = $RA_CENTER + 0.5*$RA_RANGE/dcos($DEC_CENTER) + $DEC_MIN = $DEC_CENTER - 0.5*$DEC_RANGE + $DEC_MAX = $DEC_CENTER + 0.5*$DEC_RANGE + + create tmp 0 $1 + set stars_ra = $RA_CENTER + $RA_RANGE * (rnd(tmp) - 0.5) / dcos ($DEC_CENTER) + set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) + set stars_mag = 10.0 + 10.0 * rnd(tmp) + set stars_dmag = 0.02 + 0.1 * rnd(tmp) +end + +macro plot.test.files + if ($0 != 2) + echo "USAGE: plot.test.files (N)" + break + end + + clear -s + resize 2000 1200 + + section a0 0.00 0.50 0.25 0.50 + section a1 0.25 0.50 0.25 0.50 + section a2 0.50 0.50 0.25 0.50 + section a3 0.75 0.50 0.25 0.50 + + section a4 0.00 0.00 0.25 0.50 + section a5 0.25 0.00 0.25 0.50 + section a6 0.50 0.00 0.25 0.50 + section a7 0.75 0.00 0.25 0.50 + + label -fn helvetica 12 + $BSTYLE = -tickpad 0.2 +xpad 0.2 +ypad 0.2 -ypad 5 + + for i 0 8 + sprintf name tgrp.%05d.%02d.dat $i $1 + data $name + exec ls -l --full-time $name + read Msys 4 Mrel 5 Mcal 7 Mmos 8 Mgrp 9 airmass 11 dMag 12 + + section a$i + lim Msys dMag; box $BSTYLE; plot Msys dMag -pt cir -op 0.2 -c blue + end +end Index: /branches/eam_branches/relphot.20210521/test/relphot.reference.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/relphot.reference.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/relphot.reference.dvo (revision 41602) @@ -0,0 +1,510 @@ +# -*-sh-*- + +$TAP_BREAK = 0 +$TAP_VERBOSE = 1 +input tap.dvo +input ptolemy.dvo + +# set globals +if (not($?PLOT)) set PLOT = 0 + +$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets +$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact) +$DO_2MASS = 1 +$DO_SYNTH = 1 +$DO_TYCHO = 1 +$NC_CLOUDS = 0.20 +$UC_CLOUDS = 0.02 + +## full test set: generate a set of test files, insert into a catdir, check raw catalog, run relphot, check results +macro go + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + dirname $1 -var rootdir + mkdir $rootdir + + break -auto off + exec rm -rf $catdir + exec rm -rf $fileroot* + break -auto on + + # define a set of photometric (ubercal) and non-photometric exposure times + init + + # generate the common list of stars (coords & mags) for all exposures + mkstars 1000 + + catdir $catdir + + tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3} + + # XXX first, run the analysis with only ubercal data (photometric data) + # we should be able to recover the nightly zero points fitting only tgroups + + # generate the basic images and check they were correctly ingested by dvo + for i 0 mjd_uc[] + sprintf filerootout "%s.uc.%02d" $fileroot $i + mkexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER raw + end + + for i 0 mjd_nc[] + sprintf filerootout "%s.nc.%02d" $fileroot $i + mkexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER raw + end + + mkreference $FILTER $catdir 0.1 + + break + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -tgroup-fit-airmass -tgroups tgroups.dat -images g -v -region $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + break + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end + for i 0 mjd_nc[] + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $FILTER relphot_nc + end + tapDONE +end + +# set various global variables +macro init + + # generates ptolemy.rc, dvo.photcodes, $zpt_nominal:g,r $klam_nominal:g,r + mkptolemy.rc + + $RA_CENTER = 10.0 + $DEC_CENTER = 20.0 + $PLATE_SCALE = 0.25 + $NCHIP_X = 2 + $NCHIP_Y = 2 + $CHIP_DX = 1000 + $CHIP_DY = 1000 + $DEFECT_FRAC = 0.00 + + # we have two sets of images: ubercal (photometric) and not-ubercaled + + # ********* UBERCAL IMAGES *********** + + # ubercal nights each have a distinct zero point and slope: + vlist night_uc 55000.0 55010.0 55030.0 55060.0 + vlist dz_nt_uc +0.0000 +0.0300 -0.0200 -0.0400 +# vlist dk_nt_uc +0.0000 +0.0005 -0.0005 -0.0002 + vlist dk_nt_uc +0.0000 +0.0200 -0.0200 +0.0300 + + write tgroups.dat night_uc + + $EXPTIME = 10.0 + $FILTER = g + $ZPT_OFF:$FILTER = NAN + + delete -q mjd_uc seq_uc zpt_uc klam_uc secz_uc exptime_uc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_uc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = $klam_nominal:$FILTER + dk_nt_uc + set exp_tmp = zero(night_uc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + dz_nt_uc + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) + + set seq_tmp = ramp(night_uc) + set mjd_tmp = night_uc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_uc + concat secz_tmp secz_uc + concat zpt_tmp zpt_uc + concat seq_tmp seq_uc + concat mjd_tmp mjd_uc + + concat exp_tmp exptime_uc + end + + # ********* NON-UBERCAL IMAGES *********** + + # non-ubercal nights each have a random zero points and fixed slope (klam_nominal) + # non-ubercal nights must not be the same set as the ubercal nights above + vlist night_nc 55005.0 55015.0 55035.0 55065.0 + + delete -q mjd_nc seq_nc zpt_nc klam_nc secz_nc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_nc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = zero(night_nc) + $klam_nominal:$FILTER + set exp_tmp = zero(night_nc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) - $NC_CLOUDS*(0.1 + rnd(secz_tmp)) + # observed zero points for nc nights are 0.1 - 1.1 lower than the ubercal nights + + set seq_tmp = ramp(night_nc) + set mjd_tmp = night_nc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_nc + concat secz_tmp secz_nc + concat zpt_tmp zpt_nc + concat seq_tmp seq_nc + concat mjd_tmp mjd_nc + + concat exp_tmp exptime_nc + end +end + +macro try.relphot + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end +end + +macro ckexposure + if ($0 != 8) + echo "ckexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (mode)" + echo " mode == raw or corr" + break + end + + local CATDIR MJD_IMAGE ZPT_REAL ZPT_NOMINAL EXPTIME FILTER AIRMASS + + $CATDIR = $1 + $MJD_IMAGE = $2 + $ZPT_REAL = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $FUNC_MODE = $7 + + # XXX need a function to extract the nominal zpt for a given filter / photcode from the db + $ZPT_NOMINAL = $zpt_nominal:$FILTER + $KLAM_NOMINAL = $klam_nominal:$FILTER + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + catdir $CATDIR + + # assume we still have stars_ra, stars_dec, stars_mag in hand + if ($PLOT) + dev -n 0 + region $RA_CENTER $DEC_CENTER 0.2 + images + pmeasure -all -m 15 20 + end + + # RA_MIN, etc are defined by mkstars + skyregion $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX + + ## calculate the average zero point offset + if (isnan($ZPT_OFF:$FILTER)) + avextract ra dec $FILTER + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + reindex stars_tru_m = stars_mag using index1 + set dm_tru = $FILTER - stars_tru_m + vstat -q dm_tru + $ZPT_OFF:$FILTER = $MEDIAN + end + + mextract ra dec mag xccd yccd dbflags photflags time mag:sys mag:rel mag:inst mag:ave where (abs(time - $MJD_IMAGE) < 0.0001) + + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + + reindex stars_ra_m = stars_ra using index1 + reindex stars_dec_m = stars_dec using index1 + reindex stars_mag_m = stars_mag using index1 + set dr = 3600*(ra - stars_ra_m) + set dd = 3600*(dec - stars_dec_m) + set dm = mag - stars_mag_m + + set good_photom = (not(photflags & 0x02)) + foreach field dm xccd yccd + subset $field\_good = $field if good_photom + end + + if ($PLOT) + dev -n 1 + lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002) + dev -n 2 + lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002) + dev -n 3 + lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002) + echo "red points are known bad photometry" + end + + $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0) + + if ("$FUNC_MODE" == "raw") + set dm_adjust = dm_good - $ZPT_NOMINAL + $ZPT_REAL_NORM + + vstat -q dm_adjust + sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f): %7.4f %7.4f : %6.3f" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + tapOK {abs($MEAN) < 0.005} "RAW : $line" + return + end + + if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc")) + + ## relphot does not set an absolute zero point, so we need to find that and subtract + set dm_adjust = dm_good - $ZPT_OFF:$FILTER + + vstat -q dm_adjust + sprintf line "relphot ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f) : %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + + if ("$FUNC_MODE" == "relphot_uc") + tapOK {abs($MEAN) < 0.001} "MEAN : $line" + tapOK {abs($SIGMA) < 0.001} "SIGMA: $line" + end + if ("$FUNC_MODE" == "relphot_nc") + # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias + tapOK {abs($MEAN) < 0.020} "MEAN : $line" + tapOK {abs($SIGMA) < 0.020} "SIGMA: $line" + end + + if ($PLOT) + cursor 1 + if ("$KEY" == "Escape") break + end + return + end + + echo "ERROR: unknown mode for ckexposure : $FUNC_MODE" +end + +macro mkexposure + if ($0 != 10) + echo "mkexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (fileroot) (ra) (dec)" + break + end + + local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd + + $CATDIR = $1 + $MJD = $2 + $ZPT = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $ROOT = $7 + $RAo = $8 + $DECo = $9 + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + # XX ctimes -ref $MJD -var datetime + # XX substr $datetime 0 10 date + # XX substr $datetime 11 8 time + + for ix 0 $NCHIP_X + for iy 0 $NCHIP_Y + $dx = $CHIP_DX * $PLATE_SCALE * ($ix - 0.5*$NCHIP_X + 0.5) + $dy = $CHIP_DY * $PLATE_SCALE * ($iy - 0.5*$NCHIP_Y + 0.5) + $ra = $RAo - $dx / 3600.0 / dcos($DECo) + $dec = $DECo - $dy / 3600.0 + # echo $ra $dec $dx $dy + mkinput $ROOT.$ix.$iy.in.txt $ra $dec $ZPT + + # ra,dec is the center of this chip + local options + $options = -mjd $MJD + $options = $options -radec $ra $dec + + #$options = $options -type PS1_V2 + #$options = $options -type PS1_V3 + #$options = $options -type PS1_V4 + $options = $options -type PS1_V5 + + $options = $options -coords + $options = $options -photcode GPC1.$FILTER.XY$ix\$iy + $options = $options -no-noise + $options = $options -flags + $options = $options -bad-psfqf-frac $DEFECT_FRAC + $options = $options -size $CHIP_DX $CHIP_DY + $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} + $options = $options -airmass $AIRMASS + $options = $options -exptime $EXPTIME + + tapEXEC mkcmf $ROOT.$ix.$iy.in.txt $ROOT.$ix.$iy.cmf $options + + # the fake images have inconsistent ra,dec and airmass,sidtime values + tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass + end + end +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 5) + echo "mkinput (filename) (ra) (dec) (zpt)" + break + end + + # chip coordinate of the stars + # ra,dec is at the chip center, which corresponds to pixel (CHIP_DX,CHIP_DY)/2 + set stars_X = (stars_ra - $2) * 3600.0 * dcos($3) / $PLATE_SCALE + $CHIP_DX*0.5 + set stars_Y = (stars_dec - $3) * 3600.0 / $PLATE_SCALE + $CHIP_DY*0.5 + set stars_M = (stars_mag - $4) + + set valid = (stars_X > 0) && (stars_X < $CHIP_DX) && (stars_Y > 0) && (stars_Y < $CHIP_DX) + subset stars_x = stars_X if valid + subset stars_y = stars_Y if valid + subset stars_m = stars_M if valid + subset stars_r = stars_ra if valid + subset stars_d = stars_dec if valid + subset stars_mr = stars_mag if valid + + # if we want to modify the observed magnitude based on the camera locations, add code here: + set stars_mi = stars_m + + # create bad detections / detections to ignore or avoid, save to a file? + + # create 3 types of things to ignore / avoid + # * badPhot (deviate the photometry) + # * badFlag (set a bad flag bit) + # * poorFlag (set a bad flag bit) + set badPhot = rnd(stars_r) < $DEFECT_FRAC + set badFlag = rnd(stars_r) < $DEFECT_FRAC + set poorFlag = rnd(stars_r) < $DEFECT_FRAC + + # create deviant photometry (1%) + # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated. This is a simple way to pass the info + # note that outlier rejection only occurs for Nmeas > 5 + set stars_mi = stars_mi + 0.2*badPhot + set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot + + # bad flags that we sould respect: + # PM_SOURCE_MODE_FAIL = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero) + # PM_SOURCE_MODE_SATSTAR = 0x00000080, ///< Source model peak is above saturation + # PM_SOURCE_MODE_BADPSF = 0x00000400, ///< Failed to get good estimate of object's PSF + # PM_SOURCE_MODE_DEFECT = 0x00000800, ///< Source is thought to be a defect + # PM_SOURCE_MODE_SATURATED = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail) + # PM_SOURCE_MODE_CR_LIMIT = 0x00002000, ///< Source has crNsigma above limit + # PM_SOURCE_MODE_MOMENTS_FAILURE = 0x00008000, ///< could not measure the moments + # PM_SOURCE_MODE_SKY_FAILURE = 0x00010000, ///< could not measure the local sky + # PM_SOURCE_MODE_SKYVAR_FAILURE = 0x00020000, ///< could not measure the local sky variance + # PM_SOURCE_MODE_SIZE_SKIPPED = 0x10000000, ///< size could not be determined + + # poor flags that we should respect: + # PM_SOURCE_MODE_POOR = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?) + # PM_SOURCE_MODE_PAIR = 0x00000020, ///< Source fitted with a double psf + # PM_SOURCE_MODE_BLEND = 0x00000100, ///< Source is a blend with other sources + # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N + # PM_SOURCE_MODE_BLEND_FIT = 0x00400000, ///< source was fitted as a blend + # PM_SOURCE_MODE_ON_SPIKE = 0x20000000, ///< peak lands on diffraction spike + # PM_SOURCE_MODE_ON_GHOST = 0x40000000, ///< peak lands on ghost or glint + # PM_SOURCE_MODE_OFF_CHIP = 0x80000000, ///< peak lands off edge of chip + + write -f "%10.6f %10.6f %8.4f %6.1f %6.1f %8.4f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl +end + +# generate a set of stars with raw RA, DEC, MAG values to use for all of the output cmf files +macro mkstars + if ($0 != 2) + echo "USAGE: mkstars (Nstars)" + break + end + + local RA_RANGE DEC_RANGE + + # the images are oriented along N-S, E-W lines + + # size of region of interest in linear degrees + $RA_RANGE = 1.25 * $CHIP_DX * $NCHIP_X * $PLATE_SCALE / 3600 + $DEC_RANGE = 1.25 * $CHIP_DY * $NCHIP_Y * $PLATE_SCALE / 3600 + + $RA_MIN = $RA_CENTER - 0.5*$RA_RANGE/dcos($DEC_CENTER) + $RA_MAX = $RA_CENTER + 0.5*$RA_RANGE/dcos($DEC_CENTER) + $DEC_MIN = $DEC_CENTER - 0.5*$DEC_RANGE + $DEC_MAX = $DEC_CENTER + 0.5*$DEC_RANGE + + create tmp 0 $1 + set stars_ra = $RA_CENTER + $RA_RANGE * (rnd(tmp) - 0.5) / dcos ($DEC_CENTER) + set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) + set stars_mag = 10.0 + 10.0 * rnd(tmp) + set stars_dmag = 0.02 + 0.1 * rnd(tmp) +end + +macro mkreference + if ($0 != 4) + echo "mkreference (filter) (catdir) (fraction)" + break + end + + local catdir filter fraction + $filter = $1 + $catdir = $2 + $fraction = $3 + + set tmpseq = rnd(stars_ra) + + subset stars_ref_ra = stars_ra where (tmpseq < $fraction) + subset stars_ref_dec = stars_dec where (tmpseq < $fraction) + subset stars_ref_mag = stars_mag where (tmpseq < $fraction) + subset stars_ref_dmag = stars_dmag where (tmpseq < $fraction) + set stars_ref_mag = stars_ref_mag + 0.1 + + write reference.dat stars_ref_ra stars_ref_dec stars_ref_mag stars_ref_dmag + exec addstar -D CATDIR $catdir -photcode SYNTH.$filter reference.dat +end + +macro plot.test.files + if ($0 != 2) + echo "USAGE: plot.test.files (N)" + break + end + + clear -s + resize 2000 1200 + + section a0 0.00 0.50 0.25 0.50 + section a1 0.25 0.50 0.25 0.50 + section a2 0.50 0.50 0.25 0.50 + section a3 0.75 0.50 0.25 0.50 + + section a4 0.00 0.00 0.25 0.50 + section a5 0.25 0.00 0.25 0.50 + section a6 0.50 0.00 0.25 0.50 + section a7 0.75 0.00 0.25 0.50 + + label -fn helvetica 12 + $BSTYLE = -tickpad 0.2 +xpad 0.2 +ypad 0.2 -ypad 5 + + for i 0 8 + sprintf name tgrp.%05d.%02d.dat $i $1 + data $name + exec ls -l --full-time $name + read Msys 4 Mrel 5 Mcal 7 Mmos 8 Mgrp 9 airmass 11 dMag 12 + + section a$i + lim Msys dMag; box $BSTYLE; plot Msys dMag -pt cir -op 0.2 -c blue + end +end Index: /branches/eam_branches/relphot.20210521/test/relphot.tgroups.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/relphot.tgroups.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/relphot.tgroups.dvo (revision 41602) @@ -0,0 +1,506 @@ +# -*-sh-*- + +$TAP_BREAK = 0 +$TAP_VERBOSE = 1 +input tap.dvo +input ptolemy.dvo + +# set globals +if (not($?PLOT)) set PLOT = 0 + +$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets +$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact) +$DO_2MASS = 1 +$DO_SYNTH = 1 +$DO_TYCHO = 1 +$NC_CLOUDS = 0.20 +$UC_CLOUDS = 0.02 + +## full test set: generate a set of test files, insert into a catdir, check raw catalog, run relphot, check results +macro go + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + dirname $1 -var rootdir + mkdir $rootdir + + break -auto off + exec rm -rf $catdir + exec rm -rf $fileroot* + break -auto on + + # define a set of photometric (ubercal) and non-photometric exposure times + init + + # generate the common list of stars (coords & mags) for all exposures + mkstars 1000 + + catdir $catdir + + tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3} + + # XXX first, run the analysis with only ubercal data (photometric data) + # we should be able to recover the nightly zero points fitting only tgroups + + # generate the basic images and check they were correctly ingested by dvo + for i 0 mjd_uc[] + sprintf filerootout "%s.uc.%02d" $fileroot $i + mkexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] secz_uc[$i] $FILTER raw + end + + for i 0 mjd_nc[] + sprintf filerootout "%s.nc.%02d" $fileroot $i + mkexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER $filerootout $RA_CENTER $DEC_CENTER + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] secz_nc[$i] $FILTER raw + end + + break + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -tgroup-fit-airmass -tgroups tgroups.dat -images g -v -region $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + break + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end + for i 0 mjd_nc[] + ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $FILTER relphot_nc + end + tapDONE +end + +# set various global variables +macro init + + # generates ptolemy.rc, dvo.photcodes, $zpt_nominal:g,r $klam_nominal:g,r + mkptolemy.rc + + $RA_CENTER = 10.0 + $DEC_CENTER = 20.0 + $PLATE_SCALE = 0.25 + $NCHIP_X = 1 + $NCHIP_Y = 1 + $CHIP_DX = 1000 + $CHIP_DY = 1000 + $DEFECT_FRAC = 0.00 + + # we have two sets of images: ubercal (photometric) and not-ubercaled + + # ********* UBERCAL IMAGES *********** + + # ubercal nights each have a distinct zero point and slope: + vlist night_uc 55000.0 55010.0 55030.0 55060.0 + vlist dz_nt_uc +0.0000 +0.0300 -0.0200 -0.0400 +# vlist dk_nt_uc +0.0000 +0.0005 -0.0005 -0.0002 + vlist dk_nt_uc +0.0000 +0.0200 -0.0200 +0.0300 + + write tgroups.dat night_uc + + $EXPTIME = 10.0 + $FILTER = g + $ZPT_OFF:$FILTER = NAN + + delete -q mjd_uc seq_uc zpt_uc klam_uc secz_uc exptime_uc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_uc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = $klam_nominal:$FILTER + dk_nt_uc + set exp_tmp = zero(night_uc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + dz_nt_uc + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) + + set seq_tmp = ramp(night_uc) + set mjd_tmp = night_uc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_uc + concat secz_tmp secz_uc + concat zpt_tmp zpt_uc + concat seq_tmp seq_uc + concat mjd_tmp mjd_uc + + concat exp_tmp exptime_uc + end + + # ********* NON-UBERCAL IMAGES *********** + + # non-ubercal nights each have a random zero points and fixed slope (klam_nominal) + # non-ubercal nights must not be the same set as the ubercal nights above + vlist night_nc 55005.0 55015.0 55035.0 55065.0 + + delete -q mjd_nc seq_nc zpt_nc klam_nc secz_nc + + # generate 5 exposures for each night + for i 0 5 + set secz_tmp = rnd(night_nc) + 1.0; # airmass in range 1.0 - 2.0 + set klam_tmp = zero(night_nc) + $klam_nominal:$FILTER + set exp_tmp = zero(night_nc) + $EXPTIME + set zpt_tmp = $zpt_nominal:$FILTER + 2.5*log(exp_tmp) + klam_tmp*(secz_tmp - 1.0) - $NC_CLOUDS*(0.1 + rnd(secz_tmp)) + # observed zero points for nc nights are 0.1 - 1.1 lower than the ubercal nights + + set seq_tmp = ramp(night_nc) + set mjd_tmp = night_nc + 0.33 + 0.01*$i; # exposures are 14 minutes apart starting at 10pm HST + + concat klam_tmp klam_nc + concat secz_tmp secz_nc + concat zpt_tmp zpt_nc + concat seq_tmp seq_nc + concat mjd_tmp mjd_nc + + concat exp_tmp exptime_nc + end +end + +macro try.relphot + if ($0 != 3) + echo "go (fileroot) (catdir)" + break + end + + local i fileroot rootdir catdir + + $fileroot = $1 + $catdir = $2 + + # run relphot on the db and check that the images now match the expected values + tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -statmode WT_MEAN -cloud-limit 0.5 -update -nloop 12 + + # reset the per-filter ZPT_OFF:$filter values + $ZPT_OFF:$FILTER = NAN + + for i 0 mjd_uc[] + # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $FILTER relphot_uc + end +end + +macro ckexposure + if ($0 != 8) + echo "ckexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (mode)" + echo " mode == raw or corr" + break + end + + local CATDIR MJD_IMAGE ZPT_REAL ZPT_NOMINAL EXPTIME FILTER AIRMASS + + $CATDIR = $1 + $MJD_IMAGE = $2 + $ZPT_REAL = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $FUNC_MODE = $7 + + # XXX need a function to extract the nominal zpt for a given filter / photcode from the db + $ZPT_NOMINAL = $zpt_nominal:$FILTER + $KLAM_NOMINAL = $klam_nominal:$FILTER + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + catdir $CATDIR + + # assume we still have stars_ra, stars_dec, stars_mag in hand + if ($PLOT) + dev -n 0 + region $RA_CENTER $DEC_CENTER 0.2 + images + pmeasure -all -m 15 20 + end + + # RA_MIN, etc are defined by mkstars + skyregion $RA_MIN $RA_MAX $DEC_MIN $DEC_MAX + + ## calculate the average zero point offset + if (isnan($ZPT_OFF:$FILTER)) + avextract ra dec $FILTER + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + reindex stars_tru_m = stars_mag using index1 + set dm_tru = $FILTER - stars_tru_m + vstat -q dm_tru + $ZPT_OFF:$FILTER = $MEDIAN + end + + mextract ra dec mag xccd yccd dbflags photflags time mag:sys mag:rel mag:inst mag:ave where (abs(time - $MJD_IMAGE) < 0.0001) + + match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2 + + reindex stars_ra_m = stars_ra using index1 + reindex stars_dec_m = stars_dec using index1 + reindex stars_mag_m = stars_mag using index1 + set dr = 3600*(ra - stars_ra_m) + set dd = 3600*(dec - stars_dec_m) + set dm = mag - stars_mag_m + + set good_photom = (not(photflags & 0x02)) + foreach field dm xccd yccd + subset $field\_good = $field if good_photom + end + + if ($PLOT) + dev -n 1 + lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002) + dev -n 2 + lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002) + dev -n 3 + lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002) + echo "red points are known bad photometry" + end + + $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0) + + if ("$FUNC_MODE" == "raw") + set dm_adjust = dm_good - $ZPT_NOMINAL + $ZPT_REAL_NORM + + vstat -q dm_adjust + sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f): %7.4f %7.4f : %6.3f" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + tapOK {abs($MEAN) < 0.005} "RAW : $line" + return + end + + if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc")) + + ## relphot does not set an absolute zero point, so we need to find that and subtract + set dm_adjust = dm_good - $ZPT_OFF:$FILTER + + vstat -q dm_adjust + sprintf line "relphot ZP %8.2f $FILTER %6.3f %6.3f %6.3f (%7.4f) : %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL {$ZPT_REAL_NORM - $ZPT_NOMINAL} $MEAN $SIGMA dm_adjust[] + + if ("$FUNC_MODE" == "relphot_uc") + tapOK {abs($MEAN) < 0.001} "MEAN : $line" + tapOK {abs($SIGMA) < 0.001} "SIGMA: $line" + end + if ("$FUNC_MODE" == "relphot_nc") + # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias + tapOK {abs($MEAN) < 0.020} "MEAN : $line" + tapOK {abs($SIGMA) < 0.020} "SIGMA: $line" + end + + if ($PLOT) + cursor 1 + if ("$KEY" == "Escape") break + end + return + end + + echo "ERROR: unknown mode for ckexposure : $FUNC_MODE" +end + +macro mkexposure + if ($0 != 10) + echo "mkexposure (catdir) (mjd) (zpt) (exptime) (airmass) (filter) (fileroot) (ra) (dec)" + break + end + + local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd + + $CATDIR = $1 + $MJD = $2 + $ZPT = $3 + $EXPTIME = $4 + $AIRMASS = $5 + $FILTER = $6 + $ROOT = $7 + $RAo = $8 + $DECo = $9 + + $TIMEFORMAT = mjd + $TIMEREF = 0.0 + + # XX ctimes -ref $MJD -var datetime + # XX substr $datetime 0 10 date + # XX substr $datetime 11 8 time + + for ix 0 $NCHIP_X + for iy 0 $NCHIP_Y + $dx = $CHIP_DX * $PLATE_SCALE * ($ix - 0.5*$NCHIP_X + 0.5) + $dy = $CHIP_DY * $PLATE_SCALE * ($iy - 0.5*$NCHIP_Y + 0.5) + $ra = $RAo - $dx / 3600.0 / dcos($DECo) + $dec = $DECo - $dy / 3600.0 + # echo $ra $dec $dx $dy + mkinput $ROOT.$ix.$iy.in.txt $ra $dec $ZPT + + # ra,dec is the center of this chip + local options + $options = -mjd $MJD + $options = $options -radec $ra $dec + + #$options = $options -type PS1_V2 + #$options = $options -type PS1_V3 + #$options = $options -type PS1_V4 + $options = $options -type PS1_V5 + + $options = $options -coords + $options = $options -photcode GPC1.$FILTER.XY$ix\$iy + $options = $options -no-noise + $options = $options -flags + $options = $options -bad-psfqf-frac $DEFECT_FRAC + $options = $options -size $CHIP_DX $CHIP_DY + $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} + $options = $options -airmass $AIRMASS + $options = $options -exptime $EXPTIME + + tapEXEC mkcmf $ROOT.$ix.$iy.in.txt $ROOT.$ix.$iy.cmf $options + + # the fake images have inconsistent ra,dec and airmass,sidtime values + tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass + end + end +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 5) + echo "mkinput (filename) (ra) (dec) (zpt)" + break + end + + # chip coordinate of the stars + # ra,dec is at the chip center, which corresponds to pixel (CHIP_DX,CHIP_DY)/2 + set stars_X = (stars_ra - $2) * 3600.0 * dcos($3) / $PLATE_SCALE + $CHIP_DX*0.5 + set stars_Y = (stars_dec - $3) * 3600.0 / $PLATE_SCALE + $CHIP_DY*0.5 + set stars_M = (stars_mag - $4) + + set valid = (stars_X > 0) && (stars_X < $CHIP_DX) && (stars_Y > 0) && (stars_Y < $CHIP_DX) + subset stars_x = stars_X if valid + subset stars_y = stars_Y if valid + subset stars_m = stars_M if valid + subset stars_r = stars_ra if valid + subset stars_d = stars_dec if valid + subset stars_mr = stars_mag if valid + + # if we want to modify the observed magnitude based on the camera locations, add code here: + set stars_mi = stars_m + + # create bad detections / detections to ignore or avoid, save to a file? + + # create 3 types of things to ignore / avoid + # * badPhot (deviate the photometry) + # * badFlag (set a bad flag bit) + # * poorFlag (set a bad flag bit) + set badPhot = rnd(stars_r) < $DEFECT_FRAC + set badFlag = rnd(stars_r) < $DEFECT_FRAC + set poorFlag = rnd(stars_r) < $DEFECT_FRAC + + # create deviant photometry (1%) + # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated. This is a simple way to pass the info + # note that outlier rejection only occurs for Nmeas > 5 + set stars_mi = stars_mi + 0.2*badPhot + set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot + + # bad flags that we sould respect: + # PM_SOURCE_MODE_FAIL = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero) + # PM_SOURCE_MODE_SATSTAR = 0x00000080, ///< Source model peak is above saturation + # PM_SOURCE_MODE_BADPSF = 0x00000400, ///< Failed to get good estimate of object's PSF + # PM_SOURCE_MODE_DEFECT = 0x00000800, ///< Source is thought to be a defect + # PM_SOURCE_MODE_SATURATED = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail) + # PM_SOURCE_MODE_CR_LIMIT = 0x00002000, ///< Source has crNsigma above limit + # PM_SOURCE_MODE_MOMENTS_FAILURE = 0x00008000, ///< could not measure the moments + # PM_SOURCE_MODE_SKY_FAILURE = 0x00010000, ///< could not measure the local sky + # PM_SOURCE_MODE_SKYVAR_FAILURE = 0x00020000, ///< could not measure the local sky variance + # PM_SOURCE_MODE_SIZE_SKIPPED = 0x10000000, ///< size could not be determined + + # poor flags that we should respect: + # PM_SOURCE_MODE_POOR = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?) + # PM_SOURCE_MODE_PAIR = 0x00000020, ///< Source fitted with a double psf + # PM_SOURCE_MODE_BLEND = 0x00000100, ///< Source is a blend with other sources + # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N + # PM_SOURCE_MODE_BLEND_FIT = 0x00400000, ///< source was fitted as a blend + # PM_SOURCE_MODE_ON_SPIKE = 0x20000000, ///< peak lands on diffraction spike + # PM_SOURCE_MODE_ON_GHOST = 0x40000000, ///< peak lands on ghost or glint + # PM_SOURCE_MODE_OFF_CHIP = 0x80000000, ///< peak lands off edge of chip + + write -f "%10.6f %10.6f %8.4f %6.1f %6.1f %8.4f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl +end + +# generate a set of stars with raw RA, DEC, MAG values to use for all of the output cmf files +macro mkstars + if ($0 != 2) + echo "USAGE: mkstars (Nstars)" + break + end + + local RA_RANGE DEC_RANGE + + # the images are oriented along N-S, E-W lines + + # size of region of interest in linear degrees + $RA_RANGE = 1.25 * $CHIP_DX * $NCHIP_X * $PLATE_SCALE / 3600 + $DEC_RANGE = 1.25 * $CHIP_DY * $NCHIP_Y * $PLATE_SCALE / 3600 + + $RA_MIN = $RA_CENTER - 0.5*$RA_RANGE/dcos($DEC_CENTER) + $RA_MAX = $RA_CENTER + 0.5*$RA_RANGE/dcos($DEC_CENTER) + $DEC_MIN = $DEC_CENTER - 0.5*$DEC_RANGE + $DEC_MAX = $DEC_CENTER + 0.5*$DEC_RANGE + + create tmp 0 $1 + set stars_ra = $RA_CENTER + $RA_RANGE * (rnd(tmp) - 0.5) / dcos ($DEC_CENTER) + set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) + set stars_mag = 10.0 + 10.0 * rnd(tmp) + set stars_dmag = 0.02 + 0.1 * rnd(tmp) +end + +macro plot.test.files + if ($0 != 2) + echo "USAGE: plot.test.files (N)" + break + end + + clear -s + resize 2000 1200 + + section a0 0.00 0.50 0.25 0.50 + section a1 0.25 0.50 0.25 0.50 + section a2 0.50 0.50 0.25 0.50 + section a3 0.75 0.50 0.25 0.50 + + section a4 0.00 0.00 0.25 0.50 + section a5 0.25 0.00 0.25 0.50 + section a6 0.50 0.00 0.25 0.50 + section a7 0.75 0.00 0.25 0.50 + + label -fn helvetica 12 + $BSTYLE = -tickpad 0.2 +xpad 0.2 +ypad 0.2 -ypad 5 + + for i 0 8 + sprintf name tgrp.%05d.%02d.dat $i $1 + data $name + exec ls -l --full-time $name + read Msys 4 Mrel 5 Mcal 7 Mmos 8 Mgrp 9 airmass 11 dMag 12 + + section a$i + lim Msys dMag; box $BSTYLE; plot Msys dMag -pt cir -op 0.2 -c blue + end +end + +macro plot.tcat.data + if ($0 != 2) + echo "USAGE: plot.tcat.data (Nlast)" + break + end + + $BSTYLE = -tickpad 0.2 +xpad 0.2 +ypad 0.2 -ypad 5 + + for i 0 $1 + sprintf name tcat.%02d.dat $i + data $name + dev -n $i + resize 900 500 + clear; + box $BSTYLE + read Msys 4 Mrel 5 Mcal 7 Mmos 8 Mgrp 9 airmass 11 dMag 12; + lim Msys dMag; plot Msys dMag -pt cir -op 0.2 -c blue + echo $name + end +end Index: /branches/eam_branches/relphot.20210521/test/tap.dvo =================================================================== --- /branches/eam_branches/relphot.20210521/test/tap.dvo (revision 41602) +++ /branches/eam_branches/relphot.20210521/test/tap.dvo (revision 41602) @@ -0,0 +1,107 @@ +# -*-sh-*- + +if (not($?TAP_BREAK)) set TAP_BREAK = 0 +if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 0 + +macro tapEXEC + if ($0 < 3) + echo "USAGE: tapEXEC (command)" + break + end + + local i cmd result + + $cmd = "" + for i 1 $0 + $cmd = $cmd $$i + end + + if ($TAP_VERBOSE >= 2) echo $cmd + + break -auto off + exec $cmd >& tap.log + $result = $STATUS + break -auto on + + if (not($result)) + exec cat tap.log + echo "not ok : $cmd" + $TAP_NFAIL ++ + $TAP_LAST = 0 + if ($TAP_BREAK) + break + end + else + if ($TAP_VERBOSE >= 1) echo "ok : $cmd" + $TAP_LAST = 1 + end + $TAP_NDONE ++ +end + +macro tapOK + if ($0 != 3) + echo "USAGE: tapOK (condition) (message)" + break + end + + if ($1) + if ($TAP_VERBOSE >= 1) + echo "ok : $2" + end + $TAP_LAST = 1 + else + echo "not ok : $2" + $TAP_NFAIL ++ + $TAP_LAST = 0 + if ($TAP_BREAK) + break + end + end + $TAP_NDONE ++ +end + +macro tapPLAN + if ($0 != 2) + echo "USAGE: tapPLAN (Ntests)" + break + end + + $TAP_NTEST = $1 + $TAP_NFAIL = 0 + $TAP_NSKIP = 0 + $TAP_NDONE = 0 + if (not($?TAP_BREAK)) set TAP_BREAK = 0 +end + +macro tapSKIP + if ($0 != 2) + echo "USAGE: tapSKIP (nskip)" + break + end + + $TAP_NSKIP += $1 + $TAP_NDONE += $1 +end + +macro tapDONE + if ($0 != 1) + echo "USAGE: tapDONE" + break + end + + if ($TAP_NDONE != $TAP_NTEST) + echo "planned tests ($TAP_NTEST) not equal to done tests ($TAP_NDONE)" + end + + if ($TAP_NFAIL) + echo "failed $TAP_NFAIL of $TAP_NDONE" + end + + if ($TAP_NSKIP) + echo "skipped $TAP_NSKIP of $TAP_NDONE" + end + + if ($TAP_NFAIL + $TAP_NSKIP == 0) + echo "passed $TAP_NDONE tests" + end +end