Index: /trunk/ippconfig/gpc1/psastro.config =================================================================== --- /trunk/ippconfig/gpc1/psastro.config (revision 37551) +++ /trunk/ippconfig/gpc1/psastro.config (revision 37552) @@ -398,4 +398,62 @@ END +CNP_ASTRO_PV3 METADATA + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.MAX.ALLOWED.FWHM F32 25 + PSASTRO.CATDIR STR PS1.REF.20140713 + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.GRID.MIN.ANGLE F32 -20 + PSASTRO.GRID.MAX.ANGLE F32 +20 + PSASTRO.GRID.DEL.ANGLE F32 0.5 + PSASTRO.FIELD.PADDING F32 0.5 + ## -- cannot clamp mag max as may be need for astrom + #REFSTAR_CLAMP_MAG_MAX F32 20.0 +END + +MD_ASTRO_PV3 METADATA + ZERO.POINT.BY.EXPOSURE BOOL TRUE + ZERO.POINT.USE.MEAN BOOL TRUE + DVO.GETSTAR.MAX.RHO F32 3000.0 # was globally changed to 10k, way to much and problem for fainter refcat in MD + PSASTRO.GRID.NREF.MAX S32 800 # max stars accepted for grid search from ref catalog (3-500 typically matched, lose some bright end) + PSASTRO.GRID.NRAW.MAX S32 1200 # max stars accepted for grid search from raw image to account for junk also + DVO.GETSTAR.MIN.MAG F32 12.0 # leave similar to default as MD done in marginal weather and can lose 1-3 mags at times + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 + PSASTRO.CATDIR STR PS1.REF.20140713 + ## refcat may be okay astrom but not phot -- so need to be able to further limit by ZP + ## -- concern w/ new refcat >20 extra sources w/ larger merr over-dominating scatter in ZP + REFSTAR_CLAMP_MAG_MAX F32 20.0 + REFSTAR.ZP.MIN.MAG F32 12.0 + REFSTAR.ZP.MAX.MAG F32 21.0 + ## for ZP need extra clamp by filter to avoid poorer photometry scatter on faint end + ## -- cannot restrict on bright end due to possible marginal nights +# PHOTCODE.DATA MULTI UPDATE + PHOTCODE.DATA MULTI + PHOTCODE.DATA METADATA + FILTER STR g + PHOTCODE STR g + REFSTAR.ZP.MAX.MAG F32 20.0 + END + PHOTCODE.DATA METADATA + FILTER STR r + PHOTCODE STR r + REFSTAR.ZP.MAX.MAG F32 20.0 + END + PHOTCODE.DATA METADATA + FILTER STR i + PHOTCODE STR i + REFSTAR.ZP.MAX.MAG F32 20.0 + END + PHOTCODE.DATA METADATA + FILTER STR z + PHOTCODE STR z + REFSTAR.ZP.MAX.MAG F32 19.5 + END + PHOTCODE.DATA METADATA + FILTER STR y + PHOTCODE STR y + REFSTAR.ZP.MAX.MAG F32 18.0 + END +END + TEST_REFCAT METADATA PSASTRO.CATDIR STR /data/ipp064.0/ipp/ippRefs/catdir.refcat.20120524.v0 @@ -527,7 +585,7 @@ ## if defined for MD, needs to have MD name in group so shouldn't be modified except for MD work -## -- ad hoc limit for NREF due to issues fainter than 19 mag in all filters and brighter than 17.0 mag (except y ~16.5) in MD -## -- better solution and other issues need to really be dealt with for PV3 -STATICSKY_MD_DEEPCAL METADATA +## -- was ad hoc limit for NREF~150, rho~6000 due to issues fainter than 19 mag in all filters and brighter than 17.0 mag (except y ~16.5) in MD +## -- for PV3 add refcat min/max mag REFSTAR.ZP.MIN.MAG for ZP range only +STATICSKY_MD_DEEPCAL_PV3 METADATA PSASTRO.CATDIR STR PS1.REF.20140713 REFSTAR_MASK BOOL FALSE @@ -539,5 +597,5 @@ PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels PSASTRO.MAX.NRAW S32 1500 # include more than MAX.NREF to account for extras like false-positives - PSASTRO.MAX.NREF S32 150 # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets + PSASTRO.MAX.NREF S32 500 # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # PSASTRO.MAX.INST.MAG.RAW F32 -5.0 @@ -546,6 +604,6 @@ DVO.GETSTAR.MAX.RHO F32 6000.0 # N/sqdeg, slightly larger than MAX.NREF DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... - DVO.GETSTAR.MIN.MAG F32 17.0 # only used if keyword problem so also using CLAMP_MAG_MIN - REFSTAR_CLAMP_MAG_MIN F32 17.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list. seeing offset in brighter stars + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem so also using CLAMP_MAG_MIN + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list. offset in brighter stars DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates @@ -559,4 +617,40 @@ PSASTRO.MIN.NSTAR S32 10 # edges may not have many and require special treatment - KH.CORRECT.APPLY BOOL FALSE -END + KH.CORRECT.APPLY BOOL FALSE + + REFSTAR.ZP.MIN.MAG F32 15.0 # general refcat zp min/max setting -- need to use photcode below + REFSTAR.ZP.MAX.MAG F32 19.0 + +# PHOTCODE.DATA MULTI UPDATE + PHOTCODE.DATA MULTI + PHOTCODE.DATA METADATA + FILTER STR g + PHOTCODE STR g + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 20.0 # + END + PHOTCODE.DATA METADATA + FILTER STR r + PHOTCODE STR r + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 19.0 # + END + PHOTCODE.DATA METADATA + FILTER STR i + PHOTCODE STR i + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 19.0 # + END + PHOTCODE.DATA METADATA + FILTER STR z + PHOTCODE STR z + REFSTAR.ZP.MIN.MAG F32 16.0 # + REFSTAR.ZP.MAX.MAG F32 18.0 # + END + PHOTCODE.DATA METADATA + FILTER STR y + PHOTCODE STR y + REFSTAR.ZP.MIN.MAG F32 15.0 # + REFSTAR.ZP.MAX.MAG F32 17.0 # + END +END Index: /trunk/ippconfig/recipes/psastro.config =================================================================== --- /trunk/ippconfig/recipes/psastro.config (revision 37551) +++ /trunk/ippconfig/recipes/psastro.config (revision 37552) @@ -174,4 +174,9 @@ REFSTAR_CLAMP_MAG_MAX F32 30.0 +# restrict ref catalog for zero point determination +# -- can be put into photcode groups as well +REFSTAR.ZP.MAX.MAG F32 30.0 +REFSTAR.ZP.MIN.MAG F32 -3.0 + EXTRACT_MAX_MAG F32 -15.0 @@ -276,4 +281,10 @@ END +CNP_ASTRO_PV3 METADATA +END + +MD_ASTRO_PV3 METADATA +END + SYNTH_CAT METADATA END @@ -301,2 +312,4 @@ END +STATICSKY_MD_DEEPCAL_PV3 METADATA +END Index: /trunk/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /trunk/ippconfig/recipes/reductionClasses.mdc (revision 37551) +++ /trunk/ippconfig/recipes/reductionClasses.mdc (revision 37552) @@ -1311,4 +1311,53 @@ STATICSKY_CALIBRATION STR STATICSKY_CAL END + +CNP_PV3 METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR CNP_ASTRO_PV3 + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + +MD_PV3 METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK_1DG + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK_!DG + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR MD_ASTRO_PV3 + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_MD_DEEPCAL_PV3 +END + # random user tests (may not be stable or consistent) Index: /trunk/psastro/src/psastro.h =================================================================== --- /trunk/psastro/src/psastro.h (revision 37551) +++ /trunk/psastro/src/psastro.h (revision 37552) @@ -154,7 +154,8 @@ bool psastroZeroPoint (pmConfig *config); -psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime); +psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime, float refminMag, float refmaxMag); bool psastroZeroPointAnalysis (psMetadata *header, psVector *dMag, float zeropt, psMetadata *recipe); bool psastroZeroPointFromRecipe (float *zeropt, float *exptime, float *ghostMaxMag, pmFPA *fpa, psMetadata *recipe); +bool psastroZeroPointRefMagLimitFromRecipe (float *refminMag, float *refmaxMag, pmFPA *fpa, psMetadata *recipe); psStats *psastroStatsPercentile (psVector *myVector, psMetadata *recipe); Index: /trunk/psastro/src/psastroChooseRefstars.c =================================================================== --- /trunk/psastro/src/psastroChooseRefstars.c (revision 37551) +++ /trunk/psastro/src/psastroChooseRefstars.c (revision 37552) @@ -66,5 +66,5 @@ psF32 clampMagMin = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN"); if (!status) clampMagMin = -5.0; - //MEH test + //MEH adding max option as well -- really should not be hardcoded.. psF32 clampMagMax = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MAX"); if (!status) clampMagMax = 30.0; @@ -117,5 +117,5 @@ goto skip; } - //MEH test + //MEH option for max mag clamp also if (ref->Mag > clampMagMax) { psWarning("Skipping by clamp refstar with faint magnitude %f ra: %f dec: %f\n", Index: /trunk/psastro/src/psastroZeroPoint.c =================================================================== --- /trunk/psastro/src/psastroZeroPoint.c (revision 37551) +++ /trunk/psastro/src/psastroZeroPoint.c (revision 37552) @@ -20,4 +20,5 @@ pmCell *cell = NULL; pmReadout *readout = NULL; + float refminMag,refmaxMag; // select the current recipe @@ -59,6 +60,15 @@ } + // MEH -- RefMagLimit for ZP with photcode option (merr probably better but not an option) + // -- defualt is to use general config value (no hardcode..) + if (!psastroZeroPointRefMagLimitFromRecipe (&refminMag, &refmaxMag, fpa, recipe)) { + psLogMsg ("psastro", PS_LOG_INFO, "failed to load ref min,maxMag data from recipe"); + return false; + } + // if we measure the zero point by exposure, accumulate the dMag values here: psVector *dMag = NULL; + // MEH -- also the number of refcat sources used + int zpt_nref = 0; float fpaZP = 0.0; // Average zero point @@ -81,10 +91,16 @@ // calculate dMag for the matched stars - dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime); - + //dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime); + // MEH -- additional limits should be available for ZP if just robust mean and w/ stack skycal + dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime, refminMag, refmaxMag); if (!byExposure) { // calculate dMag for the matched stars just for this readout (well, chip) psMetadata *header = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER"); psastroZeroPointAnalysis (header, dMag, zeropt, recipe); + // MEH -- per readout N refcat, min/max mag -- added here, not part of median/edge process + zpt_nref += dMag->n; + psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "N refcat sources for zero point", dMag->n); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag); psFree (dMag); dMag = NULL; @@ -95,4 +111,5 @@ numZP++; } + @@ -110,4 +127,6 @@ } psastroZeroPointAnalysis (header, dMag, zeropt, recipe); + //MEH -- all N refcat sources here + zpt_nref = dMag->n; psFree (dMag); dMag = NULL; @@ -141,9 +160,12 @@ // calculate dMag for the matched stars just for this readout (well, chip) psMetadata *header = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER"); - psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_OBS", PS_META_REPLACE, "measured zero point", zptObs); psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_REF", PS_META_REPLACE, "reference zero point", zptRef); psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_ERR", PS_META_REPLACE, "error on zero point", zptErr); psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_OFF", PS_META_REPLACE, "zero point offset", zptOff); + //MEH -- per readout N refcat, MIN/MAX for consistency + psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "N refcat sources for zero point", zpt_nref); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag); } } @@ -151,4 +173,14 @@ } + // MEH add N refcat, min/max mag to primary header + psMetadata *header = psMetadataLookupMetadata (&status, fpa->analysis, "PSASTRO.HEADER"); + if (!header) { + header = psMetadataAlloc (); + psMetadataAddMetadata (fpa->analysis, PS_LIST_TAIL, "PSASTRO.HEADER", PS_META_REPLACE, "psastro header stats", header); + psFree (header); + } + psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "Total N refcat sources for zero point", zpt_nref); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag); + psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag); psMetadataItem *item = psMetadataLookup(fpa->concepts, "FPA.ZP"); @@ -162,5 +194,5 @@ * accumulate the dMag values from this readout */ -psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime) { +psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime, float refminMag, float refmaxMag) { bool status; @@ -191,6 +223,9 @@ // XXX I should be applying the airmass term here or in psastroLoadRefstars // (ie, ref->Mag would be a prediction for a given star. - float value = ref->Mag - raw->Mag - 2.5*log10(exptime); - psVectorAppend (dMag, value); + // MEH -- only append if in allowed ref mag range -- may want merr but not available.. + if ( ref->Mag >= refminMag && ref->Mag <= refmaxMag ){ + float value = ref->Mag - raw->Mag - 2.5*log10(exptime); + psVectorAppend (dMag, value); + } } return dMag; @@ -490,5 +525,9 @@ continue; } - } + //MEH null is a pain.. so only log if set + psLogMsg ("psastro", PS_LOG_INFO, "found GHOST_MAX_MAG %f",*ghostMaxMag); + } + //MEH what zpt is set to + psLogMsg ("psastro", PS_LOG_INFO, "found ZEROPT %f",*zeropt); psFree (iter); return true; @@ -497,2 +536,75 @@ return false; } + + + +bool psastroZeroPointRefMagLimitFromRecipe (float *refminMag, float *refmaxMag, pmFPA *fpa, psMetadata *recipe) { + + bool status; + float tmpmin,tmpmax; + + // MEH -- duplicate psastroZeroPointFromRecipe for ZeroPoint adding RefMag limits with photcode option for extra restrictions + // must have non-photcode dependent setting -- use as default rather than a hardcoded value + *refminMag = psMetadataLookupF32 (&status, recipe, "REFSTAR.ZP.MIN.MAG"); + //if (!status) *refminMag = -5.0; + if (!status) { + psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit is missing REFSTAR.ZP.MIN.MAG"); + return false; + } + *refmaxMag = psMetadataLookupF32 (&status, recipe, "REFSTAR.ZP.MAX.MAG"); + //if (!status) *refmaxMag = 30.0; + if (!status) { + psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit is missing REFSTAR.ZP.MAX.MAG"); + return false; + } + + // select the filter; default to any photcode and mag limit otherwise + char *filter = psMetadataLookupStr (&status, fpa->concepts, "FPA.FILTERID"); + if (!status) ESCAPE ("RefMagLimit missing FPA.FILTER in concepts"); + + // need to select the PHOTCODE.DATA folder that matches our filter + psMetadataItem *item = psMetadataLookup (recipe, "PHOTCODE.DATA"); + if (!item) ESCAPE ("RefMagLimit PHOTCODE.DATA folders missing"); + if (item->type != PS_DATA_METADATA_MULTI) ESCAPE ("RefMagLimit PHOTCODE.DATA not a multi"); + + // PHOTCODE.DATA is a multi of metadata items + psListIterator *iter = psListIteratorAlloc(item->data.list, PS_LIST_HEAD, false); + + psMetadataItem *refItem = NULL; + while ((refItem = psListGetAndIncrement (iter))) { + if (refItem->type != PS_DATA_METADATA) ESCAPE ("RefMagLimit PHOTCODE.DATA entry is not a metadata folder"); + + char *refFilter = psMetadataLookupStr (&status, refItem->data.md, "FILTER"); + if (!status) { + // always seems to happen once, commented out in ZeroPoint above as well + //psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit PHOTCODE.DATA recipe folder is missing FILTER"); + continue; + } + + // does this entry match the current filter? + if (strcmp (refFilter, filter)) continue; + + psLogMsg ("psastro", PS_LOG_DETAIL, "RefMagLimit PHOTCODE.DATA found for filter %s", filter); + + // -- may set one or other or both -- no reason to log if found or often not + tmpmin = psMetadataLookupF32 (&status, refItem->data.md, "REFSTAR.ZP.MIN.MAG"); + if (status) { + *refminMag = tmpmin; + } + tmpmax = psMetadataLookupF32 (&status, refItem->data.md, "REFSTAR.ZP.MAX.MAG"); + if (status) { + *refmaxMag = tmpmax; + } + // once finds filter, should free and return true but multiple groups cause issue so just go through all groups now + //psFree (iter); + //return true; + } + + // log out set values + psLogMsg ("psastro", PS_LOG_INFO, "using REFSTAR.ZP.MIN.MAG %f",*refminMag); + psLogMsg ("psastro", PS_LOG_INFO, "using REFSTAR.ZP.MAX.MAG %f",*refmaxMag); + + psFree (iter); + return true; +} +