Index: /branches/eam_branch_20080706/ippconfig/cfh12k/ppMerge.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/cfh12k/ppMerge.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/cfh12k/ppMerge.config (revision 18475) @@ -28,5 +28,4 @@ FRACLOW F32 0.1 # Fraction of low pixels to reject immediately NKEEP S32 5 # Minimum number of pixels in stack to keep - MASKVAL S32 0xff # Mask value for input data WEIGHTS BOOL TRUE # Use image weights? COMBINE STR MEAN # Statistic to use for combination: Index: /branches/eam_branch_20080706/ippconfig/lbc_red/ppMerge.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lbc_red/ppMerge.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/lbc_red/ppMerge.config (revision 18475) @@ -28,5 +28,4 @@ FRACLOW F32 0.1 # Fraction of low pixels to reject immediately NKEEP S32 5 # Minimum number of pixels in stack to keep - MASKVAL S32 0xff # Mask value for input data WEIGHTS BOOL TRUE # Use image weights? COMBINE STR MEAN # Statistic to use for combination: Index: /branches/eam_branch_20080706/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/filerules-split.mdc (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/filerules-split.mdc (revision 18475) @@ -138,5 +138,5 @@ PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF -PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF NONE CHIP TRUE MEF SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE Index: /branches/eam_branch_20080706/ippconfig/recipes/masks.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/masks.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/masks.config (revision 18475) @@ -1,10 +1,26 @@ ### Recipe specifying values for various mask concepts + +# this file defines bit values for various abstract mask concepts. It +# is not necessary for all bit values to be unique. Note that there +# are two main classes: detector-intrinsic and detector-extrinsic +# effects. The former will default to DETECTOR and the latter to +# RANGE if those values are not specifed + +# Note that it is necessary to leave at least a single unassigned bit +# for internal marking (or ppImage will complain) + +# these mask values are used for detector intrinsic effects +# BLANK and FLAT will be set to DETECTOR if not specified +DETECTOR U8 0x01 # Generic detector-intrinsic defect BLANK U8 0x01 # The pixel is blank or has no (valid) data +DARK U8 0x02 # The pixel is not correctable for dark current FLAT U8 0x02 # The pixel is non-positive in the flat-field -DETECTOR U8 0x02 # The detector pixel is bad (e.g., bad column, charge trap) + +# these mask values are used for detector extrinsic effects +# SAT and BAD will be set to RANGE if not specified +RANGE U8 0x04 # The pixel is out of range in the image of interest SAT U8 0x04 # The pixel is saturated in the image of interest BAD U8 0x04 # The pixel is low in the image of interest -RANGE U8 0x04 # The pixel is out of range in the image of interest + CR U8 0x08 # The pixel is probably a CR -SUSPECT U8 0x40 # The pixel is suspected of being bad, but may not be -MARK U8 0x80 # The pixel is marked as temporarily ignored +GHOST U8 0x08 # The pixel is probably a CR Index: /branches/eam_branch_20080706/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/ppImage.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/ppImage.config (revision 18475) @@ -11,5 +11,7 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL FALSE # Mask bad pixels -MASK.VALUE STR SAT,BAD,DETECTOR # Mask pixels with these attributes + +# XXX EAM : this is being deprecated +# MASK.VALUE STR SAT,BAD,DETECTOR # Mask pixels with these attributes MASK.BUILD BOOL FALSE # Build internal mask image Index: /branches/eam_branch_20080706/ippconfig/recipes/ppMerge.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/ppMerge.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/ppMerge.config (revision 18475) @@ -20,6 +20,6 @@ MASK.CHIPSTATS BOOL TRUE # Measure stats for masking by chip (otherwise by readout)? MASK.GROW S32 0 # Grow bad pixels by this radius -MASK.GROWVAL STR SUSPECT # Give grown mask pixels this value -MASKVAL STR SAT,BAD # Mask value for input data +MASK.SET.VALUE STR FLAT # set this bit in the output mask + COMBINE STR CLIPPED # Statistic to use for combination MEAN STR ROBUST_MEDIAN # Statistic to use to measure the mean @@ -83,4 +83,5 @@ MASK.BAD F32 0.2 # Threshold for bad pixels (sigma) MASK.MODE STR FRACTION # Mode for identifying bad pixels in the suspect map + MASK.SET.VALUE STR DARK # set this bit in the output mask END @@ -90,4 +91,5 @@ MASK.BAD F32 0.2 # Threshold for bad pixels (sigma) MASK.MODE STR FRACTION # Mode for identifying bad pixels in the suspect map + MASK.SET.VALUE STR FLAT # set this bit in the output mask END Index: /branches/eam_branch_20080706/ippconfig/recipes/ppStats.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/ppStats.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/ppStats.config (revision 18475) @@ -3,5 +3,9 @@ # Options governing statistics SAMPLE F32 0.1 # Fraction of cell to sample -MASKVAL STR SAT,BAD # Mask value to use for statistics + +# Mask value to use for statistics. this is only used for stand-alone +# operations. ppStats library calls pass a mask value as needed +MASKVAL STR SAT,BAD + # Define the outputs as MULTI Index: /branches/eam_branch_20080706/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/psphot.config (revision 18474) +++ /branches/eam_branch_20080706/ippconfig/recipes/psphot.config (revision 18475) @@ -15,6 +15,4 @@ ZERO_POINT F32 25.000 # zero point used by DVO ZERO_PT F32 25.000 # zero point used by DVO - -MASKVAL STR SAT,BAD,BLANK # Mask these types of pixels OUTPUT.FORMAT STR PS1_DEV_1 Index: /branches/eam_branch_20080706/ppMerge/src/ppMergeArguments.c =================================================================== --- /branches/eam_branch_20080706/ppMerge/src/ppMergeArguments.c (revision 18474) +++ /branches/eam_branch_20080706/ppMerge/src/ppMergeArguments.c (revision 18475) @@ -138,5 +138,8 @@ psMetadataAddS32(arguments, PS_LIST_TAIL, "-nkeep", 0, "Minimum number of pixels in stack to keep", 0); psMetadataAddBool(arguments, PS_LIST_TAIL, "-weights", 0, "Use image weights in combination?", false); - psMetadataAddStr(arguments, PS_LIST_TAIL, "-maskval", 0, "Mask value for input data", NULL); + + // XXX EAM : not clear this should be allowed on the command line. + // psMetadataAddStr(arguments, PS_LIST_TAIL, "-maskval", 0, "Mask value for input data", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-combine", 0, "Statistic to use for combination", NULL); psMetadataAddStr(arguments, PS_LIST_TAIL, "-mean", 0, "Statistic to use to measure the mean", NULL); @@ -159,5 +162,5 @@ psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-mode", 0, "Mode to identify bad pixels", NULL); psMetadataAddS32(arguments, PS_LIST_TAIL, "-mask-grow", 0, "Number of pixels to grow final mask", 0); - psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-growval", 0, "Value to give grown mask pixels", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-set-valud",0, "Value to set for output mask pixels", NULL); psMetadataAddBool(arguments, PS_LIST_TAIL, "-mask-chip", 0, "Measure mask statistics by chip?", false); @@ -264,5 +267,8 @@ VALUE_ARG_RECIPE_INT("-nkeep", "NKEEP", S32, 0); VALUE_ARG_RECIPE_BOOL("-weights", "WEIGHTS"); - VALUE_ARG_RECIPE_MASK("-maskval", "MASKVAL"); + + // XXX we do not supply this on the command line + // VALUE_ARG_RECIPE_MASK("-maskval", "MASKVAL"); + VALUE_ARG_RECIPE_STAT("-combine", "COMBINE"); VALUE_ARG_RECIPE_STAT("-mean", "MEAN"); @@ -281,6 +287,6 @@ VALUE_ARG_RECIPE_FLOAT("-mask-suspect", "MASK.SUSPECT", F32); VALUE_ARG_RECIPE_FLOAT("-mask-bad", "MASK.BAD", F32); - VALUE_ARG_RECIPE_INT("-mask-grow", "MASK.GROW", S32, -1); - VALUE_ARG_RECIPE_MASK("-mask-growval", "MASK.GROWVAL"); + VALUE_ARG_RECIPE_INT("-mask-grow", "MASK.GROW", S32, 0); + VALUE_ARG_RECIPE_STR("-mask-set-value", "MASK.SET.VALUE"); VALUE_ARG_RECIPE_BOOL("-mask-chip", "MASK.CHIPSTATS"); Index: /branches/eam_branch_20080706/ppMerge/src/ppMergeLoop.c =================================================================== --- /branches/eam_branch_20080706/ppMerge/src/ppMergeLoop.c (revision 18474) +++ /branches/eam_branch_20080706/ppMerge/src/ppMergeLoop.c (revision 18475) @@ -39,5 +39,4 @@ float frachigh = psMetadataLookupF32(NULL, arguments, "FRACHIGH"); // Reject fraction of hi pixels int nKeep = psMetadataLookupS32(NULL, arguments, "NKEEP"); // Minimum number of values to keep - psMaskType maskVal = psMetadataLookupU8(NULL, arguments, "MASKVAL"); // Value to mask psStatsOptions combineStat = psMetadataLookupS32(NULL, arguments, "COMBINE"); // Combination statistic bool useWeights = psMetadataLookupBool(NULL, arguments, "WEIGHTS"); // Use weights? @@ -49,7 +48,15 @@ int fringeSmoothY = psMetadataLookupS32(NULL, arguments, "FRINGE.YSMOOTH"); // Smoothing regions in y + // set the mask and mark bit values based on the named masks + psMaskType maskVal; + psMaskType markVal; + if (!pmConfigMaskSetBits (&maskVal, &markVal, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + pmCombineParams *combination = pmCombineParamsAlloc(combineStat); // Combination parameters combination->maskVal = maskVal; - combination->blank = pmConfigMask("BLANK", config); + combination->blank = pmConfigMaskGet("BLANK", config); combination->nKeep = nKeep; combination->fracHigh = frachigh; @@ -294,5 +301,5 @@ } - if (!ppStatsFPA(stats, outFPA, view, maskVal | pmConfigMask("BLANK", config), config)) { + if (!ppStatsFPA(stats, outFPA, view, maskVal, config)) { psError(PS_ERR_UNKNOWN, true, "Unable to generate stats for image."); goto ERROR; Index: /branches/eam_branch_20080706/ppMerge/src/ppMergeMask.c =================================================================== --- /branches/eam_branch_20080706/ppMerge/src/ppMergeMask.c (revision 18474) +++ /branches/eam_branch_20080706/ppMerge/src/ppMergeMask.c (revision 18475) @@ -18,5 +18,5 @@ psRandom *rng, // Random number generator psMetadata *stats // Statistics output - ) + ) { assert(config); @@ -28,5 +28,4 @@ psStatsOptions meanStat = psMetadataLookupS32(NULL, config->arguments, "MEAN"); // Statistic for mean psStatsOptions stdevStat = psMetadataLookupS32(NULL, config->arguments, "STDEV"); // Statistic for stdev - psMaskType maskVal = psMetadataLookupU8(NULL, config->arguments, "MASKVAL"); // Value to mask int sample = psMetadataLookupS32(NULL, config->arguments, "SAMPLE"); // Size of sample for statistics bool chipStats = psMetadataLookupBool(&mdok, config->arguments, "MASK.CHIPSTATS"); // Statistics on chip? @@ -35,9 +34,22 @@ pmMaskIdentifyMode maskMode = psMetadataLookupS32(NULL, config->arguments, "MASK.MODE"); // Mode for identifying bad pixels int maskGrow = psMetadataLookupS32(NULL, config->arguments, "MASK.GROW"); // Radius to grow mask - psMaskType maskGrowVal = psMetadataLookupU8(NULL, config->arguments, "MASK.GROWVAL"); // Value for grown mask bool smoothSuspect = psMetadataLookupBool(&mdok, config->arguments, "MASK.SMOOTH.SUSPECT"); // Radius to grow mask float smoothScale = psMetadataLookupF32(&mdok, config->arguments, "MASK.SMOOTH.SCALE"); // Radius to grow mask + psMaskType markVal; + psMaskType maskValRaw; + if (!pmConfigMaskSetBits (&maskValRaw, &markVal, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + + char *maskOutName = psMetadataLookupStr (&mdok, config->arguments, "MASK.SET.VALUE"); + psMaskType maskValOut = pmConfigMaskGet (maskOutName, config); + if (!maskValOut) { + psError (PS_ERR_UNKNOWN, true, "Undefined output mask bit value"); + return false; + } + psStats *statistics = psStatsAlloc(meanStat | stdevStat); // Statistics for background @@ -137,9 +149,7 @@ int x = pixel % numCols; int y = pixel / numCols; - if ((mask && (mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal)) || - !isfinite(image->data.F32[y][x]) || - (outMask && (outMask->data.PS_TYPE_MASK_DATA[y][x] & maskVal))) { - continue; - } + if (mask && (mask->data.PS_TYPE_MASK_DATA[y][x] & maskValRaw)) continue; + if (outMask && (outMask->data.PS_TYPE_MASK_DATA[y][x] & maskValOut)) continue; + if (!isfinite(image->data.F32[y][x])) continue; values->data.F32[valueIndex++] = image->data.F32[y][x]; @@ -155,6 +165,10 @@ } - if (!pmMaskFlagSuspectPixels(outRO, readout, psStatsGetValue(statistics, meanStat), - psStatsGetValue(statistics, stdevStat), maskSuspect, maskVal)) { + float mean = psStatsGetValue(statistics, meanStat); + float stdev = psStatsGetValue(statistics, stdevStat); + + // this function increments the count for each suspect pixel in each input plane + // maskValRaw is used to test for valid input pixels + if (!pmMaskFlagSuspectPixels(outRO, readout, mean, stdev, maskSuspect, maskValRaw)) { psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i); psFree(inView); @@ -189,8 +203,6 @@ pmHDU *hdu = pmHDUFromCell(inCell); // HDU for cell - if (!hdu || hdu->blankPHU) { - // No data here - continue; - } + if (!hdu || hdu->blankPHU) continue; + pmReadout *readout = inCell->readouts->data[0]; // Readout of interest @@ -198,6 +210,8 @@ pmReadout *outRO = pmFPAfileThisReadout(config->files, inView, "PPMERGE.OUTPUT.MASK"); - if (!pmMaskFlagSuspectPixels(outRO, readout, psStatsGetValue(statistics, meanStat), - psStatsGetValue(statistics, stdevStat), maskSuspect, maskVal)) { + float mean = psStatsGetValue(statistics, meanStat); + float stdev = psStatsGetValue(statistics, stdevStat); + + if (!pmMaskFlagSuspectPixels(outRO, readout, mean, stdev, maskSuspect, maskValRaw)) { psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i); goto MERGE_MASK_ERROR; @@ -229,12 +243,9 @@ while ((outCell = pmFPAviewNextCell(outView, outFPA, 1))) { - // skipp inactive cells + // skip inactive cells if (!outCell->process) continue; pmHDU *hdu = pmHDUFromCell(outCell); // HDU for cell - if (!hdu || hdu->blankPHU) { - // No data here - continue; - } + if (!hdu || hdu->blankPHU) continue; psTrace("ppMerge", 1, "Getting bad pixels for chip %d cell %d", outView->chip, outView->cell); @@ -250,5 +261,6 @@ } - if (!pmMaskIdentifyBadPixels(outRO, maskVal, maskBad, maskMode)) { + // set the bad pixels to the value 'maskVal' + if (!pmMaskIdentifyBadPixels(outRO, maskValOut, maskBad, maskMode)) { psError(PS_ERR_UNKNOWN, false, "Unable to mask bad pixels"); goto MERGE_MASK_ERROR; @@ -274,6 +286,6 @@ } - if (maskGrowVal > 0) { - psImage *grown = psImageGrowMask(NULL, outRO->mask, maskVal, maskGrow, maskGrowVal); // Grown mask + if (maskGrow > 0) { + psImage *grown = psImageGrowMask(NULL, outRO->mask, maskValOut, maskGrow, maskValOut); // Grown mask psFree(outRO->mask); outRO->mask = grown; @@ -307,6 +319,5 @@ outRO->image = psImageAlloc(outRO->mask->numCols, outRO->mask->numRows, PS_TYPE_F32); psImageInit(outRO->image, 1.0); - if (!ppStatsFPA(stats, outRO->parent->parent->parent, outView, - maskVal | pmConfigMask("BLANK", config), config)) { + if (!ppStatsFPA(stats, outRO->parent->parent->parent, outView, maskValOut, config)) { psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image."); psFree(outRO); @@ -337,8 +348,4 @@ } - - - - bool ppMergeMask(pmConfig *config) { @@ -381,4 +388,7 @@ goto PPMERGE_MASK_ERROR; } + + // XXX this function should use pmConfigMaskReadHeader () to get the named values defined + // for the input masks. psString outName = ppMergeOutputFile(config); // Name of output file @@ -414,4 +424,7 @@ psListAdd(inChips, PS_LIST_TAIL, chip); } + + // XXX I need to call pmConfigMaskWriteHeader for the PHU somewhere, after it is created! + if (!pmConceptsAverageChips(outChip, inChips, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to average Chip concepts."); Index: /branches/eam_branch_20080706/ppMerge/src/ppMergeScaleZero.c =================================================================== --- /branches/eam_branch_20080706/ppMerge/src/ppMergeScaleZero.c (revision 18474) +++ /branches/eam_branch_20080706/ppMerge/src/ppMergeScaleZero.c (revision 18475) @@ -21,6 +21,7 @@ psStatsOptions meanStat = psMetadataLookupS32(NULL, config->arguments, "MEAN"); // Statistic for mean psStatsOptions stdevStat = psMetadataLookupS32(NULL, config->arguments, "STDEV"); // Statistic for stdev - psMaskType maskVal = psMetadataLookupU8(NULL, config->arguments, "MASKVAL"); // Value to mask int shutterSize = psMetadataLookupS32(NULL, config->arguments, "SHUTTER.SIZE"); // Size of shutter region + + psMaskType maskVal = pmConfigMaskGet("MASK.VALUE", config); // Value to mask psVector *gains = NULL; // Gains for each cell Index: /branches/eam_branch_20080706/ppStats/src/ppStatsFPA.c =================================================================== --- /branches/eam_branch_20080706/ppStats/src/ppStatsFPA.c (revision 18474) +++ /branches/eam_branch_20080706/ppStats/src/ppStatsFPA.c (revision 18475) @@ -21,8 +21,6 @@ } - // Override recipe mask value if one is provided - if (maskVal != 0) { - data->maskVal = maskVal; - } + // Override recipe mask value + data->maskVal = maskVal; if (data->fpa) { Index: /branches/eam_branch_20080706/ppStats/src/ppStatsMetadata.c =================================================================== --- /branches/eam_branch_20080706/ppStats/src/ppStatsMetadata.c (revision 18474) +++ /branches/eam_branch_20080706/ppStats/src/ppStatsMetadata.c (revision 18475) @@ -29,8 +29,6 @@ data->doStats = false; - // Override recipe mask value if one is provided - if (maskVal != 0) { - data->maskVal = maskVal; - } + // Override recipe mask value + data->maskVal = maskVal; if (data->fpa) { Index: /branches/eam_branch_20080706/ppStats/src/ppStatsPixels.c =================================================================== --- /branches/eam_branch_20080706/ppStats/src/ppStatsPixels.c (revision 18474) +++ /branches/eam_branch_20080706/ppStats/src/ppStatsPixels.c (revision 18475) @@ -28,8 +28,6 @@ data->concepts = psListAlloc(NULL); - // Override recipe mask value if one is provided - if (maskVal != 0) { - data->maskVal = maskVal; - } + // Override recipe mask value + data->maskVal = maskVal; if (data->fpa) { Index: /branches/eam_branch_20080706/ppStats/src/ppStatsSetupFromRecipe.c =================================================================== --- /branches/eam_branch_20080706/ppStats/src/ppStatsSetupFromRecipe.c (revision 18474) +++ /branches/eam_branch_20080706/ppStats/src/ppStatsSetupFromRecipe.c (revision 18475) @@ -127,8 +127,9 @@ } + // set the mask value used for stand-alone analyses. if (data->maskVal == 0) { const char *names = psMetadataLookupStr(&mdok, recipe, "MASKVAL"); // Names for mask value if (mdok) { - data->maskVal = pmConfigMask(names, config); + data->maskVal = pmConfigMaskGet(names, config); } else { psWarning("MASKVAL in recipe is not of type STR --- retaining default.\n");