Index: /branches/eam_branches/ipp-20220316/ppImage/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/Makefile.am (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/Makefile.am (revision 42372) @@ -24,4 +24,5 @@ ppImageDetrendRecord.c \ ppImageDetrendNonLinear.c \ + ppImageDetrendNewNonLinear.c \ ppImageDetrendFringe.c \ ppImageDetrendFree.c \ Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImage.h =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImage.h (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImage.h (revision 42372) @@ -35,4 +35,5 @@ bool doAuxMask; // apply auxillary mask bool doNonLin; // Non-linearity correction + bool doNewNonLin; // Non-linearity correction bool doOverscan; // Overscan subtraction bool doNoiseMap; // Bias subtraction @@ -157,4 +158,8 @@ ppImageOptions *ppImageParseCamera(pmConfig *config); bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options); +bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned + psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned + psMetadata *source // Source of mask bits + ); // apply the cell flips to the input data before analysis @@ -171,4 +176,6 @@ bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options); + +bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config); bool ppImageDetrendNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config); Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageArguments.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageArguments.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageArguments.c (revision 42372) @@ -25,4 +25,5 @@ fprintf(stderr, "\t-fringe/-fringelist: Fringe image and data.\n"); fprintf(stderr, "\t-linearity/-linearlist: linearity correction file.\n"); + fprintf(stderr, "\t-newnonlin/-newnonlinlist: non-linearity correction file (v 2023.01).\n"); fprintf(stderr, "\n"); exit (2); @@ -116,4 +117,5 @@ pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "NEWNONLIN", "-newnonlin", "-newnonlinlist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined"); pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.DEAD.CELLS", "-pattern-dead-cells", "-not-defined"); Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendFree.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendFree.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendFree.c (revision 42372) @@ -14,4 +14,5 @@ "PPIMAGE.SHUTTER", "PPIMAGE.LINEARITY", + "PPIMAGE.NEWNONLIN", NULL }; Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNewNonLinear.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNewNonLinear.c (revision 42372) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNewNonLinear.c (revision 42372) @@ -0,0 +1,42 @@ +#include "ppImage.h" + +bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *detview, pmConfig *config) { + + bool status; + + pmFPAfile *linearity_file = psMetadataLookupPtr(&status, config->files, "PPIMAGE.NEWNONLIN"); + psFits *linearity_fits = linearity_file->fits; + + char *extname = psMetadataLookupStr(&status, input->parent->concepts, "CELL.NAME"); + if (!extname) { + psError(PS_ERR_IO, false, "missing CELL.NAME in concepts"); + return(false); + } + + // if pmFPAfile has been loaded (by ppImageDefineFile in ppImageParseCamera), then + // the file corresponding to the current chip is found and opened + // NOTE: if the extname is missing, we skip the correction + if (!psFitsMoveExtName(linearity_fits, extname)) { + psLogMsg ("ppImageDetrendNewNonLinear", 4, "Unable to move to non-linearity (v2023) table %s, skipping", extname); + return(true); + } + + psArray *table = psFitsReadTable(linearity_fits); + if (!table) { + psError(PS_ERR_IO, false, "Unable to read non-linearity table.\n"); + return(false); + } + + if (!pmNewNonLinearityApply(input,table)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply non-linearity corrections.\n"); + psFree (table); + return(false); + } + psFree (table); + + return true; +} + +// the new non-linearity correction is a set of splines, one per cell, with the +// xKnots equal to the log10(DN) in the pixel, and the spline value at a point +// equal to a fractional multiplier (i.e., 1 + dF) Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNonLinear.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNonLinear.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNonLinear.c (revision 42372) @@ -1,6 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - #include "ppImage.h" Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendReadout.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendReadout.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendReadout.c (revision 42372) @@ -149,4 +149,13 @@ // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout")); } + // New Non-linearity correction (exclusive of the above) + if (options->doNewNonLin) { + if (!ppImageDetrendNewNonLinear(input, detview, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to correct Non-Linearity with new version (2023)"); + psFree(detview); + return(false); + } + // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout")); + } // set up the dark and bias @@ -213,39 +222,4 @@ psImageUnbin (noiseImage, noiseMap->image, binning); psFree (binning); - // Stolen from pmSkySubtract.c - // CZW: Unneeded, as psImageUnbin does the bilinear interpolation? It still looks blocky. -/* psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME); // Recipe */ -/* psAssert(recipe, "Should be there!"); */ - -/* psS32 xBinFactor = psMetadataLookupS32(NULL,recipe,"NOISE.XBIN"); */ -/* psS32 yBinFactor = psMetadataLookupS32(NULL,recipe,"NOISE.YBIN"); */ -/* psImageInterpolateOptions *interp = psImageInterpolateOptionsAlloc(PS_INTERPOLATE_BILINEAR, */ -/* noiseMap->image, */ -/* NULL, NULL, */ -/* 0, 0.0, 0.0, 0, 0, 0.0); */ -/* for (psS32 row = 0; row < input->image->numRows; row++) { */ -/* for (psS32 col = 0; col < input->image->numCols; col++) { */ -/* // We calculate the F32 value of the pixel coordinates in the */ -/* // binned image and then use a pixel interpolation routine to */ -/* // determine the value of the pixel at that location. */ -/* psF32 binRowF64 = ((psF32) row) / ((psF32) yBinFactor); */ -/* psF32 binColF64 = ((psF32) col) / ((psF32) xBinFactor); */ -/* // We add 0.5 to the pixel locations since the pixel */ -/* // interpolation routine defines the location of pixel */ -/* // (i, j) as (i+0.5, j+0.5). */ -/* binRowF64+= 0.5; */ -/* binColF64+= 0.5; */ - -/* double binPixel; */ -/* if (!psImagePixelInterpolate(&binPixel, NULL, NULL, binColF64, binRowF64, interp)) { */ -/* psError(PS_ERR_UNKNOWN, false, "Unable to interpolate image."); */ -/* psFree(interp); */ -/* psFree(noiseImage); */ -/* return NULL; */ -/* } */ -/* noiseImage->data.F32[row][col] = binPixel; */ -/* } */ -/* } */ -/* psFree(interp); */ } Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendRecord.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendRecord.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendRecord.c (revision 42372) @@ -75,5 +75,6 @@ detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", "Fringe filename"); - detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename"); + detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename"); + detrendRecord(options->doNewNonLin, detrend, config, view, "PPIMAGE.NEWNONLIN","DETREND.NEWNONLIN","Non-linearity table filename (v2023)"); detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename"); Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageOptions.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageOptions.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageOptions.c (revision 42372) @@ -28,4 +28,5 @@ options->doAuxMask = false; // apply auxillary mask options->doNonLin = false; // Non-linearity correction + options->doNewNonLin = false; // New Non-linearity correction (v2023) options->doOverscan = false; // Overscan subtraction options->doNoiseMap = false; // Apply Read Noise Map @@ -175,4 +176,12 @@ // XXX PAP: The overscan stuff needs to be updated following the reworked API + // New Non-linearity (v 2023) recipe options + // non-linearity corrections are loaded from a file defined in the detrend system or on the command-line + if (psMetadataLookupBool(NULL, recipe, "NEWNONLIN")) { + options->doNewNonLin = true; + } + + // XXX PAP: The overscan stuff needs to be updated following the reworked API + // Overscan recipe options // XXX EAM : we should abort on invalid options. default options? Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageParseCamera.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageParseCamera.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageParseCamera.c (revision 42372) @@ -104,10 +104,21 @@ if (options->doNonLin) { - if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.LINEARITY", "LINEARITY", - PM_FPA_FILE_LINEARITY, PM_DETREND_TYPE_LINEARITY)) { - psError(PS_ERR_IO, false, "Can't find a non-linearity correction source"); - psFree(options); - return NULL; - } + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.LINEARITY", "LINEARITY", + PM_FPA_FILE_LINEARITY, PM_DETREND_TYPE_LINEARITY)) { + psError(PS_ERR_IO, false, "Can't find a non-linearity correction source"); + psFree(options); + return NULL; + } + } + if (options->doNewNonLin) { + // if the file has been specified on the command-line (-newnonlin file), then the file + // is identified by the NEWNONLIN entry in config->arguments. otherwise, load from the + // detrend system as type NEWNONLIN + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NEWNONLIN", "NEWNONLIN", + PM_FPA_FILE_NEWNONLIN, PM_DETREND_TYPE_NEWNONLIN)) { + psError(PS_ERR_IO, false, "Can't find a new non-linearity correction source"); + psFree(options); + return NULL; + } } Index: /branches/eam_branches/ipp-20220316/ppImage/src/ppImageSetMaskBits.c =================================================================== --- /branches/eam_branches/ipp-20220316/ppImage/src/ppImageSetMaskBits.c (revision 42371) +++ /branches/eam_branches/ipp-20220316/ppImage/src/ppImageSetMaskBits.c (revision 42372) @@ -5,9 +5,61 @@ #include "ppImage.h" +// Structure to hold the properties of a mask value +typedef struct { + char *badMaskName; // name for "bad" (i.e., mask me please) pixels + char *fallbackName; // Fallback name in case a bad mask name is not defined + psImageMaskType defaultMaskValue; // Default value in case a bad mask name and its fallback are not defined + bool isBad; // include this value as part of the MASK.VALUE entry (generically bad) +} pmConfigMaskInfo; + +static pmConfigMaskInfo ppimagemasks[] = { + // Features of the detector + { "DETECTOR", NULL, 0x01, true }, // Something is wrong with the detector + { "FLAT", "DETECTOR", 0x01, true }, // Pixel doesn't flat-field properly + { "DARK", "DETECTOR", 0x01, true }, // Pixel doesn't dark-subtract properly + { "BLANK", "DETECTOR", 0x01, true }, // Pixel doesn't contain valid data + { "CTE", "DETECTOR", 0x01, false }, // Pixel has poor CTE + { "BURNTOOL", NULL, 0x04, false }, // Pixel has been touched by burntool + // Invalid signal ranges + { "SAT", NULL, 0x02, true }, // Pixel is saturated or non-linear + { "LOW", "SAT", 0x02, true }, // Pixel is low + { "SUSPECT", NULL, 0x04, false }, // Pixel is suspected of being bad + // Non-astronomical structures + { "CR", NULL, 0x08, true }, // Pixel contains a cosmic ray + { "SPIKE", NULL, 0x08, false }, // Pixel contains a diffraction spike + { "GHOST", NULL, 0x08, false }, // Pixel contains an optical ghost + { "STREAK", NULL, 0x08, false }, // Pixel contains a streak + { "CROSSTALK", NULL, 0x08, false }, // Pixel contains crosstalk data + { "STARCORE", NULL, 0x08, false }, // Pixel contains a bright star core + // Effects of convolution and interpolation + { "CONV.BAD", NULL, 0x02, true }, // Pixel is bad after convolution with a bad pixel + { "CONV.POOR", NULL, 0x04, false }, // Pixel is poor after convolution with a bad pixel +}; + bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options) { - if (!pmConfigMaskSetBits(&options->maskValue, &options->markValue, config)) { - psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); - return false; + + // Look up recipe values + psMetadata *pprecipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME); + + psAssert(pprecipe, "We checked this earlier, so it should be here."); + bool doDetectCTE = psMetadataLookupBool(NULL, pprecipe, "DETECT.CTE"); // Do detections on pixels underneath CTE masks + + // this function sets the required single-image mask bits + if(!doDetectCTE) { + if (!pmConfigMaskSetBits (&options->maskValue, &options->markValue, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + } else { + psMetadata *maskrecipe = psMetadataLookupMetadata(NULL, config->recipes, "MASKS"); // The recipe + if (!maskrecipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find MASKS recipe."); + return false; + } + if (!ppImageMaskSetInMetadata(&options->maskValue, &options->markValue, maskrecipe)) { + psError (PS_ERR_UNKNOWN, true, "Unable to determine the mask value"); + return false; + } } @@ -66,2 +118,70 @@ return true; } + + +bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned + psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned + psMetadata *source // Source of mask bits + ) +{ + PS_ASSERT_METADATA_NON_NULL(source, false); + + // Ensure all the bad mask names exist, and set the value to catch all bad pixels + psImageMaskType maskValue = 0; // Value to mask to catch all the bad pixels + psImageMaskType allMasks = 0; // Value to mask to catch all masked bits (to set MARK) + + int nMasks = sizeof (ppimagemasks) / sizeof (pmConfigMaskInfo); + + for (int i = 0; i < nMasks; i++) { + bool mdok; // Status of MD lookup + psImageMaskType value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].badMaskName); // Value of mask + if (!mdok) { + psWarning ("problem with mask value %s\n", ppimagemasks[i].badMaskName); + } + + if (!value) { + if (ppimagemasks[i].fallbackName) { + value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].fallbackName); + } + if (!value) { + value = ppimagemasks[i].defaultMaskValue; + } + psMetadataAddImageMask(source, PS_LIST_TAIL, ppimagemasks[i].badMaskName, PS_META_REPLACE, NULL, value); + } + if (ppimagemasks[i].isBad) { + maskValue |= value; + } + allMasks |= value; + } + + // search for an unset bit to use for MARK: + psImageMaskType markValue = 0x00; + psImageMaskType markTrial = 0x01; + + int nBits = sizeof(psImageMaskType) * 8; + for (int i = 0; !markValue && (i < nBits); i++) { + if (allMasks & markTrial) { + markTrial <<= 1; + } else { + markValue = markTrial; + } + } + if (!markValue) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the MARK bit mask: all bits taken!"); + return false; + } + + // update the list with the results + psMetadataAddImageMask(source, PS_LIST_TAIL, "MASK.VALUE", PS_META_REPLACE, NULL, maskValue); + psMetadataAddImageMask(source, PS_LIST_TAIL, "MARK.VALUE", PS_META_REPLACE, NULL, markValue); + + if (outMaskValue) { + *outMaskValue = maskValue; + } + if (outMarkValue) { + *outMarkValue = markValue; + } + + return true; +} +