Index: trunk/ippconfig/recipes/filerules-mef.mdc =================================================================== --- trunk/ippconfig/recipes/filerules-mef.mdc (revision 43037) +++ trunk/ippconfig/recipes/filerules-mef.mdc (revision 43038) @@ -72,4 +72,5 @@ PPIMAGE.PATTERN.DEAD.CELLS INPUT @DETDB FPA PATTERN.DEAD.CELLS PPIMAGE.NEWNONLIN INPUT @DETDB CHIP NEWNONLIN +PPIMAGE.VALID.OVERSCAN INPUT @DETDB FPA VALID.OVERSCAN ## Files used by ppMerge Index: trunk/ippconfig/recipes/filerules-simple.mdc =================================================================== --- trunk/ippconfig/recipes/filerules-simple.mdc (revision 43037) +++ trunk/ippconfig/recipes/filerules-simple.mdc (revision 43038) @@ -32,4 +32,5 @@ PPIMAGE.LINEARITY INPUT @DETDB CHIP LINEARITY PPIMAGE.NEWNONLIN INPUT @DETDB CHIP NEWNONLIN +PPIMAGE.VALID.OVERSCAN INPUT @DETDB FPA VALID.OVERSCAN FPCAMERA.OUTPUT STR FPCAMERA.OUT.CMF Index: trunk/ippconfig/recipes/filerules-split.mdc =================================================================== --- trunk/ippconfig/recipes/filerules-split.mdc (revision 43037) +++ trunk/ippconfig/recipes/filerules-split.mdc (revision 43038) @@ -56,4 +56,5 @@ PPIMAGE.PATTERN.ROW.AMP INPUT @DETDB FPA PATTERN.ROW.AMP PPIMAGE.PATTERN.DEAD.CELLS INPUT @DETDB FPA PATTERN.DEAD.CELLS +PPIMAGE.VALID.OVERSCAN INPUT @DETDB FPA VALID.OVERSCAN ## Files used by psvideophot Index: trunk/ippconfig/recipes/ppImage.config =================================================================== --- trunk/ippconfig/recipes/ppImage.config (revision 43037) +++ trunk/ippconfig/recipes/ppImage.config (revision 43038) @@ -19,4 +19,5 @@ MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails MASK.AUXMASK BOOL FALSE # augment mask with auxillary mask +MASK.BAD.OVERSCAN BOOL FALSE # augment mask with auxillary mask AUX.VIDEO.MASK STR NULL # name of aditional masking for video cells APPLY.BURNTOOL BOOL FALSE # apply burntool coorection Index: trunk/ppImage/src/Makefile.am =================================================================== --- trunk/ppImage/src/Makefile.am (revision 43037) +++ trunk/ppImage/src/Makefile.am (revision 43038) @@ -33,4 +33,5 @@ ppImageMosaicBackground.c \ ppImageMaskStats.c \ + ppImageMaskBadOverscan.c \ ppImagePhotom.c \ ppImageAstrom.c \ Index: trunk/ppImage/src/ppImage.h =================================================================== --- trunk/ppImage/src/ppImage.h (revision 43037) +++ trunk/ppImage/src/ppImage.h (revision 43038) @@ -36,4 +36,5 @@ bool doNonLin; // Non-linearity correction bool doNewNonLin; // Non-linearity correction + bool doMaskBadOverscan; // Mask cells/readouts with out-of-range overscan bool doOverscan; // Overscan subtraction bool doNoiseMap; // Bias subtraction @@ -180,4 +181,6 @@ bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options); +bool ppImageMaskBadOverscan(pmReadout *input, pmConfig *config); + bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config); Index: trunk/ppImage/src/ppImageArguments.c =================================================================== --- trunk/ppImage/src/ppImageArguments.c (revision 43037) +++ trunk/ppImage/src/ppImageArguments.c (revision 43038) @@ -26,4 +26,5 @@ fprintf(stderr, "\t-linearity/-linearlist: linearity correction file.\n"); fprintf(stderr, "\t-newnonlin/-newnonlinlist: non-linearity correction file (v 2023.01).\n"); + fprintf(stderr, "\t-valid-overscan/-valid-overscan-list: overscan valid range file.\n"); fprintf(stderr, "\n"); exit (2); @@ -118,4 +119,5 @@ pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "NEWNONLIN", "-newnonlin", "-newnonlinlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VALID.OVERSCAN", "-valid-overscan", "-valid-overscan-list"); pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined"); pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.DEAD.CELLS", "-pattern-dead-cells", "-not-defined"); Index: trunk/ppImage/src/ppImageDetrendFree.c =================================================================== --- trunk/ppImage/src/ppImageDetrendFree.c (revision 43037) +++ trunk/ppImage/src/ppImageDetrendFree.c (revision 43038) @@ -15,4 +15,5 @@ "PPIMAGE.LINEARITY", "PPIMAGE.NEWNONLIN", + "PPIMAGE.VALID.OVERSCAN", NULL }; Index: trunk/ppImage/src/ppImageDetrendReadout.c =================================================================== --- trunk/ppImage/src/ppImageDetrendReadout.c (revision 43037) +++ trunk/ppImage/src/ppImageDetrendReadout.c (revision 43038) @@ -167,4 +167,13 @@ } + // Mask bad cells based on the overscan values + if (options->doMaskBadOverscan) { + if (!ppImageMaskBadOverscan(input, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask readout based on overscan"); + psFree(detview); + return(false); + } + } + // measure the overscan-subtracted readoutBackground here (or subtract the overscan value?) // XXX this is the measurements and should be independent of the pattern masking Index: trunk/ppImage/src/ppImageDetrendRecord.c =================================================================== --- trunk/ppImage/src/ppImageDetrendRecord.c (revision 43037) +++ trunk/ppImage/src/ppImageDetrendRecord.c (revision 43038) @@ -75,6 +75,7 @@ detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", "Fringe filename"); - detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename"); - detrendRecord(options->doNewNonLin, detrend, config, view, "PPIMAGE.NEWNONLIN","DETREND.NEWNONLIN","Non-linearity table filename (v2023)"); + detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY", "DETREND.NONLIN", "Non-linearity table filename"); + detrendRecord(options->doNewNonLin, detrend, config, view, "PPIMAGE.NEWNONLIN", "DETREND.NEWNONLIN", "Non-linearity table filename (v2023)"); + detrendRecord(options->doMaskBadOverscan, detrend, config, view, "PPIMAGE.VALID.OVERSCAN", "DETREND.VALID.OVERSCAN", "Valid Overscan table filename"); detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename"); Index: trunk/ppImage/src/ppImageMaskBadOverscan.c =================================================================== --- trunk/ppImage/src/ppImageMaskBadOverscan.c (revision 43038) +++ trunk/ppImage/src/ppImageMaskBadOverscan.c (revision 43038) @@ -0,0 +1,64 @@ +#include "ppImage.h" + +bool ppImageMaskBadOverscan(pmReadout *input, pmConfig *config) { + + bool status; + + pmFPAfile *overscan_file = psMetadataLookupPtr(&status, config->files, "PPIMAGE.VALID.OVERSCAN"); + psFits *overscan_fits = overscan_file->fits; + + char *chipname = psMetadataLookupStr(&status, input->parent->parent->concepts, "CHIP.NAME"); + if (!chipname) { + psError(PS_ERR_IO, false, "missing CHIP.NAME in concepts"); + return false; + } + char *cellname = psMetadataLookupStr(&status, input->parent->concepts, "CELL.NAME"); + if (!cellname) { + psError(PS_ERR_IO, false, "missing CELL.NAME in concepts"); + return false; + } + + // if pmFPAfile has been loaded (by ppImageDefineFile in ppImageParseCamera), then + // the file corresponding to the current chip is found and opened + // NOTE: if the chipname is missing, we skip the correction + if (!psFitsMoveExtName(overscan_fits, chipname)) { + psLogMsg ("ppImageMaskBadOverscan", 4, "Unable to move to valid overscan table %s, skipping", chipname); + return true; + } + + psArray *table = psFitsReadTable(overscan_fits); + if (!table) { + psError(PS_ERR_IO, false, "Unable to read valid overscan table.\n"); + return false; + } + + psImageMaskType badMask = pmConfigMaskGet("DETECTOR", config); + if (!badMask) { + psError(PS_ERR_IO, false, "Invalid mask name DETECTOR.\n"); + return false; + } + + // find this cell in the table + for (int i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; + + char *thisCell = psMetadataLookupStr(&status, row, "CELL.NAME"); + if (strcmp(thisCell, cellname)) continue; + + // we have the correct cell now + float minValid = psMetadataLookupF32(&status, row, "MIN_VALID"); + float maxValid = psMetadataLookupF32(&status, row, "MAX_VALID"); + + if (!pmMaskBadOverscanApply(input, minValid, maxValid, badMask)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply valid overscan range.\n"); + psFree (table); + return false; + } + psFree (table); + return true; + } + psFree (table); + return false; +} + +// the valid overscan file contains tables stored by chip, with one row per cell Index: trunk/ppImage/src/ppImageOptions.c =================================================================== --- trunk/ppImage/src/ppImageOptions.c (revision 43037) +++ trunk/ppImage/src/ppImageOptions.c (revision 43038) @@ -29,4 +29,5 @@ options->doNonLin = false; // Non-linearity correction options->doNewNonLin = false; // New Non-linearity correction (v2023) + options->doMaskBadOverscan = false; // Mask cells / readouts if overscan is out of bounds options->doOverscan = false; // Overscan subtraction options->doNoiseMap = false; // Apply Read Noise Map @@ -182,4 +183,10 @@ } + // Mask cells/readouts with out-of-range overscan + // nominal overscan ranges are loaded from a file defined in the detrend system or on the command-line + if (psMetadataLookupBool(NULL, recipe, "MASK.BAD.OVERSCAN")) { + options->doMaskBadOverscan = true; + } + // XXX PAP: The overscan stuff needs to be updated following the reworked API Index: trunk/ppImage/src/ppImageParseCamera.c =================================================================== --- trunk/ppImage/src/ppImageParseCamera.c (revision 43037) +++ trunk/ppImage/src/ppImageParseCamera.c (revision 43038) @@ -117,4 +117,15 @@ if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NEWNONLIN", "NEWNONLIN", PM_FPA_FILE_NEWNONLIN, PM_DETREND_TYPE_NEWNONLIN)) { + psError(PS_ERR_IO, false, "Can't find a new non-linearity correction source"); + psFree(options); + return NULL; + } + } + if (options->doMaskBadOverscan) { + // if the file has been specified on the command-line (-valid-overscan file), then the file + // is identified by the VALID.OVERSCAN entry in config->arguments. otherwise, load from the + // detrend system as type VALID.OVERSCAN + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.VALID.OVERSCAN", "VALID.OVERSCAN", + PM_FPA_FILE_VALID_OVERSCAN, PM_DETREND_TYPE_VALID_OVERSCAN)) { psError(PS_ERR_IO, false, "Can't find a new non-linearity correction source"); psFree(options); Index: trunk/psModules/src/camera/pmFPAfile.c =================================================================== --- trunk/psModules/src/camera/pmFPAfile.c (revision 43037) +++ trunk/psModules/src/camera/pmFPAfile.c (revision 43038) @@ -542,4 +542,7 @@ if (!strcasecmp(type, "NEWNONLIN")) { return PM_FPA_FILE_NEWNONLIN; + } + if (!strcasecmp(type, "VALID.OVERSCAN")) { + return PM_FPA_FILE_VALID_OVERSCAN; } if (!strcasecmp(type, "ASTROM")) { Index: trunk/psModules/src/camera/pmFPAfile.h =================================================================== --- trunk/psModules/src/camera/pmFPAfile.h (revision 43037) +++ trunk/psModules/src/camera/pmFPAfile.h (revision 43038) @@ -56,4 +56,5 @@ PM_FPA_FILE_LINEARITY, PM_FPA_FILE_NEWNONLIN, + PM_FPA_FILE_VALID_OVERSCAN, PM_FPA_FILE_EXPNUM, } pmFPAfileType; Index: trunk/psModules/src/camera/pmFPAfileIO.c =================================================================== --- trunk/psModules/src/camera/pmFPAfileIO.c (revision 43037) +++ trunk/psModules/src/camera/pmFPAfileIO.c (revision 43038) @@ -286,4 +286,5 @@ case PM_FPA_FILE_LINEARITY: case PM_FPA_FILE_NEWNONLIN: + case PM_FPA_FILE_VALID_OVERSCAN: break; default: @@ -653,4 +654,5 @@ case PM_FPA_FILE_LINEARITY: case PM_FPA_FILE_NEWNONLIN: + case PM_FPA_FILE_VALID_OVERSCAN: case PM_FPA_FILE_EXPNUM: psTrace ("psModules.camera", 5, "closing %s (%s) (%d:%d:%d)\n", file->filename, file->name, view->chip, view->cell, view->readout); @@ -739,4 +741,5 @@ case PM_FPA_FILE_LINEARITY: case PM_FPA_FILE_NEWNONLIN: + case PM_FPA_FILE_VALID_OVERSCAN: psTrace ("psModules.camera", 5, "nothing to free for %s (%s)\n", file->filename, file->name); return true; @@ -900,4 +903,5 @@ case PM_FPA_FILE_LINEARITY: case PM_FPA_FILE_NEWNONLIN: + case PM_FPA_FILE_VALID_OVERSCAN: case PM_FPA_FILE_EXPNUM: psTrace ("psModules.camera", 5, "opening %s (%s) (%d:%d:%d)\n", Index: trunk/psModules/src/detrend/Makefile.am =================================================================== --- trunk/psModules/src/detrend/Makefile.am (revision 43037) +++ trunk/psModules/src/detrend/Makefile.am (revision 43038) @@ -11,4 +11,5 @@ pmNonLinear.c \ pmNewNonLinear.c \ + pmMaskBadOverscan.c \ pmBias.c \ pmOverscan.c \ @@ -32,4 +33,5 @@ pmNonLinear.h \ pmNewNonLinear.h \ + pmMaskBadOverscan.h \ pmBias.h \ pmOverscan.h \ Index: trunk/psModules/src/detrend/pmDetrendDB.h =================================================================== --- trunk/psModules/src/detrend/pmDetrendDB.h (revision 43037) +++ trunk/psModules/src/detrend/pmDetrendDB.h (revision 43038) @@ -44,4 +44,5 @@ PM_DETREND_TYPE_PATTERN_ROW_AMP, PM_DETREND_TYPE_PATTERN_DEAD_CELLS, + PM_DETREND_TYPE_VALID_OVERSCAN, } pmDetrendType; Index: trunk/psModules/src/detrend/pmMaskBadOverscan.c =================================================================== --- trunk/psModules/src/detrend/pmMaskBadOverscan.c (revision 43038) +++ trunk/psModules/src/detrend/pmMaskBadOverscan.c (revision 43038) @@ -0,0 +1,41 @@ +#include +#include + +#include "pmFPA.h" +#include "pmHDU.h" +#include "pmHDUUtils.h" +#include "pmMaskBadOverscan.h" + +// model is stored as a FITS table, read into an array +// each row is one cell, with (min, max) valid overscan range +bool pmMaskBadOverscanApply(pmReadout *inputReadout, float minValid, float maxValid, psImageMaskType maskValue) +{ + PS_ASSERT_PTR_NON_NULL(inputReadout, false); + PS_ASSERT_PTR_NON_NULL(inputReadout->image, false); + PS_ASSERT_PTR_NON_NULL(inputReadout->mask, false); + PS_ASSERT_IMAGE_TYPE(inputReadout->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_TYPE(inputReadout->mask, PS_TYPE_IMAGE_MASK, false); + + pmHDU *hdu = pmHDUFromReadout(inputReadout); // HDU of interest + + bool status; + float overscan = psMetadataLookupF32(&status, hdu->header, "OVER_VAL"); + if (!status) { + // no overscan value measured + return false; + } + + // readout is either all good or all bad + if ((overscan >= minValid) && (overscan <= maxValid)) return true; + + psImage *mask = inputReadout->mask; + + // completely mask the readout + for (int i = 0; i < mask->numRows; i++) { // Loop over rows : note: problem with discontinuity here + for (int j = 0; j < mask->numCols; j++) { // Loop over columns + mask->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + return true; +} + Index: trunk/psModules/src/detrend/pmMaskBadOverscan.h =================================================================== --- trunk/psModules/src/detrend/pmMaskBadOverscan.h (revision 43038) +++ trunk/psModules/src/detrend/pmMaskBadOverscan.h (revision 43038) @@ -0,0 +1,17 @@ +/* @file pmMaskBadOverscan.h + * @brief mask entire cells based on the overscan value + * @author Eugene Magnier, IfA + * Copyright 2026 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_MASK_BAD_OVERSCAN_H +#define PM_MASK_BAD_OVERSCAN_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// mask bad readouts +bool pmMaskBadOverscanApply(pmReadout *inputReadout, float minValid, float maxValid, psImageMaskType maskValue); + +/// @} +#endif Index: trunk/psModules/src/psmodules.h =================================================================== --- trunk/psModules/src/psmodules.h (revision 43037) +++ trunk/psModules/src/psmodules.h (revision 43038) @@ -75,4 +75,5 @@ #include #include +#include #include #include