Changeset 42372

Timestamp:

Feb 4, 2023, 12:15:28 PM (3 years ago)

Author:

eugene

Message:

merge changes from trunk: new nonlinearity, allow detections under CTE masks

Location:

branches/eam_branches/ipp-20220316/ppImage

Files:

• . (modified) (1 prop)
• src/Makefile.am (modified) (1 diff)
• src/ppImage.h (modified) (3 diffs)
• src/ppImageArguments.c (modified) (2 diffs)
• src/ppImageDetrendFree.c (modified) (1 diff)
• src/ppImageDetrendNewNonLinear.c (copied) (copied from trunk/ppImage/src/ppImageDetrendNewNonLinear.c )
• src/ppImageDetrendNonLinear.c (modified) (1 diff)
• src/ppImageDetrendReadout.c (modified) (2 diffs)
• src/ppImageDetrendRecord.c (modified) (1 diff)
• src/ppImageOptions.c (modified) (2 diffs)
• src/ppImageParseCamera.c (modified) (1 diff)
• src/ppImageSetMaskBits.c (modified) (2 diffs)

branches/eam_branches/ipp-20220316/ppImage/src/Makefile.am

@@ -24 +24 @@
         ppImageDetrendRecord.c \
         ppImageDetrendNonLinear.c \
+        ppImageDetrendNewNonLinear.c \
         ppImageDetrendFringe.c \
         ppImageDetrendFree.c \

branches/eam_branches/ipp-20220316/ppImage/src/ppImage.h

@@ -35 +35 @@
     bool doAuxMask;                     // apply auxillary mask
     bool doNonLin;                      // Non-linearity correction
+    bool doNewNonLin;                   // Non-linearity correction
     bool doOverscan;                    // Overscan subtraction
     bool doNoiseMap;                    // Bias subtraction
@@ -157 +158 @@
 ppImageOptions *ppImageParseCamera(pmConfig *config);
 bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options);
+bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned
+                               psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned
+                               psMetadata *source  // Source of mask bits
+  );
 
 // apply the cell flips to the input data before analysis
@@ -171 +176 @@
 
 bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options);
+
+bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config);
 
 bool ppImageDetrendNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config);

branches/eam_branches/ipp-20220316/ppImage/src/ppImageArguments.c

@@ -25 +25 @@
     fprintf(stderr, "\t-fringe/-fringelist: Fringe image and data.\n");
     fprintf(stderr, "\t-linearity/-linearlist: linearity correction file.\n");
+    fprintf(stderr, "\t-newnonlin/-newnonlinlist: non-linearity correction file (v 2023.01).\n");
     fprintf(stderr, "\n");
     exit (2);
@@ -116 +117 @@
     pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist");
     pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist");
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "NEWNONLIN", "-newnonlin", "-newnonlinlist");
     pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined");
     pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.DEAD.CELLS", "-pattern-dead-cells", "-not-defined");

branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendFree.c

@@ -14 +14 @@
     "PPIMAGE.SHUTTER",
     "PPIMAGE.LINEARITY",
+    "PPIMAGE.NEWNONLIN",
     NULL
 };

branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendNonLinear.c

@@ -1 @@
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include "ppImage.h"
 

branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendReadout.c

@@ -149 +149 @@
       // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout"));
     }
+    // New Non-linearity correction (exclusive of the above)
+    if (options->doNewNonLin) {
+      if (!ppImageDetrendNewNonLinear(input, detview, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to correct Non-Linearity with new version (2023)");
+        psFree(detview);
+        return(false);
+      }
+      // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout"));
+    }
 
     // set up the dark and bias
@@ -213 +222 @@
             psImageUnbin (noiseImage, noiseMap->image, binning);
             psFree (binning);
-            // Stolen from pmSkySubtract.c
-            // CZW: Unneeded, as psImageUnbin does the bilinear interpolation?  It still looks blocky. 
-/*          psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME); // Recipe */
-/*          psAssert(recipe, "Should be there!"); */
-
-/*          psS32 xBinFactor = psMetadataLookupS32(NULL,recipe,"NOISE.XBIN"); */
-/*          psS32 yBinFactor = psMetadataLookupS32(NULL,recipe,"NOISE.YBIN"); */
-/*          psImageInterpolateOptions *interp = psImageInterpolateOptionsAlloc(PS_INTERPOLATE_BILINEAR, */
-/*                                                                             noiseMap->image, */
-/*                                                                             NULL, NULL, */
-/*                                                                             0, 0.0, 0.0, 0, 0, 0.0); */
-/*          for (psS32 row = 0; row < input->image->numRows; row++) { */
-/*            for (psS32 col = 0; col < input->image->numCols; col++) { */
-/*              // We calculate the F32 value of the pixel coordinates in the */
-/*              // binned image and then use a pixel interpolation routine to */
-/*              // determine the value of the pixel at that location. */
-/*              psF32 binRowF64 = ((psF32) row) / ((psF32) yBinFactor); */
-/*              psF32 binColF64 = ((psF32) col) / ((psF32) xBinFactor); */
-/*              // We add 0.5 to the pixel locations since the pixel */
-/*              // interpolation routine defines the location of pixel */
-/*              // (i, j) as (i+0.5, j+0.5). */
-/*              binRowF64+= 0.5; */
-/*              binColF64+= 0.5; */
-
-/*              double binPixel; */
-/*              if (!psImagePixelInterpolate(&binPixel, NULL, NULL, binColF64, binRowF64, interp)) { */
-/*                psError(PS_ERR_UNKNOWN, false, "Unable to interpolate image."); */
-/*                psFree(interp); */
-/*                psFree(noiseImage); */
-/*                return NULL; */
-/*              } */
-/*              noiseImage->data.F32[row][col] = binPixel; */
-/*            } */
-/*          } */
-/*          psFree(interp); */
         }
 

branches/eam_branches/ipp-20220316/ppImage/src/ppImageDetrendRecord.c

@@ -75 +75 @@
     detrendRecord(options->doFringe,   detrend, config, view, "PPIMAGE.FRINGE",   "DETREND.FRINGE",   "Fringe filename");
 
-    detrendRecord(options->doNonLin,   detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN",   "Non-linearity table filename");
+    detrendRecord(options->doNonLin,    detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN",   "Non-linearity table filename");
+    detrendRecord(options->doNewNonLin, detrend, config, view, "PPIMAGE.NEWNONLIN","DETREND.NEWNONLIN","Non-linearity table filename (v2023)");
 
     detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename");

branches/eam_branches/ipp-20220316/ppImage/src/ppImageOptions.c

@@ -28 +28 @@
     options->doAuxMask       = false;   // apply auxillary mask
     options->doNonLin        = false;   // Non-linearity correction
+    options->doNewNonLin     = false;   // New Non-linearity correction (v2023)
     options->doOverscan      = false;   // Overscan subtraction
     options->doNoiseMap      = false;   // Apply Read Noise Map
@@ -175 +176 @@
     // XXX PAP: The overscan stuff needs to be updated following the reworked API
 
+    // New Non-linearity (v 2023) recipe options
+    // non-linearity corrections are loaded from a file defined in the detrend system or on the command-line
+    if (psMetadataLookupBool(NULL, recipe, "NEWNONLIN")) {
+        options->doNewNonLin = true;
+    }
+
+    // XXX PAP: The overscan stuff needs to be updated following the reworked API
+
     // Overscan recipe options
     // XXX EAM : we should abort on invalid options. default options?

branches/eam_branches/ipp-20220316/ppImage/src/ppImageParseCamera.c

@@ -104 +104 @@
 
     if (options->doNonLin) {
-      if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.LINEARITY", "LINEARITY",
-                             PM_FPA_FILE_LINEARITY, PM_DETREND_TYPE_LINEARITY)) {
-        psError(PS_ERR_IO, false, "Can't find a non-linearity correction source");
-        psFree(options);
-        return NULL;
-      }
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.LINEARITY", "LINEARITY",
+                               PM_FPA_FILE_LINEARITY, PM_DETREND_TYPE_LINEARITY)) {
+            psError(PS_ERR_IO, false, "Can't find a non-linearity correction source");
+            psFree(options);
+            return NULL;
+        }
+    }
+    if (options->doNewNonLin) {
+        // if the file has been specified on the command-line (-newnonlin file), then the file
+        // is identified by the NEWNONLIN entry in config->arguments.  otherwise, load from the
+        // detrend system as type NEWNONLIN
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NEWNONLIN", "NEWNONLIN",
+                               PM_FPA_FILE_NEWNONLIN, PM_DETREND_TYPE_NEWNONLIN)) {
+            psError(PS_ERR_IO, false, "Can't find a new non-linearity correction source");
+            psFree(options);
+            return NULL;
+        }
     }
 

branches/eam_branches/ipp-20220316/ppImage/src/ppImageSetMaskBits.c

@@ -5 +5 @@
 #include "ppImage.h"
 
+// Structure to hold the properties of a mask value
+typedef struct {
+    char *badMaskName;                  // name for "bad" (i.e., mask me please) pixels
+    char *fallbackName;                 // Fallback name in case a bad mask name is not defined
+    psImageMaskType defaultMaskValue;   // Default value in case a bad mask name and its fallback are not defined
+    bool isBad; // include this value as part of the MASK.VALUE entry (generically bad)
+} pmConfigMaskInfo;
+
+static pmConfigMaskInfo ppimagemasks[] = {
+    // Features of the detector
+    { "DETECTOR",  NULL,       0x01, true }, // Something is wrong with the detector
+    { "FLAT",      "DETECTOR", 0x01, true }, // Pixel doesn't flat-field properly
+    { "DARK",      "DETECTOR", 0x01, true }, // Pixel doesn't dark-subtract properly
+    { "BLANK",     "DETECTOR", 0x01, true }, // Pixel doesn't contain valid data
+    { "CTE",       "DETECTOR", 0x01, false }, // Pixel has poor CTE
+    { "BURNTOOL",  NULL,       0x04, false }, // Pixel has been touched by burntool
+    // Invalid signal ranges
+    { "SAT",       NULL,       0x02, true  }, // Pixel is saturated or non-linear
+    { "LOW",       "SAT",      0x02, true  }, // Pixel is low
+    { "SUSPECT",   NULL,       0x04, false }, // Pixel is suspected of being bad
+    // Non-astronomical structures
+    { "CR",        NULL,       0x08, true  }, // Pixel contains a cosmic ray
+    { "SPIKE",     NULL,       0x08, false  }, // Pixel contains a diffraction spike
+    { "GHOST",     NULL,       0x08, false  }, // Pixel contains an optical ghost
+    { "STREAK",    NULL,       0x08, false  }, // Pixel contains a streak
+    { "CROSSTALK", NULL,       0x08, false  }, // Pixel contains crosstalk data
+    { "STARCORE",  NULL,       0x08, false  }, // Pixel contains a bright star core
+    // Effects of convolution and interpolation
+    { "CONV.BAD",  NULL,       0x02, true  }, // Pixel is bad after convolution with a bad pixel
+    { "CONV.POOR", NULL,       0x04, false }, // Pixel is poor after convolution with a bad pixel
+};
+
 bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options) {
 
-    if (!pmConfigMaskSetBits(&options->maskValue, &options->markValue, config)) {
-        psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values");
-        return false;
+
+    // Look up recipe values
+    psMetadata *pprecipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME);
+
+    psAssert(pprecipe, "We checked this earlier, so it should be here.");
+    bool doDetectCTE = psMetadataLookupBool(NULL, pprecipe, "DETECT.CTE"); // Do detections on pixels underneath CTE masks
+
+    // this function sets the required single-image mask bits
+    if(!doDetectCTE) {
+      if (!pmConfigMaskSetBits (&options->maskValue, &options->markValue, config)) {
+          psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values");
+          return false;
+      }
+    } else {
+      psMetadata *maskrecipe = psMetadataLookupMetadata(NULL, config->recipes, "MASKS"); // The recipe
+      if (!maskrecipe) {
+          psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find MASKS recipe.");
+          return false;
+      }
+      if (!ppImageMaskSetInMetadata(&options->maskValue, &options->markValue, maskrecipe)) {
+          psError (PS_ERR_UNKNOWN, true, "Unable to determine the mask value");
+          return false;
+      }
     }
 
@@ -66 +118 @@
     return true;
 }
+
+
+bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned
+                               psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned
+                               psMetadata *source  // Source of mask bits
+    )
+{
+    PS_ASSERT_METADATA_NON_NULL(source, false);
+
+    // Ensure all the bad mask names exist, and set the value to catch all bad pixels
+    psImageMaskType maskValue = 0;           // Value to mask to catch all the bad pixels
+    psImageMaskType allMasks = 0;            // Value to mask to catch all masked bits (to set MARK)
+
+    int nMasks = sizeof (ppimagemasks) / sizeof (pmConfigMaskInfo);
+
+    for (int i = 0; i < nMasks; i++) {
+        bool mdok;                      // Status of MD lookup
+        psImageMaskType value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].badMaskName); // Value of mask
+        if (!mdok) {
+            psWarning ("problem with mask value %s\n", ppimagemasks[i].badMaskName);
+        }
+
+        if (!value) {
+            if (ppimagemasks[i].fallbackName) {
+                value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].fallbackName);
+            }
+            if (!value) {
+                value = ppimagemasks[i].defaultMaskValue;
+            }
+            psMetadataAddImageMask(source, PS_LIST_TAIL, ppimagemasks[i].badMaskName, PS_META_REPLACE, NULL, value);
+        }
+        if (ppimagemasks[i].isBad) {
+            maskValue |= value;
+        }
+        allMasks |= value;
+    }
+
+    // search for an unset bit to use for MARK:
+    psImageMaskType markValue = 0x00;
+    psImageMaskType markTrial = 0x01;
+
+    int nBits = sizeof(psImageMaskType) * 8;
+    for (int i = 0; !markValue && (i < nBits); i++) {
+        if (allMasks & markTrial) {
+            markTrial <<= 1;
+        } else {
+            markValue = markTrial;
+        }
+    }
+    if (!markValue) {
+        psError (PS_ERR_UNKNOWN, true, "Unable to define the MARK bit mask: all bits taken!");
+        return false;
+    }
+
+    // update the list with the results
+    psMetadataAddImageMask(source, PS_LIST_TAIL, "MASK.VALUE", PS_META_REPLACE, NULL, maskValue);
+    psMetadataAddImageMask(source, PS_LIST_TAIL, "MARK.VALUE", PS_META_REPLACE, NULL, markValue);
+
+    if (outMaskValue) {
+        *outMaskValue = maskValue;
+    }
+    if (outMarkValue) {
+        *outMarkValue = markValue;
+    }
+
+    return true;
+}
+