Changeset 42379 for trunk/psModules/src/detrend

Timestamp:

Feb 8, 2023, 11:54:08 AM (3 years ago)

Author:

eugene

Message:

merge from eam_branches/ipp-20220316. no_warn strncpy; avoid passing NULL to sprintf %s; add code for PATTERN_DEAD_CELLS; fix error in PSF residual image evaluation; drop detailed mask analysis for binned images (not actually used)

Location:

trunk/psModules

Files:

• . (modified) (1 prop)
• src/detrend/pmDark.c (modified) (1 diff)
• src/detrend/pmDetrendDB.c (modified) (1 diff)
• src/detrend/pmDetrendDB.h (modified) (1 diff)
• src/detrend/pmOverscan.c (modified) (4 diffs)
• src/detrend/pmOverscan.h (modified) (1 diff)
• src/detrend/pmPattern.c (modified) (1 diff)
• src/detrend/pmPattern.h (modified) (1 diff)
• src/detrend/pmPatternIO.c (modified) (2 diffs)
• src/detrend/pmPatternIO.h (modified) (1 diff)

trunk/psModules/src/detrend/pmDark.c

@@ -129 +129 @@
     } else {
         if (!ordinateParseConcept(value, readout, name)) {
-            psError(PM_ERR_CONFIG, false, "trouble parsing rule %s for DARK.ORDINATE %s", rule, name);
+            psError(PM_ERR_CONFIG, false, "trouble parsing rule %s for DARK.ORDINATE %s", "NULL", name);
             return false;
         }

trunk/psModules/src/detrend/pmDetrendDB.c

@@ -113 +113 @@
         DETREND_STRING_CASE(KH_CORRECT);
         DETREND_STRING_CASE(PATTERN_ROW_AMP);
+        DETREND_STRING_CASE(PATTERN_DEAD_CELLS);
     default:
         return NULL;

trunk/psModules/src/detrend/pmDetrendDB.h

@@ -43 +43 @@
     PM_DETREND_TYPE_KH_CORRECT,
     PM_DETREND_TYPE_PATTERN_ROW_AMP,
+    PM_DETREND_TYPE_PATTERN_DEAD_CELLS,
 } pmDetrendType;
 

trunk/psModules/src/detrend/pmOverscan.c

@@ -38 +38 @@
     opts->order = order;
     opts->stat = psMemIncrRefCounter(stat);
+
+    opts->minValid = 0.0; // default value if not defined
+    opts->maxValid = (float) 0x10000; // default value if not defined
+    opts->maskVal = 0x0001; // default value if not defined
 
     // Smoothing
@@ -314 +318 @@
         psFree(reducedScalar);
 
+        // EAM 2022.03.29 : if the calculated overscan value is below the threshold,
+        // declare the readout dead and mask
+
+        if ((reduced < overscanOpts->minValid) || (reduced > overscanOpts->maxValid)) {
+            fprintf (stderr, "bad overscan (1) %f, masking readout\n", reduced);
+            psImage *mask = input->mask;
+            for (int y = 0; y < mask->numRows; y++) {
+                for (int x = 0; x < mask->numCols; x++) {
+                    mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal;
+                }
+            }
+        }
+
         psFree(stats);
         return true;
@@ -371 +388 @@
         // generate stats of overscan vector for header
         { 
-          psString comment = NULL;    // Comment to add
-          psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
-          if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) {
-              psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
-              return false;
-          }
-          psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean);
-          psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, "");
-          psFree(comment);
-
-          // write metadata header value
-          psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean);
-          psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev);
-          psFree (vectorStats);
+            psString comment = NULL;    // Comment to add
+            psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
+            if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) {
+                psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+                return false;
+            }
+            psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean);
+            psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, "");
+            psFree(comment);
+
+            // write metadata header value
+            psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean);
+            psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev);
+
+            // EAM 2022.03.29 : if the calculated overscan value is below the threshold,
+            // declare the readout dead and mask
+          
+            if ((vectorStats->sampleMean < overscanOpts->minValid) || (vectorStats->sampleMean > overscanOpts->maxValid)) {
+                fprintf (stderr, "bad overscan (2) %f, masking readout\n", vectorStats->sampleMean);
+                psImage *mask = input->mask;
+                for (int y = 0; y < mask->numRows; y++) {
+                    for (int x = 0; x < mask->numCols; x++) {
+                        mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal;
+                    }
+                }
+            }
+
+            psFree (vectorStats);
         }
 
@@ -469 +500 @@
           psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean);
           psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev);
+
+          // EAM 2022.03.29 : if the calculated overscan value is below the threshold,
+          // declare the readout dead and mask
+          
+          if ((vectorStats->sampleMean < overscanOpts->minValid) || (vectorStats->sampleMean > overscanOpts->maxValid)) {
+              fprintf (stderr, "bad overscan (3) %f, masking readout\n", vectorStats->sampleMean);
+              psImage *mask = input->mask;
+              for (int y = 0; y < mask->numRows; y++) {
+                  for (int x = 0; x < mask->numCols; x++) {
+                      mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal;
+                  }
+              }
+          }
           psFree (vectorStats);
         }

trunk/psModules/src/detrend/pmOverscan.h

@@ -43 +43 @@
     int boxcar;                         ///< Boxcar smoothing radius
     float gauss;                        ///< Gaussian smoothing sigma
+    float minValid;                     ///< if overscan is too low, readout is dead : mask
+    float maxValid;                     ///< if overscan is too high, readout is dead : mask
+    psImageMaskType maskVal;            ///< Mask value to give dead readouts
+
     // Outputs
     psPolynomial1D *poly;               ///< Result of polynomial fit

trunk/psModules/src/detrend/pmPattern.c

@@ -1005 +1005 @@
     psFree(meanMask);
 
+    return true;
+}
+
+// Compare the backs (cellBackgrounds) vector to each of the patterns loaded for this chip.
+// Choose the one that matches best & mask as appropriate
+// can we pass the backgrounds in using an image (8 x 8)
+bool pmPatternDeadCells(pmChip *chip, const psVector *backs, psImageMaskType maskVal)
+{
+    PS_ASSERT_PTR_NON_NULL(chip, false);
+    PS_ASSERT_VECTOR_NON_NULL(backs, false);
+    PS_ASSERT_VECTOR_SIZE(backs, chip->cells->n, false);
+    PS_ASSERT_VECTOR_TYPE(backs, PS_TYPE_F32, false);
+
+    // Look for the dead cell pattern cube in the analysis metadata.
+    // NOTE: not all chips have the pattern, skip if not found.
+    bool mdok;
+    psImage *deadCellPattern = (psImage *) psMetadataLookupPtr (&mdok, chip->analysis, "PTN.DEAD.CELL");
+    if (!mdok) {
+        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No DEAD CELL pattern for chip, skipping\n");
+        return true;
+    }
+
+    int numCells = backs->n;            // Number of cells
+    int numColsD = deadCellPattern->numCols;
+    int numRowsD = deadCellPattern->numRows;
+
+    assert (backs->n == numRowsD);
+
+    // The background values need to be normalized (divide by the median).
+    // First extract the valid data values
+    psVector *valid = psVectorAllocEmpty(backs->n, PS_TYPE_F32); // Mean for each cell
+    for (int i = 0; i < backs->n; i++) {
+        float value = backs->data.F32[i];
+        if (!isfinite(value)) { backs->data.F32[i] = NAN; continue; }
+        if (value > 50000.0)  { backs->data.F32[i] = NAN; continue; } // XXX warning: hard-wired value
+        psVectorAppend (valid, value);
+    }
+    if (valid->n == 0) {
+        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No valid backgrounds, skipping\n");
+        psFree (valid);
+        return true;
+    }
+
+    // Second, calculate the median
+    psVectorSortInPlace (valid);
+    int midPt = valid->n / 2.0;
+    float median = valid->n % 2 ? valid->data.F32[midPt] : 0.5*(valid->data.F32[midPt] + valid->data.F32[midPt-1]);
+    psFree (valid);
+
+    // Finally, renormalize:
+    for (int i = 0; i < backs->n; i++) {
+        if (!isfinite(backs->data.F32[i])) { continue; }
+        backs->data.F32[i] /= median;
+    }
+
+    // there are 2 columns (value & mask) for each mode, plus the constant mode
+    int nModes = numColsD / 2 + 1;
+    
+    psVector *stdev = psVectorAlloc(nModes, PS_TYPE_F32); // Mean for each cell
+
+    // measure stdev for each comparison
+    for (int i = 0; i < nModes; i++) {
+        float Sum1 = 0.0;
+        float Sum2 = 0.0;
+        int   Npts = 0;
+
+        // choose the column with the background values for this mode
+        int nModeColumn = 2*(i - 1);
+
+        for (int j = 0; j < backs->n; j++) {
+
+            float valObs = backs->data.F32[j];
+            float valRef = (i == 0) ? 0.0 : deadCellPattern->data.F32[j][nModeColumn];
+
+            if (!isfinite(valObs)) continue;
+            if (!isfinite(valRef)) continue;
+
+            float dS = valObs - valRef;
+            Sum1 += dS;
+            Sum2 += dS*dS;
+            Npts ++;
+        }
+        if (Npts == 0) {
+            // is this is an error?
+            // no valid data, ignore this test
+            psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No valid backgrounds, skipping\n");
+            psFree (stdev);
+            return true;
+        }
+
+        float mean = Sum1 / Npts;
+        float sigma = sqrt(Sum2 / Npts - mean*mean);
+        stdev->data.F32[i] = sigma;
+        fprintf (stderr, "mode: %d, stdev: %f\n", i, sigma);
+    }
+
+    // loop over stdev and choose the lowest one
+    int   minI = 0;
+    float minV = stdev->data.F32[minI];
+
+    for (int i = 1; i < nModes; i++) {
+        if (stdev->data.F32[i] < minV) {
+            minI = i;
+            minV = stdev->data.F32[i];
+        }
+    }
+    psFree (stdev);
+
+    if (minI == 0) return true;
+
+    int nModeColumnMask = 2*(minI - 1) + 1;
+
+    // now mask bad cells
+    for (int i = 0; i < numCells; i++) {
+        if (deadCellPattern->data.F32[i][nModeColumnMask] > 0) continue;
+
+        pmCell *cell = chip->cells->data[i]; // Cell of interest
+        pmReadout *ro = cell->readouts->data[0]; // Readout of interest
+
+        psImage *mask = ro->mask; // mask of interest
+        int numCols = mask->numCols, numRows = mask->numRows; // Size of image
+        
+        for (int y = 0; y < numRows; y++) {
+            for (int x = 0; x < numCols; x++) {
+                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskVal;
+            }
+        }
+    }
     return true;
 }

trunk/psModules/src/detrend/pmPattern.h

@@ -86 +86 @@
 
 
+bool pmPatternDeadCells(pmChip *chip, const psVector *backs, psImageMaskType maskVal);
+
 /// @}
 #endif

trunk/psModules/src/detrend/pmPatternIO.c

@@ -482 +482 @@
 }
 
-// read the set of tables, one for each chip
+// Read the set of tables, one for each chip.  The values are saved on the cell->analysis
+// metadata of the pmFPAfile associated with the pattern file.  Later, when this is used (e.g.,
+// ppImageDetrendPatternRowApply), the values are transferred to the cell->analysis metadata of
+// the pmFPAfile for the image being processed.
 bool pmPatternRowAmpReadChips (pmFPAfile *file) {
 
@@ -560 +563 @@
     return true;
 }
+
+/**************** PatternDeadCells I/O *************************/
+
+bool pmPatternDeadCellsRead (const pmFPAview *view, pmFPAfile *file, const pmConfig *config)
+    {
+        // read the full model in one pass: require the level to be FPA
+        if (view->chip != -1) {
+            psError(PS_ERR_IO, false, "Pattern Dead Cells must be read at the FPA level");
+            return false;
+        }
+
+        if (!pmPatternDeadCellsReadFPA (file)) {
+            psError(PS_ERR_IO, false, "Failed to read Pattern Dead Cells for fpa");
+            return false;
+        }
+        return true;
+    }
+
+// read in all chip-level Pattern Dead Cells data for this FPA
+bool pmPatternDeadCellsReadFPA (pmFPAfile *file) {
+
+    if (!pmPatternDeadCellsReadChips (file)) {
+        psError(PS_ERR_IO, false, "Failed to read Pattern Dead Cells for chips");
+        return false;
+    }
+
+    return true;
+}
+
+// Read the set of dead cell image cubes, one for each chip.  The values are saved on the
+// chip->analysis metadata of the pmFPAfile associated with the pattern file.  Later, when this
+// is used (e.g., ppImageDetrendPatternDeadCellsApply), the values are transferred to the
+// chip->analysis metadata of the pmFPAfile for the image being processed.
+bool pmPatternDeadCellsReadChips (pmFPAfile *file) {
+
+    bool haveData, status;
+
+    // loop over the extensions
+    // for each extension, use the extname (eg, XY01.ded) to assign to a chip
+
+    // move to the start of the file
+    haveData = psFitsMoveExtNum (file->fits, 1, false);
+    if (!haveData) {
+        psError(PS_ERR_IO, false, "Failed to read even the first extension?");
+        return false;
+    }
+
+    int nGood = 0;
+    while (haveData) {
+
+        // load the header
+        psMetadata *header = psFitsReadHeader(NULL, file->fits); // The FITS header
+        if (!header) psAbort("cannot read dead cell header");
+
+        // load the full model in one shot
+        psImage *deadCellData = psFitsReadImage(file->fits, psRegionSet(0,0,0,0), 0); // dead cell patterns
+        if (!deadCellData) psAbort("cannot read dead cell pattern");
+        
+        // determine the chip (not all chips have DEAD CELL patterns)
+        char *extname = psMetadataLookupStr (&status, header, "EXTNAME");
+        psLogMsg ("psModules.detrend", 8, "read dead cell pattern for extname %s\n", extname);
+
+        // I expect to find a name of the form: chipName.ded (eg, XY01.ded)
+        // where chipName like 'XY01'
+        psAssert (strlen(extname) == 8, "invalid extension %s", extname);
+        psAssert (extname[5] == 'd', "invalid extension %s", extname);
+        psAssert (extname[6] == 'e', "invalid extension %s", extname);
+        psAssert (extname[7] == 'd', "invalid extension %s", extname);
+
+        char chipName[5];
+        strncpy (chipName, extname, 4);
+        chipName[4] = 0;
+
+        pmChip *chip = pmConceptsChipFromName (file->fpa, chipName);
+        if (!chip) psAbort ("invalid chip?");
+
+        psMetadataAddImage (chip->analysis, PS_LIST_TAIL, "PTN.DEAD.CELL", PS_META_REPLACE, "", deadCellData);
+        psFree (deadCellData);
+        psFree (header);
+
+        // move to the next extension
+        haveData = psFitsMoveExtNum (file->fits, 1, true);
+        nGood ++;
+    }
+    psLogMsg ("psModules.detrend", 4, "read patterns for %d chips from Pattern Dead Cells file\n", nGood);
+
+    return true;
+}

trunk/psModules/src/detrend/pmPatternIO.h

@@ -40 +40 @@
 bool pmPatternRowAmpReadChips (pmFPAfile *file);
 
+bool pmPatternDeadCellsRead (const pmFPAview *view, pmFPAfile *file, const pmConfig *config);
+bool pmPatternDeadCellsReadFPA (pmFPAfile *file);
+bool pmPatternDeadCellsReadChips (pmFPAfile *file);
+
 #endif