Changeset 42168 for branches/eam_branches/ipp-20220316/psModules

Timestamp:

Apr 13, 2022, 10:54:54 AM (4 years ago)

Author:

eugene

Message:

add PATTERN.DEAD.CELLS functions in pmPattern and related I/O bits

Location:

branches/eam_branches/ipp-20220316/psModules/src

Files:

• camera/pmFPAfile.c (modified) (2 diffs)
• camera/pmFPAfile.h (modified) (1 diff)
• camera/pmFPAfileFitsIO.c (modified) (1 diff)
• camera/pmFPAfileIO.c (modified) (8 diffs)
• detrend/pmDetrendDB.c (modified) (1 diff)
• detrend/pmDetrendDB.h (modified) (1 diff)
• detrend/pmPattern.c (modified) (1 diff)
• detrend/pmPattern.h (modified) (1 diff)
• detrend/pmPatternIO.c (modified) (2 diffs)
• detrend/pmPatternIO.h (modified) (1 diff)

branches/eam_branches/ipp-20220316/psModules/src/camera/pmFPAfile.c

@@ -554 +554 @@
     if (!strcasecmp(type, "PATTERN.ROW.AMP"))     {
         return PM_FPA_FILE_PATTERN_ROW_AMP;
+    }
+    if (!strcasecmp(type, "PATTERN.DEAD.CELLS"))     {
+        return PM_FPA_FILE_PATTERN_DEAD_CELLS;
     }
     if (!strcasecmp(type, "SUBKERNEL"))     {
@@ -611 +614 @@
       case PM_FPA_FILE_PATTERN_ROW_AMP:
         return ("PATTERN.ROW.AMP");
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
+        return ("PATTERN.DEAD.CELLS");
       case PM_FPA_FILE_SUBKERNEL:
         return ("SUBKERNEL");

branches/eam_branches/ipp-20220316/psModules/src/camera/pmFPAfile.h

@@ -53 +53 @@
     PM_FPA_FILE_PATTERN,
     PM_FPA_FILE_PATTERN_ROW_AMP,
+    PM_FPA_FILE_PATTERN_DEAD_CELLS,
     PM_FPA_FILE_LINEARITY,
     PM_FPA_FILE_EXPNUM,

branches/eam_branches/ipp-20220316/psModules/src/camera/pmFPAfileFitsIO.c

@@ -153 +153 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
         {
           pmHDU *hdu = pmFPAviewThisHDU(view, fpa);

branches/eam_branches/ipp-20220316/psModules/src/camera/pmFPAfileIO.c

@@ -256 +256 @@
         status = pmPatternRowAmpRead(view, file, config);
         break;
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
+        status = pmPatternDeadCellsRead(view, file, config);
+        break;
       case PM_FPA_FILE_SX:
       case PM_FPA_FILE_RAW:
@@ -367 +370 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
       case PM_FPA_FILE_JPEG:
       case PM_FPA_FILE_KAPA:
@@ -459 +463 @@
       return true;
     }
+    if (file->type == PM_FPA_FILE_PATTERN_DEAD_CELLS) {
+      psTrace("psModules.camera", 6, "skip write for %s, no write function defined", file->name);
+      return true;
+    }
 
     // open the file if not yet opened
@@ -559 +567 @@
       case PM_FPA_FILE_PATTERN_ROW_AMP:
         psError(PS_ERR_IO, true, "cannot write type PATTERN.ROW.AMP (%s)", file->name);
+        break;
+
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
+        psError(PS_ERR_IO, true, "cannot write type PATTERN.DEAD.CELLS (%s)", file->name);
         break;
 
@@ -637 +649 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
       case PM_FPA_FILE_LINEARITY:
       case PM_FPA_FILE_EXPNUM:
@@ -716 +729 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
       case PM_FPA_FILE_EXPNUM:
         psTrace ("psModules.camera", 6, "NOT freeing %s (%s) : save for further analysis\n", file->filename, file->name);
@@ -880 +894 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
       case PM_FPA_FILE_LINEARITY:
       case PM_FPA_FILE_EXPNUM:
@@ -1085 +1100 @@
       case PM_FPA_FILE_KH_CORRECT:
       case PM_FPA_FILE_PATTERN_ROW_AMP:
+      case PM_FPA_FILE_PATTERN_DEAD_CELLS:
       case PM_FPA_FILE_SX:
       case PM_FPA_FILE_RAW:

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmDetrendDB.c

@@ -113 +113 @@
         DETREND_STRING_CASE(KH_CORRECT);
         DETREND_STRING_CASE(PATTERN_ROW_AMP);
+        DETREND_STRING_CASE(PATTERN_DEAD_CELLS);
     default:
         return NULL;

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmDetrendDB.h

@@ -42 +42 @@
     PM_DETREND_TYPE_KH_CORRECT,
     PM_DETREND_TYPE_PATTERN_ROW_AMP,
+    PM_DETREND_TYPE_PATTERN_DEAD_CELLS,
 } pmDetrendType;
 

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmPattern.c

@@ -1005 +1005 @@
     psFree(meanMask);
 
+    return true;
+}
+
+// Compare the backs (cellBackgrounds) vector to each of the patterns loaded for this chip.
+// Choose the one that matches best & mask as appropriate
+// can we pass the backgrounds in using an image (8 x 8)
+bool pmPatternDeadCells(pmChip *chip, const psVector *backs, psImageMaskType maskVal)
+{
+    PS_ASSERT_PTR_NON_NULL(chip, false);
+    PS_ASSERT_VECTOR_NON_NULL(backs, false);
+    PS_ASSERT_VECTOR_SIZE(backs, chip->cells->n, false);
+    PS_ASSERT_VECTOR_TYPE(backs, PS_TYPE_F32, false);
+
+    // Look for the dead cell pattern cube in the analysis metadata.
+    // NOTE: not all chips have the pattern, skip if not found.
+    bool mdok;
+    psImage *deadCellPattern = (psImage *) psMetadataLookupPtr (&mdok, chip->analysis, "PTN.DEAD.CELL");
+    if (!mdok) {
+        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No DEAD CELL pattern for chip, skipping\n");
+        return true;
+    }
+
+    int numCells = backs->n;            // Number of cells
+    int numColsD = deadCellPattern->numCols;
+    int numRowsD = deadCellPattern->numRows;
+
+    assert (backs->n == numRowsD);
+
+    // The background values need to be normalized (divide by the median).
+    // First extract the valid data values
+    psVector *valid = psVectorAllocEmpty(backs->n, PS_TYPE_F32); // Mean for each cell
+    for (int i = 0; i < backs->n; i++) {
+        float value = backs->data.F32[i];
+        if (!isfinite(value)) { backs->data.F32[i] = NAN; continue; }
+        if (value > 50000.0)  { backs->data.F32[i] = NAN; continue; } // XXX warning: hard-wired value
+        psVectorAppend (valid, value);
+    }
+    if (valid->n == 0) {
+        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No valid backgrounds, skipping\n");
+        psFree (valid);
+        return true;
+    }
+
+    // Second, calculate the median
+    psVectorSortInPlace (valid);
+    int midPt = valid->n / 2.0;
+    float median = valid->n % 2 ? valid->data.F32[midPt] : 0.5*(valid->data.F32[midPt] + valid->data.F32[midPt-1]);
+    psFree (valid);
+
+    // Finally, renormalize:
+    for (int i = 0; i < backs->n; i++) {
+        if (!isfinite(backs->data.F32[i])) { continue; }
+        backs->data.F32[i] /= median;
+    }
+
+    // there are 2 columns (value & mask) for each mode, plus the constant mode
+    int nModes = numColsD / 2 + 1;
+    
+    psVector *stdev = psVectorAlloc(nModes, PS_TYPE_F32); // Mean for each cell
+
+    // measure stdev for each comparison
+    for (int i = 0; i < nModes; i++) {
+        float Sum1 = 0.0;
+        float Sum2 = 0.0;
+        int   Npts = 0;
+
+        // choose the column with the background values for this mode
+        int nModeColumn = 2*(i - 1);
+
+        for (int j = 0; j < backs->n; j++) {
+
+            float valObs = backs->data.F32[j];
+            float valRef = (i == 0) ? 0.0 : deadCellPattern->data.F32[j][nModeColumn];
+
+            if (!isfinite(valObs)) continue;
+            if (!isfinite(valRef)) continue;
+
+            float dS = valObs - valRef;
+            Sum1 += dS;
+            Sum2 += dS*dS;
+            Npts ++;
+        }
+        if (Npts == 0) {
+            // is this is an error?
+            // no valid data, ignore this test
+            psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No valid backgrounds, skipping\n");
+            psFree (stdev);
+            return true;
+        }
+
+        float mean = Sum1 / Npts;
+        float sigma = sqrt(Sum2 / Npts - mean*mean);
+        stdev->data.F32[i] = sigma;
+        fprintf (stderr, "mode: %d, stdev: %f\n", i, sigma);
+    }
+
+    // loop over stdev and choose the lowest one
+    int   minI = 0;
+    float minV = stdev->data.F32[minI];
+
+    for (int i = 1; i < nModes; i++) {
+        if (stdev->data.F32[i] < minV) {
+            minI = i;
+            minV = stdev->data.F32[i];
+        }
+    }
+    psFree (stdev);
+
+    if (minI == 0) return true;
+
+    int nModeColumnMask = 2*(minI - 1) + 1;
+
+    // now mask bad cells
+    for (int i = 0; i < numCells; i++) {
+        if (deadCellPattern->data.F32[i][nModeColumnMask] > 0) continue;
+
+        pmCell *cell = chip->cells->data[i]; // Cell of interest
+        pmReadout *ro = cell->readouts->data[0]; // Readout of interest
+
+        psImage *mask = ro->mask; // mask of interest
+        int numCols = mask->numCols, numRows = mask->numRows; // Size of image
+        
+        for (int y = 0; y < numRows; y++) {
+            for (int x = 0; x < numCols; x++) {
+                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskVal;
+            }
+        }
+    }
     return true;
 }

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmPattern.h

@@ -86 +86 @@
 
 
+bool pmPatternDeadCells(pmChip *chip, const psVector *backs, psImageMaskType maskVal);
+
 /// @}
 #endif

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmPatternIO.c

@@ -482 +482 @@
 }
 
-// read the set of tables, one for each chip
+// Read the set of tables, one for each chip.  The values are saved on the cell->analysis
+// metadata of the pmFPAfile associated with the pattern file.  Later, when this is used (e.g.,
+// ppImageDetrendPatternRowApply), the values are transferred to the cell->analysis metadata of
+// the pmFPAfile for the image being processed.
 bool pmPatternRowAmpReadChips (pmFPAfile *file) {
 
@@ -560 +563 @@
     return true;
 }
+
+/**************** PatternDeadCells I/O *************************/
+
+bool pmPatternDeadCellsRead (const pmFPAview *view, pmFPAfile *file, const pmConfig *config)
+    {
+        // read the full model in one pass: require the level to be FPA
+        if (view->chip != -1) {
+            psError(PS_ERR_IO, false, "Pattern Dead Cells must be read at the FPA level");
+            return false;
+        }
+
+        if (!pmPatternDeadCellsReadFPA (file)) {
+            psError(PS_ERR_IO, false, "Failed to read Pattern Dead Cells for fpa");
+            return false;
+        }
+        return true;
+    }
+
+// read in all chip-level Pattern Dead Cells data for this FPA
+bool pmPatternDeadCellsReadFPA (pmFPAfile *file) {
+
+    if (!pmPatternDeadCellsReadChips (file)) {
+        psError(PS_ERR_IO, false, "Failed to read Pattern Dead Cells for chips");
+        return false;
+    }
+
+    return true;
+}
+
+// Read the set of dead cell image cubes, one for each chip.  The values are saved on the
+// chip->analysis metadata of the pmFPAfile associated with the pattern file.  Later, when this
+// is used (e.g., ppImageDetrendPatternDeadCellsApply), the values are transferred to the
+// chip->analysis metadata of the pmFPAfile for the image being processed.
+bool pmPatternDeadCellsReadChips (pmFPAfile *file) {
+
+    bool haveData, status;
+
+    // loop over the extensions
+    // for each extension, use the extname (eg, XY01.ded) to assign to a chip
+
+    // move to the start of the file
+    haveData = psFitsMoveExtNum (file->fits, 1, false);
+    if (!haveData) {
+        psError(PS_ERR_IO, false, "Failed to read even the first extension?");
+        return false;
+    }
+
+    int nGood = 0;
+    while (haveData) {
+
+        // load the header
+        psMetadata *header = psFitsReadHeader(NULL, file->fits); // The FITS header
+        if (!header) psAbort("cannot read dead cell header");
+
+        // load the full model in one shot
+        psImage *deadCellData = psFitsReadImage(file->fits, psRegionSet(0,0,0,0), 0); // dead cell patterns
+        if (!deadCellData) psAbort("cannot read dead cell pattern");
+        
+        // determine the chip (not all chips have DEAD CELL patterns)
+        char *extname = psMetadataLookupStr (&status, header, "EXTNAME");
+        psLogMsg ("psModules.detrend", 8, "read dead cell pattern for extname %s\n", extname);
+
+        // I expect to find a name of the form: chipName.ded (eg, XY01.ded)
+        // where chipName like 'XY01'
+        psAssert (strlen(extname) == 8, "invalid extension %s", extname);
+        psAssert (extname[5] == 'd', "invalid extension %s", extname);
+        psAssert (extname[6] == 'e', "invalid extension %s", extname);
+        psAssert (extname[7] == 'd', "invalid extension %s", extname);
+
+        char chipName[5];
+        strncpy (chipName, extname, 4);
+        chipName[4] = 0;
+
+        pmChip *chip = pmConceptsChipFromName (file->fpa, chipName);
+        if (!chip) psAbort ("invalid chip?");
+
+        psMetadataAddImage (chip->analysis, PS_LIST_TAIL, "PTN.DEAD.CELL", PS_META_REPLACE, "", deadCellData);
+        psFree (deadCellData);
+        psFree (header);
+
+        // move to the next extension
+        haveData = psFitsMoveExtNum (file->fits, 1, true);
+        nGood ++;
+    }
+    psLogMsg ("psModules.detrend", 4, "read patterns for %d chips from Pattern Dead Cells file\n", nGood);
+
+    return true;
+}

branches/eam_branches/ipp-20220316/psModules/src/detrend/pmPatternIO.h

@@ -40 +40 @@
 bool pmPatternRowAmpReadChips (pmFPAfile *file);
 
+bool pmPatternDeadCellsRead (const pmFPAview *view, pmFPAfile *file, const pmConfig *config);
+bool pmPatternDeadCellsReadFPA (pmFPAfile *file);
+bool pmPatternDeadCellsReadChips (pmFPAfile *file);
+
 #endif