Changeset 41894 for trunk/ppImage

Timestamp:

Nov 4, 2021, 6:07:14 PM (5 years ago)

Author:

eugene

Message:

merge changes from eam_branches/ipp-dev-20210817 (threaded analysis of pattern row, add pattern row amplitude file I/O)

Location:

trunk/ppImage/src

Files:

• Makefile.am (modified) (1 diff)
• ppImage.h (modified) (2 diffs)
• ppImageArguments.c (modified) (1 diff)
• ppImageDetrendPattern.c (modified) (5 diffs)
• ppImageParseCamera.c (modified) (2 diffs)
• ppImageSetThreads.c (modified) (1 diff)

trunk/ppImage/src/Makefile.am

@@ -42 +42 @@
         ppImageDefineFile.c \
         ppImageSetMaskBits.c \
+        ppImageSetThreads.c \
         ppImageBurntoolMask.c \
         ppImageParityFlip.c \

trunk/ppImage/src/ppImage.h

@@ -193 +193 @@
 bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
 
-bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options);
+bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options);
+bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options);
 
 // Do background continuity step
@@ -326 +327 @@
 bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options, psMetadata *stats);
 
+bool ppImageSetThreads (void);
+
 #endif

trunk/ppImage/src/ppImageArguments.c

@@ -116 +116 @@
     pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist");
     pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist");
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined");
 
     if ((argnum = psArgumentGet(argc, argv, "-burntool"))) {

trunk/ppImage/src/ppImageDetrendPattern.c

@@ -14 +14 @@
 
 bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView,
-                                const ppImageOptions *options)
+                                ppImageOptions *options)
 {
     pmCell *cell = NULL;
@@ -31 +31 @@
     // CELLS in the format:CHIPS metadata table)
 
+    // New 2021 October : During ppImageParseCamera, we have loaded a file identified by
+    // PPIMAGE.PATTERN.ROW.AMP on config->files.  This file contains a collection of FITS
+    // tables, one per chip.  Each table lists the typical bias-drift amplitude for cell,
+    // measured from a collection of dark images.  These values are passed (via
+    // cell->analysis) to pmPatternRow which choosed to apply the correction based on the
+    // ratio of the poisson noise due to the median background (+ read noise) and the
+    // typical amplitude.  Tests show that if the amplitude / noise < 1/6, then the
+    // row-by-row variations are insignificant.
+
     // We also check the chip header for the boolean 'PTRN_ROW' : if this is true, we have
     // already applied this correct to this data, so we simply skip the correction for this
@@ -52 +61 @@
 
         pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT");
+
+        // grab the PATTERN.ROW.AMP file
+        pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP");
+        if (!PRAfile) {
+          psLogMsg("ppImage", PS_LOG_INFO, "Pattern Row Amplitude file not found, applying to all (with limits from ROW.SUBSET) ");
+        }
+        
         pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
         *view = *inputView;
@@ -75 +91 @@
             }
 
+            // grab the corresponding cell
+            if (PRAfile) {
+              pmCell *PRAcell = pmFPAviewThisCell (view, PRAfile->fpa);
+              psAssert (PRAcell, "found Pattern Row Amplitude file, but not cell?");
+
+              // find the nominal signal amplitude (check the ghost and/or crosstalk recipe file)
+              float amplitude = psMetadataLookupF32 (&status, PRAcell->analysis, "PTN.ROW.AMP");
+              if (!status) amplitude = NAN;
+            
+              // put the value on the science cell
+              psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "PTN.ROW.AMP", PS_META_REPLACE, "", amplitude);
+            }
+
             bool doPattern = false;
             if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.ROW.SUBSET")) {
                 ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
             }
-            //psWarning ("Row: %d\n", doPattern);
             if (!doPattern) continue;
 
-            // A very detailed log message. 
-            const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
-            const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");
-            psLogMsg("ppImage", PS_LOG_INFO, "Performing row pattern correction for %s, %s\n", chipName, cellName);
-
-            // process each of the readouts
-            pmReadout *readout;         // Readout from cell
-            while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) {
-                if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
-                    ESCAPE(false, "load failure for Readout");
-                }
-                if (!readout->data_exists) {
-                    continue;
-                }
-
-                // Perform pattern correction
-                if (!pmPatternRow(readout, options->patternRowOrder, options->patternRowIter,
-                                  options->patternRowRej, options->patternRowThresh, options->patternRowMean,
-                                  options->patternRowStdev, options->maskValue, options->darkMask)) {
-                    psFree(view);
-                    return(false);
-                }
-            }
-
-        }
+            // switch to test threaded version
+            if (true) {
+                // I need to allocate a view here to be freed by the
+                // called function below.  
+                pmFPAview *myView = pmFPAviewAlloc(0); // View for local processing
+                *myView = *view;
+
+                // allocate a job, construct the arguments for this job
+                psThreadJob *job = psThreadJobAlloc("PPIMAGE_PATTERN_ROW_CELL");
+                psArrayAdd(job->args, 1, config);
+                psArrayAdd(job->args, 1, input->fpa);
+                psArrayAdd(job->args, 1, chip);
+                psArrayAdd(job->args, 1, cell);
+                psArrayAdd(job->args, 1, myView);
+                psArrayAdd(job->args, 1, options);
+                if (!psThreadJobAddPending(job)) {
+                    return false;
+                }
+            } else {
+                // bump the counter since it must be freed by the function below.  
+                psMemIncrRefCounter (view); // View for local processing
+                if (!ppImageDetrendPatternApplyCell (config, input->fpa, chip, cell, view, options)) {
+                    return false;
+                }
+            }
+
+        }
+
+        // wait here for the threaded jobs to finish
+        // if no threads are allocated, this 
+        if (!psThreadPoolWait(true, true)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction.");
+            return false;
+        }
+
         psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_ROW",PS_META_REPLACE,"PATTERN.ROW correction applied",true);
         psFree(view);
@@ -266 +304 @@
     return false;
 }
+
+// thread safety : 
+bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options) {
+  
+    // process each of the readouts
+    pmReadout *readout;         // Readout from cell
+    while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+        if (!readout->data_exists) {
+            continue;
+        }
+
+        // Perform pattern correction
+        if (!pmPatternRow(readout, options->patternRowOrder, options->patternRowIter,
+                          options->patternRowRej, options->patternRowThresh, options->patternRowMean,
+                          options->patternRowStdev, options->maskValue, options->darkMask)) {
+            psFree (view);
+            return false;
+        }
+    }
+    psFree (view); // supplied view must be freeable
+    return true;
+}
+

trunk/ppImage/src/ppImageParseCamera.c

@@ -117 +117 @@
         pmDetrendSetThreadTasks(nScanRows);
     }
+    ppImageSetThreads (); // even if threading is off, we need to identify the ppImageDetrendPattern thread function
 
     if (options->doPatternRow || options->doPatternCell) {
@@ -180 +181 @@
             psError (PS_ERR_IO, false, "Can't find a fringe image source");
             return NULL;
+        }
+    }
+
+    if (options->doPatternRow) {
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.PATTERN.ROW.AMP", "PATTERN.ROW.AMP",
+                               PM_FPA_FILE_PATTERN_ROW_AMP, PM_DETREND_TYPE_PATTERN_ROW_AMP)) {
+            psWarning ("Can't find a pattern row amplitude source, will apply to all cells or defined subset");
+            // an empty or invalid file may have been generated -- we want to skip, not raise an error
+            pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP");
+            if (PRAfile) {
+              PRAfile->state |= PM_FPA_STATE_INACTIVE;
+            }
         }
     }

trunk/ppImage/src/ppImageSetThreads.c

@@ -15 +15 @@
 }
 
-bool ppImageSetThreads () {
+bool ppImageThread_ppImageDetrendPatternApplyCell (psThreadJob *job) {
 
-    psThreadTask *task = NULL;
+    pmConfig *config        = job->args->data[0];
+    pmFPA *fpa              = job->args->data[1];
+    pmChip *chip            = job->args->data[2];
+    pmCell *cell            = job->args->data[3];
+    pmFPAview *view         = job->args->data[4];
+    ppImageOptions *options = job->args->data[5];
 
-    task = psThreadTaskAlloc ("PPIMAGE_DETREND_READOUT", 3);
-    task->function = &ppImageThread_ppImageDetrendReadout;
-    psThreadTaskAdd (task);
+    bool status = ppImageDetrendPatternApplyCell (config, fpa, chip, cell, view, options);
+    return status;
+}
 
+bool ppImageSetThreads (void) {
+
+# if (0)
+    // *** Detrend Readout (now unused : threading is done at the readout level in pmDetrend)
+    {
+        psThreadTask *task = psThreadTaskAlloc ("PPIMAGE_DETREND_READOUT", 3);
+        task->function = &ppImageThread_ppImageDetrendReadout;
+        psThreadTaskAdd (task);
+    }
+# endif
+
+    // *** Detrend Pattern Row
+    {
+        psThreadTask *task = psThreadTaskAlloc("PPIMAGE_PATTERN_ROW_CELL", 6);
+        task->function = &ppImageThread_ppImageDetrendPatternApplyCell;
+        psThreadTaskAdd(task);
+        psFree(task);
+    }
     return true;
 }