Changeset 42776

Timestamp:

Dec 29, 2024, 4:02:56 PM (19 months ago)

Author:

eugene

Message:

set freeLevel for inputs to CELL so data can be freed after readout loop; change API to set the dataLevel and freeLevel to specific values (no user option); plug a mem leak (free zerors); free input readout data after each cell is done

Location:

branches/eam_branches/ipp-ppmerge-20241229/ppMerge/src

Files:

• ppMerge.h (modified) (1 diff)
• ppMergeFiles.c (modified) (2 diffs)
• ppMergeLoop.c (modified) (13 diffs)
• ppMergeMask.c (modified) (1 diff)

branches/eam_branches/ipp-ppmerge-20241229/ppMerge/src/ppMerge.h

@@ -117 +117 @@
     );
 
+bool ppMergeFileFreeInput(pmConfig *config, const pmFPAview *view, int num);
+
 /**
  * Set the data level for files specified by name; return an array of the files
  */
 psArray *ppMergeFileDataLevel(const pmConfig *config, ///< Configuration
-                              const char *name, ///< Name of files
-                              pmFPALevel level ///< Level for file data level
+                              const char *name ///< Name of files
     );
 

branches/eam_branches/ipp-ppmerge-20241229/ppMerge/src/ppMergeFiles.c

@@ -105 +105 @@
 }
 
-psArray *ppMergeFileDataLevel(const pmConfig *config, const char *name, pmFPALevel level)
+bool ppMergeFileFreeInput(pmConfig *config, const pmFPAview *view, int num)
+{
+    {
+        pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", num);
+        if (!pmFPAfileFreeData(input, view)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to free data for image file %d", num);
+            return false;
+        }
+    }
+    bool mdok;          // Status of MD lookup
+    if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS")) {
+        pmFPAfile *mask = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.MASK", num); // Mask file
+        if (!pmFPAfileFreeData(mask, view)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to free data for mask file %d", num);
+            return false;
+        }
+    }
+    if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.VARIANCES")) {
+        pmFPAfile *variance = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.VARIANCE", num); // Variance file
+        if (!pmFPAfileFreeData(variance, view)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to free data for variance file %d", num);
+            return false;
+        }
+    }
+    return true;
+}
+
+// Select the specified input files and assign the dataLevel and freeLevel,
+// saving a pointer on output array.
+// This function takes the argument pmFPALevel and sets the dataLevel based on that value.
+// However, this is only used with the value PM_FPA_LEVEL_READOUT, and it must be set
+// to that level for the rest of ppMerge to work correctly.  Furthermore, the freeLevel
+// must be set to Cell.  Remove the (false) option.
+psArray *ppMergeFileDataLevel(const pmConfig *config, const char *name)
 {
     assert(config);
@@ -115 +148 @@
     for (int i = 0; i < numFiles; i++) {
         pmFPAfile *file = pmFPAfileSelectSingle(config->files, name, i); // Image file
-        file->dataLevel = level;
-        file->freeLevel = level;
+        file->dataLevel = PM_FPA_LEVEL_READOUT;
+        file->freeLevel = PM_FPA_LEVEL_CELL;
         files->data[i] = psMemIncrRefCounter(file);
     }

branches/eam_branches/ipp-ppmerge-20241229/ppMerge/src/ppMergeLoop.c

@@ -39 +39 @@
     bool haveVariances = psMetadataLookupBool(&mdok, arguments, "INPUTS.VARIANCES"); // Do we have variances?
 
-    psArray *inputs = ppMergeFileDataLevel(config, "PPMERGE.INPUT", PM_FPA_LEVEL_READOUT); // Input images
-    psArray *masks = NULL, *variances = NULL; // Input masks and variances
-    if (haveMasks) {
-        masks = ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK", PM_FPA_LEVEL_READOUT);
-    }
-    if (haveVariances) {
-        variances = ppMergeFileDataLevel(config, "PPMERGE.INPUT.VARIANCE", PM_FPA_LEVEL_READOUT);
-    }
+    psArray *inputs    = ppMergeFileDataLevel(config, "PPMERGE.INPUT"); // Input images
+    psArray *masks     = haveMasks ? ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK") : NULL; // Input masks 
+    psArray *variances = haveVariances ? ppMergeFileDataLevel(config, "PPMERGE.INPUT.VARIANCE") : NULL; // Input variances
 
     int nThreads = psMetadataLookupS32 (&mdok, arguments, "NTHREADS");
@@ -135 +130 @@
     psString darkNorm = psMetadataLookupStr(&mdok, arguments, "DARK.NORM"); ///< Dark normalisation
 
-
     if (!ppMergeFileActivate(config, PPMERGE_FILES_ALL, true)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to activate files.");
@@ -152 +146 @@
     }
 
-
     // Average concepts across inputs
     {
@@ -180 +173 @@
         {
             psList *inChips = psListAlloc(NULL);
-            for (int i=0; i < numFiles; i++) {
+            for (int i = 0; i < numFiles; i++) {
                 pmChip *chip = pmFPAviewThisChip(view, ((pmFPAfile *)inputs->data[i])->fpa);
                 psListAdd(inChips, PS_LIST_TAIL, chip);
@@ -193 +186 @@
 
         pmCell *outCell;                ///< Cell of interest
-
-        // XXX TEST : force a single loop
-        // outCell = pmFPAviewNextCell(view, outFPA, 1); {
 
         while ((outCell = pmFPAviewNextCell(view, outFPA, 1))) {
@@ -305 +295 @@
                 psAbort("Should never get here.");
             }
+            // XXX EAM : for testing purposes
+            fprintf (stderr, "after pmDarkCombinePrepare / pmReadoutCombinePrepare:\n");
+            psMemStats(true, NULL, NULL);
 
             // Read input data by chunks
@@ -320 +313 @@
                     break;
                 }
+
+                
 
                 // Start a job
@@ -357 +352 @@
                       // call: pmDarkCombine(outCell, fileGroup->readouts, iter, rej, maskVal);
                       if (!psThreadJobAddPending(job)) {
-                        //                          goto ERROR;
-                        // continue; // I don't care.
+                        // CZW commented this out with a note that this was needed to allow ppMerge to work even if some inputs are invalid.
+                        // EAM thinks pmDarkCombine will only return FALSE if there is a programming or setup error (see pmDark.c:pmDarkCombine)
+                        // if so, we should raise an error here.
+                        // goto ERROR;
                       }
                       break;
@@ -400 +397 @@
 
             psFree(fileGroups);
-            //            psFree(zeros);
 
             // XXX eventually need to keep both the shutter and the pattern, as we do with dark
@@ -541 +537 @@
             }
 
+            // free the data used by the input files (we need to do the work ourselves)
+            for (int i = 0; i < numFiles; i++) {
+                if (!ppMergeFileFreeInput(config, view, i)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to open file %d", i);
+                    goto ERROR;
+                }
+            }
+
             psFree(outRO);
             cellNum++;
@@ -547 +551 @@
                 goto ERROR;
             }
+
+            pmFPAfileIOList (config);
         }
 
@@ -564 +570 @@
     psFree(variances);
     psFree(stats);
+    psFree(zeros);
     return true;
 
@@ -573 +580 @@
     psFree(variances);
     psFree(stats);
+    psFree(zeros);
     return false;
 }

branches/eam_branches/ipp-ppmerge-20241229/ppMerge/src/ppMergeMask.c

@@ -427 +427 @@
     }
 
-    psArray *inputs = ppMergeFileDataLevel(config, "PPMERGE.INPUT", PM_FPA_LEVEL_READOUT); ///< Input images
+    psArray *inputs = ppMergeFileDataLevel(config, "PPMERGE.INPUT"); ///< Input images
     psFree(inputs);
     if (haveMasks) {
-        psArray *masks = ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK", PM_FPA_LEVEL_READOUT);
+        psArray *masks = ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK");
         psFree(masks);
     }
     if (haveVariances) {
-        psArray *variances = ppMergeFileDataLevel(config, "PPMERGE.INPUT.VARIANCE", PM_FPA_LEVEL_READOUT);
+        psArray *variances = ppMergeFileDataLevel(config, "PPMERGE.INPUT.VARIANCE");
         psFree(variances);
     }