Changeset 41908

Timestamp:

Nov 8, 2021, 7:57:10 AM (5 years ago)

Author:

eugene

Message:

merge changes from ppStack.20211015: adding stack-by-percent, some code cleanups (remove commented out blocks), fix thread failure error

Location:

branches/eam_branches/ipp-20211108/ppStack

Files:

• . (modified) (1 prop)
• src/Makefile.am (modified) (1 diff)
• src/ppStack.h (modified) (1 diff)
• src/ppStackCombineFinal.c (modified) (1 diff)
• src/ppStackCombineInitial.c (modified) (1 diff)
• src/ppStackCombinePercent.c (copied) (copied from branches/eam_branches/ppStack.20211015/src/ppStackCombinePercent.c )
• src/ppStackCombinePrepare.c (modified) (4 diffs)
• src/ppStackLoop.c (modified) (5 diffs)
• src/ppStackLoop.h (modified) (3 diffs)
• src/ppStackLoopByPercent.c (copied) (copied from branches/eam_branches/ppStack.20211015/src/ppStackLoopByPercent.c )
• src/ppStackMatch.c (modified) (1 diff)
• src/ppStackOptions.c (modified) (1 diff)
• src/ppStackOptions.h (modified) (1 diff)
• src/ppStackReadout.c (modified) (12 diffs)
• src/ppStackSetup.c (modified) (1 diff)
• src/ppStackThread.c (modified) (9 diffs)
• src/ppStackThread.h (modified) (2 diffs)

branches/eam_branches/ipp-20211108/ppStack/src/Makefile.am

@@ -29 +29 @@
         ppStackPrepare.c        \
         ppStackConvolve.c       \
+        ppStackLoopByPercent.c  \
+        ppStackCombinePercent.c \
         ppStackCombinePrepare.c \
         ppStackCombineInitial.c \

branches/eam_branches/ipp-20211108/ppStack/src/ppStack.h

@@ -115 +115 @@
 bool ppStackReadoutFinalThread(psThreadJob *job // Job to process
     );
+
+// Thread entry point for ppStackReadoutPercent
+bool ppStackReadoutPercentThread(psThreadJob *job);
+
+bool ppStackReadoutPercent(const pmConfig *config,
+                           pmReadout *outRO,
+                           const psArray *readouts,
+                           const psVector *mask,
+                           const psVector *weightings,
+                           const psVector *exposures,
+                           const psVector *addVariance);
+
 // Perform median stacking for background
 bool ppStackCombineBackground(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config);

branches/eam_branches/ipp-20211108/ppStack/src/ppStackCombineFinal.c

@@ -2 +2 @@
 
 // This is the doomsday switch.
-// #define TESTING                         // Enable test output
+// # define TESTING                         // Enable test output
 
 //MEH -- adhoc addition to blank mask border of final stack since rejection different/none on order of KERNEL.SIZE with overlap in CombineInitial 

branches/eam_branches/ipp-20211108/ppStack/src/ppStackCombineInitial.c

@@ -9 +9 @@
     psAssert(options, "Require options");
     psAssert(config, "Require configuration");
-
-/*     if (!options->convolve) { */
-/*         // No need to do initial combination when we haven't convolved */
-/*         // XXX either allocate inspect and rejected here, or do not require them downstream */
-/*         return true; */
-/*     } */
 
     psTimerStart("PPSTACK_INITIAL");

branches/eam_branches/ipp-20211108/ppStack/src/ppStackCombinePrepare.c

@@ -31 +31 @@
       options->expRO = pmReadoutAlloc(expCell); //Output readout
     }
-/*     else { */
-/*       options->expRO = NULL; */
-/*     } */
+
     int bkg_r0 = 0,bkg_c0 = 0;
     int bkg_nC = 0,bkg_nR = 0;
@@ -53 +51 @@
       
       options->bkgRO = pmReadoutAlloc(bkgCell); // BKG readout
-      //      if (!pmHDUGenerateForFPA(options->bkgRO->parent->parent->parent)) {
       options->bkgRO->parent->parent->parent->hdu = pmHDUAlloc(NULL);
+
       if (!options->bkgRO->parent->parent->parent->hdu) {
         fprintf(stderr,"failed to generate a HDU for this thing.\n");
@@ -68 +66 @@
                                                                         ro->parent->parent->parent->concepts);
 
-    }
-    else {
+    } else {
       options->bkgRO = NULL;
     }
@@ -97 +94 @@
         return false;
       }
-      
-
-      
     }
 

branches/eam_branches/ipp-20211108/ppStack/src/ppStackLoop.c

@@ -31 +31 @@
     ppStackMemDump("prepare");
     if (options->quality) return true; // Can't do anything else
+
+    // if we want to skip the rejection steps, and instead use a percentile clipping:
+    if (options->clipPercent) {
+      bool result = ppStackLoopByPercent (config, options);
+      return result;
+    }
 
     // Convolve inputs
@@ -88 +94 @@
         pmCellFreeData(options->cells->data[i]);
     }
-    //MEH -- must uncomment back out -- unclear is should be moved after pixel rejection
+    // MEH -- must uncomment back out -- unclear is should be moved after pixel rejection
     psFree(stack);
 
@@ -126 +132 @@
         return false;
       }
-    }
-    else {
+    } else {
       // Since we haven't convolved, I believe we do need to normalize here.
-      //MEH -- see below for comment on ppStackCombineFinal and bscaleOffset
-      if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, true, true, true)) {
+      // MEH -- see below for comment on ppStackCombineFinal and bscaleOffset
+      // last 4 parameters are: safe, normalize, grow, bscaleoffset
+      // EAM : looks like we do NOT need to normalize here eithr because ppStackConvolve is still called and
+      // the inputs are normalized
+      if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, false, true, true)) {
         psError(psErrorCodeLast(), false, "Unable to perform final combination.");
         psFree(stack);
@@ -154 +162 @@
       return false;
     }
-    ppStackFileActivation(config,PPSTACK_FILES_BKG ,false);    
+    ppStackFileActivation(config, PPSTACK_FILES_BKG, false);    
+
     // Photometry
     psTrace("ppStack", 1, "Photometering stacked image....\n");
@@ -184 +193 @@
         return false;
     }
-    //MEH -- also must uncomment back out..
+    // MEH -- also must uncomment back out..
     psFree(stack);
     psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Cleanup, WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS"));

branches/eam_branches/ipp-20211108/ppStack/src/ppStackLoop.h

@@ -7 +7 @@
     ppStackOptions *options             // Options for stacking
     );
+
 // Median only loop.
 bool ppStackMedianLoop(
@@ -12 +13 @@
     ppStackOptions *options             // Options for stacking
     );
+
+// stack by percentile range
+bool ppStackLoopByPercent(
+  pmConfig *config,
+  ppStackOptions *options
+  );
 
 // Setup
@@ -92 +99 @@
 
 
+bool ppStackCombinePercent(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config);
+
 bool ppStackUpdateHeader(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config);
 bool ppStackJPEGs(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config);

branches/eam_branches/ipp-20211108/ppStack/src/ppStackMatch.c

@@ -524 +524 @@
         psFree(conv);
     } else {
+        // this branch is done if options->convolve is FALSE
         // Match the normalisation
         float norm = powf(10.0, -0.4 * options->norm->data.F32[index]); // Normalisation

branches/eam_branches/ipp-20211108/ppStack/src/ppStackOptions.c

@@ -56 +56 @@
     options->num = 0;
     options->quality = 0;
+
+    options->clipPercent = false;
+
     options->psf = NULL;
     options->sumExposure = NAN;

branches/eam_branches/ipp-20211108/ppStack/src/ppStackOptions.h

@@ -16 +16 @@
     int num;                            // Number of inputs
     int quality;                        // Bad data quality flag
+
+    bool clipPercent;                   // use percentile range to clip?
+
     // Prepare
     pmPSF *psf;                         // Target PSF

branches/eam_branches/ipp-20211108/ppStack/src/ppStackReadout.c

@@ -19 +19 @@
                                          weightings, exposures, addVariance);
     thread->busy = false;
+
+    thread->status = job->results ? PPSTACK_THREAD_SUCCESS : PPSTACK_THREAD_FAILURE;
+
+    psAssert(job->results, "Stacking failed.");
 
     return job->results ? true : false;
@@ -48 +52 @@
     thread->busy = false;
 
+    thread->status = status ? PPSTACK_THREAD_SUCCESS : PPSTACK_THREAD_FAILURE;
+
     psAssert(status, "Stacking failed.");
 
@@ -53 +59 @@
 }
 
+bool ppStackReadoutPercentThread(psThreadJob *job)
+{
+    PS_ASSERT_THREAD_JOB_NON_NULL(job, false);
+
+    psArray *args = job->args;          // Arguments
+    ppStackThread *thread = args->data[0]; // Thread
+    ppStackOptions *options = args->data[1]; // Options
+    pmConfig *config = args->data[2];   // Configuration
+
+    pmReadout *outRO = options->outRO;  // Output readout
+    psVector *mask = options->inputMask; // Mask for inputs
+    psVector *weightings = options->weightings; // Weightings (1/noise^2) for each image
+    psVector *exposures = options->exposures;   // Exposure times for each image
+    psVector *addVariance = options->matchChi2; // Additional variance when rejecting
+
+    // fprintf (stderr, "start thread %d : scan range %d - %d\n", thread->entry, thread->firstScan, thread->lastScan);
+
+    bool status = ppStackReadoutPercent(config, outRO, thread->readouts, mask, weightings, exposures, addVariance);
+
+    thread->busy = false;
+
+    thread->status = status ? PPSTACK_THREAD_SUCCESS : PPSTACK_THREAD_FAILURE;
+
+    // fprintf (stderr, "finished thread %d, status %d : scan range %d - %d\n", thread->entry, thread->status, thread->firstScan, thread->lastScan);
+
+    psAssert(status, "Stacking failed.");
+
+    return status;
+}
+
+//////////////////////////////////// 
 
 bool ppStackInspect(psThreadJob *job)
@@ -145 +182 @@
 
     int num = readouts->n;              // Number of inputs
-    psArray *stack = psArrayAlloc(num); // Array for stacking
+    psArray *stackData = psArrayAlloc(num); // Array of data to be stacked
 
     for (int i = 0; i < num; i++) {
@@ -160 +197 @@
         }
 
-        stack->data[i] = pmStackDataAlloc(ro, weightings->data.F32[i], exposures->data.F32[i],
+        // stackData is an array of pmStackData structures
+        stackData->data[i] = pmStackDataAlloc(ro, weightings->data.F32[i], exposures->data.F32[i],
                                           addVariance->data.F32[i]);
     }
 
-    if (!pmStackCombine(outRO, NULL, stack, maskBad, maskSuspect, maskBlank, kernelSize, iter,
+    if (!pmStackCombine(outRO, NULL, stackData, maskBad, maskSuspect, maskBlank, kernelSize, iter,
                         combineRej, combineSys, combineDiscard, useVariance, safe, nminpix, false)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection.");
-        psFree(stack);
+        psFree(stackData);
+        
+        // XXX EAM : if pmStackCombine fails above this return will cause the thread to block.
+        // The failure status results in a psThread.fault.  the psThreadLauncher function, which
+        // each background thread runs as a loop, will sleep until the fault is cleared externally
+        // by the handler.  psThreadPoolWait() catches, counts, and clears these faults, but
+        // ppStackThreadRead does not.  
         return false;
     }
@@ -175 +219 @@
     psArray *reject = psArrayAlloc(num);  // List of pixels rejected
     for (int i = 0; i < num; i++) {
-        pmStackData *data = stack->data[i]; // Data for this image
+        pmStackData *data = stackData->data[i]; // Data for this image
         if (!data) {
             continue;
@@ -186 +230 @@
         reject->data[i] = psMemIncrRefCounter(data->reject);
     }
-    psFree(stack);
+    psFree(stackData);
 
     //MEH change to trace
@@ -247 +291 @@
 
     int num = readouts->n;              // Number of inputs
-    psArray *stack = psArrayAlloc(num); // Array for stacking
+    psArray *stackData = psArrayAlloc(num); // Array of data to be stacked
 
     // We have rejection from a previous combination: combine without flagging pixels to inspect
@@ -272 +316 @@
                                              addVariance ? addVariance->data.F32[i] : NAN);
         data->reject = rejected ? psMemIncrRefCounter(rejected->data[i]) : NULL;
-        stack->data[i] = data;
+        stackData->data[i] = data;
 
         //MEH -- apply bscale offset before norm   
@@ -288 +332 @@
     }
 
-    if (!pmStackCombine(outRO, expRO, stack, maskBad, maskSuspect, maskBlank, 0, iter, combineRej,
+    if (!pmStackCombine(outRO, expRO, stackData, maskBad, maskSuspect, maskBlank, 0, iter, combineRej,
                         combineSys, combineDiscard, useVariance, safe, nminpix, rejected)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts.");
-        psFree(stack);
+        psFree(stackData);
         return false;
     }
@@ -307 +351 @@
     expChip->data_exists = true;
 
-    psFree(stack);
+    psFree(stackData);
 
     //MEH change to trace
@@ -316 +360 @@
     return true;
 }
+
+bool ppStackReadoutPercent(const pmConfig *config, pmReadout *outRO, const psArray *readouts,
+                               const psVector *mask, const psVector *weightings, const psVector *exposures,
+                               const psVector *addVariance)
+{
+    assert(config);
+    assert(outRO);
+    assert(readouts);
+    assert(mask && mask->n == readouts->n && mask->type.type == PS_TYPE_VECTOR_MASK);
+    assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32);
+    assert(addVariance && addVariance->n == readouts->n && addVariance->type.type == PS_TYPE_F32);
+    static int sectionNum = 0;          // Section number; for debugging outputs
+
+    // fprintf (stderr, "starting ReadoutPercent, %d\n", sectionNum);
+
+    // Get the recipe values
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
+    psAssert(recipe, "We've thrown an error on this before.");
+
+    bool mdok;                          // Status of MD lookup
+    psF64 minRange = psMetadataLookupF32(&mdok, recipe, "COMBINE.MIN.RANGE"); // Combination threshold
+
+    char defaultBlankStr[16] = "BLANK";
+
+    psString maskBadStr     = psMetadataLookupStr(&mdok, recipe, "MASK.VAL"); // Name of bits to mask for bad
+    psString maskSuspectStr = psMetadataLookupStr(&mdok, recipe, "MASK.SUSPECT"); // Name of suspect mask bits
+    psString maskBlankStr   = psMetadataLookupStr(&mdok, recipe, "MASK.BLANK"); // Name of bits to set for empty pixels
+    if (!maskBlankStr) {
+      maskBlankStr = psMetadataLookupStr(&mdok, recipe, "MASK.BAD"); // Old name for MASK.BLANK
+    }
+    if (!maskBlankStr) {
+      maskBlankStr = defaultBlankStr; // this is statically allocated above
+    }
+
+    psImageMaskType maskBad     = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
+    psImageMaskType maskSuspect = pmConfigMaskGet(maskSuspectStr, config); // Suspect bits
+    psImageMaskType maskBlank   = pmConfigMaskGet(maskBlankStr, config); // Bits to mask for bad pixels
+
+    int num = readouts->n;              // Number of inputs
+    psArray *stackData = psArrayAlloc(num); // Array of data to be stacked
+
+    for (int i = 0; i < num; i++) {
+        pmReadout *ro = readouts->data[i];
+        if (!ro || mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
+            // Bad image
+            continue;
+        }
+
+        // Ensure there is a mask, or pmStackCombine will complain
+        if (!ro->mask) {
+            ro->mask = psImageAlloc(ro->image->numCols, ro->image->numRows, PS_TYPE_IMAGE_MASK);
+            psImageInit(ro->mask, 0);
+        }
+
+        stackData->data[i] = pmStackDataAlloc(ro, weightings->data.F32[i], exposures->data.F32[i],
+                                              addVariance->data.F32[i]);
+    }
+
+    // XXX rename minRange to rejectFraction
+    // XXX add expmaps
+    if (!pmStackCombineByPercentile(outRO, NULL, stackData, minRange, maskBad, maskSuspect, maskBlank)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection.");
+        psFree(stackData);
+        return false;
+    }
+
+    outRO->data_exists = true; // output readout
+    outRO->parent->data_exists = true; // output cell
+    outRO->parent->parent->data_exists = true; // output chip
+
+# if 0
+      pmCell *expCell = expRO->parent;    // Exposure cell
+      pmChip *expChip = expCell->parent;  // Exposure chip
+      expRO->data_exists = true;
+      expCell->data_exists = true;
+      expChip->data_exists = true;
+# endif
+
+    psFree(stackData);
+    sectionNum++;
+
+    return true;
+}
+
+

branches/eam_branches/ipp-20211108/ppStack/src/ppStackSetup.c

@@ -18 +18 @@
     options->convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images?
     options->doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
+
+    options->clipPercent  = psMetadataLookupBool(NULL, recipe, "COMBINE.PERCENT"); // use percentile range to clip?
 
     if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {

branches/eam_branches/ipp-20211108/ppStack/src/ppStackThread.c

@@ -19 +19 @@
     thread->firstScan = 0;
     thread->lastScan = 0;
+    thread->entry = -1; // this gets assigned after allocation
+    thread->status = PPSTACK_THREAD_NEW;
     return thread;
 }
@@ -124 +126 @@
             readouts->data[j] = ro;
         }
-        threads->data[i] = ppStackThreadAlloc(readouts);
+        ppStackThread *thread = ppStackThreadAlloc(readouts);
+        thread->entry = i;
+        threads->data[i] = thread;
         psFree(readouts);               // Drop reference
     }
@@ -157 +161 @@
     while (true) {
         // check for any groups which can read data
+        // NOTE: threads->n is set (in ppStackThreadDataSetup) to be 1 more than the argument to -threads
         for (int j = 0; j < threads->n; j++) {
             ppStackThread *thread = threads->data[j];
+            // fprintf (stderr, "grab thread %d, scan: %d - %d : busy: %d read: %d status: %d\n", thread->entry, thread->firstScan, thread->lastScan, thread->busy, thread->read, thread->status);
             if (thread->read) {
                 continue;
@@ -166 +172 @@
             thread->lastScan = stack->lastScan + rows;
             stack->lastScan = thread->lastScan;
+            thread->status = PPSTACK_THREAD_READ;
 
             psArray *readouts = thread->readouts;
@@ -172 +179 @@
 
             psTrace("ppStack", 2, "Reading data for chunk %d into group %d....\n", numChunk, j);
+            // fprintf (stderr, "Reading data for chunk %d into group %d (thread %d, scan: %d - %d)....\n", numChunk, j, thread->entry, thread->firstScan, thread->lastScan);
             for (int i = 0; i < readouts->n; i++) {
                 pmReadout *ro = readouts->data[i]; // Input readout
@@ -177 +185 @@
                     continue;
                 }
+
+                // fprintf (stderr, "read data for readout %d\n", i);
 
                 // override the recorded last scan
@@ -232 +242 @@
 
             thread->read = thread->busy = true;
+            thread->status = PPSTACK_THREAD_RUN;
             return thread;
         }
@@ -239 +250 @@
         for (int j = 0; j < threads->n; j++) {
             ppStackThread *thread = threads->data[j];
+            // fprintf (stderr, "test thread %d, scan: %d - %d : busy: %d, read: %d, status: %d\n", thread->entry, thread->firstScan, thread->lastScan, thread->busy, thread->read, thread->status);
+
             if (thread->busy) {
                 continue;
             }
+            thread->status = PPSTACK_THREAD_NEW;
             thread->read = false;
             wait = false;
@@ -283 +297 @@
     }
 
+    {
+        psThreadTask *task = psThreadTaskAlloc("PPSTACK_PERCENT_COMBINE", 3);
+        task->function = &ppStackReadoutPercentThread;
+        psThreadTaskAdd(task);
+        psFree(task);
+    }
     return;
 }

branches/eam_branches/ipp-20211108/ppStack/src/ppStackThread.h

@@ -11 +11 @@
     int firstScan;                      // First row of the chunk to be read for this group
     int lastScan;                       // Last row of the chunk to be read for this group
+    int entry;                          // thread number for tracking progress
+    int status;                         // thread number for tracking progress
 } ppStackThread;
 
@@ -47 +49 @@
 
 
+enum {PPSTACK_THREAD_NEW, PPSTACK_THREAD_READ, PPSTACK_THREAD_RUN, PPSTACK_THREAD_SUCCESS, PPSTACK_THREAD_FAILURE};
+
+
 #endif