Index: trunk/ppImage/src/Makefile.am =================================================================== --- trunk/ppImage/src/Makefile.am (revision 41382) +++ trunk/ppImage/src/Makefile.am (revision 41894) @@ -42,4 +42,5 @@ ppImageDefineFile.c \ ppImageSetMaskBits.c \ + ppImageSetThreads.c \ ppImageBurntoolMask.c \ ppImageParityFlip.c \ Index: trunk/ppImage/src/ppImage.h =================================================================== --- trunk/ppImage/src/ppImage.h (revision 41382) +++ trunk/ppImage/src/ppImage.h (revision 41894) @@ -193,5 +193,6 @@ bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout); -bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options); +bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); +bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options); // Do background continuity step @@ -326,3 +327,5 @@ bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options, psMetadata *stats); +bool ppImageSetThreads (void); + #endif Index: trunk/ppImage/src/ppImageArguments.c =================================================================== --- trunk/ppImage/src/ppImageArguments.c (revision 41382) +++ trunk/ppImage/src/ppImageArguments.c (revision 41894) @@ -116,4 +116,5 @@ pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined"); if ((argnum = psArgumentGet(argc, argv, "-burntool"))) { Index: trunk/ppImage/src/ppImageDetrendPattern.c =================================================================== --- trunk/ppImage/src/ppImageDetrendPattern.c (revision 41382) +++ trunk/ppImage/src/ppImageDetrendPattern.c (revision 41894) @@ -14,5 +14,5 @@ bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, - const ppImageOptions *options) + ppImageOptions *options) { pmCell *cell = NULL; @@ -31,4 +31,13 @@ // CELLS in the format:CHIPS metadata table) + // New 2021 October : During ppImageParseCamera, we have loaded a file identified by + // PPIMAGE.PATTERN.ROW.AMP on config->files. This file contains a collection of FITS + // tables, one per chip. Each table lists the typical bias-drift amplitude for cell, + // measured from a collection of dark images. These values are passed (via + // cell->analysis) to pmPatternRow which choosed to apply the correction based on the + // ratio of the poisson noise due to the median background (+ read noise) and the + // typical amplitude. Tests show that if the amplitude / noise < 1/6, then the + // row-by-row variations are insignificant. + // We also check the chip header for the boolean 'PTRN_ROW' : if this is true, we have // already applied this correct to this data, so we simply skip the correction for this @@ -52,4 +61,11 @@ pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT"); + + // grab the PATTERN.ROW.AMP file + pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP"); + if (!PRAfile) { + psLogMsg("ppImage", PS_LOG_INFO, "Pattern Row Amplitude file not found, applying to all (with limits from ROW.SUBSET) "); + } + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing *view = *inputView; @@ -75,36 +91,58 @@ } + // grab the corresponding cell + if (PRAfile) { + pmCell *PRAcell = pmFPAviewThisCell (view, PRAfile->fpa); + psAssert (PRAcell, "found Pattern Row Amplitude file, but not cell?"); + + // find the nominal signal amplitude (check the ghost and/or crosstalk recipe file) + float amplitude = psMetadataLookupF32 (&status, PRAcell->analysis, "PTN.ROW.AMP"); + if (!status) amplitude = NAN; + + // put the value on the science cell + psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "PTN.ROW.AMP", PS_META_REPLACE, "", amplitude); + } + bool doPattern = false; if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.ROW.SUBSET")) { ESCAPE(false, "Unable to determine whether row pattern matching should be applied."); } - //psWarning ("Row: %d\n", doPattern); if (!doPattern) continue; - // A very detailed log message. - const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); - const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); - psLogMsg("ppImage", PS_LOG_INFO, "Performing row pattern correction for %s, %s\n", chipName, cellName); - - // process each of the readouts - pmReadout *readout; // Readout from cell - while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { - if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - ESCAPE(false, "load failure for Readout"); - } - if (!readout->data_exists) { - continue; - } - - // Perform pattern correction - if (!pmPatternRow(readout, options->patternRowOrder, options->patternRowIter, - options->patternRowRej, options->patternRowThresh, options->patternRowMean, - options->patternRowStdev, options->maskValue, options->darkMask)) { - psFree(view); - return(false); - } - } - - } + // switch to test threaded version + if (true) { + // I need to allocate a view here to be freed by the + // called function below. + pmFPAview *myView = pmFPAviewAlloc(0); // View for local processing + *myView = *view; + + // allocate a job, construct the arguments for this job + psThreadJob *job = psThreadJobAlloc("PPIMAGE_PATTERN_ROW_CELL"); + psArrayAdd(job->args, 1, config); + psArrayAdd(job->args, 1, input->fpa); + psArrayAdd(job->args, 1, chip); + psArrayAdd(job->args, 1, cell); + psArrayAdd(job->args, 1, myView); + psArrayAdd(job->args, 1, options); + if (!psThreadJobAddPending(job)) { + return false; + } + } else { + // bump the counter since it must be freed by the function below. + psMemIncrRefCounter (view); // View for local processing + if (!ppImageDetrendPatternApplyCell (config, input->fpa, chip, cell, view, options)) { + return false; + } + } + + } + + // wait here for the threaded jobs to finish + // if no threads are allocated, this + if (!psThreadPoolWait(true, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction."); + return false; + } + psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_ROW",PS_META_REPLACE,"PATTERN.ROW correction applied",true); psFree(view); @@ -266,2 +304,25 @@ return false; } + +// thread safety : +bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options) { + + // process each of the readouts + pmReadout *readout; // Readout from cell + while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) { + if (!readout->data_exists) { + continue; + } + + // Perform pattern correction + if (!pmPatternRow(readout, options->patternRowOrder, options->patternRowIter, + options->patternRowRej, options->patternRowThresh, options->patternRowMean, + options->patternRowStdev, options->maskValue, options->darkMask)) { + psFree (view); + return false; + } + } + psFree (view); // supplied view must be freeable + return true; +} + Index: trunk/ppImage/src/ppImageParseCamera.c =================================================================== --- trunk/ppImage/src/ppImageParseCamera.c (revision 41382) +++ trunk/ppImage/src/ppImageParseCamera.c (revision 41894) @@ -117,4 +117,5 @@ pmDetrendSetThreadTasks(nScanRows); } + ppImageSetThreads (); // even if threading is off, we need to identify the ppImageDetrendPattern thread function if (options->doPatternRow || options->doPatternCell) { @@ -180,4 +181,16 @@ psError (PS_ERR_IO, false, "Can't find a fringe image source"); return NULL; + } + } + + if (options->doPatternRow) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.PATTERN.ROW.AMP", "PATTERN.ROW.AMP", + PM_FPA_FILE_PATTERN_ROW_AMP, PM_DETREND_TYPE_PATTERN_ROW_AMP)) { + psWarning ("Can't find a pattern row amplitude source, will apply to all cells or defined subset"); + // an empty or invalid file may have been generated -- we want to skip, not raise an error + pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP"); + if (PRAfile) { + PRAfile->state |= PM_FPA_STATE_INACTIVE; + } } } Index: trunk/ppImage/src/ppImageSetThreads.c =================================================================== --- trunk/ppImage/src/ppImageSetThreads.c (revision 41382) +++ trunk/ppImage/src/ppImageSetThreads.c (revision 41894) @@ -15,12 +15,35 @@ } -bool ppImageSetThreads () { +bool ppImageThread_ppImageDetrendPatternApplyCell (psThreadJob *job) { - psThreadTask *task = NULL; + pmConfig *config = job->args->data[0]; + pmFPA *fpa = job->args->data[1]; + pmChip *chip = job->args->data[2]; + pmCell *cell = job->args->data[3]; + pmFPAview *view = job->args->data[4]; + ppImageOptions *options = job->args->data[5]; - task = psThreadTaskAlloc ("PPIMAGE_DETREND_READOUT", 3); - task->function = &ppImageThread_ppImageDetrendReadout; - psThreadTaskAdd (task); + bool status = ppImageDetrendPatternApplyCell (config, fpa, chip, cell, view, options); + return status; +} +bool ppImageSetThreads (void) { + +# if (0) + // *** Detrend Readout (now unused : threading is done at the readout level in pmDetrend) + { + psThreadTask *task = psThreadTaskAlloc ("PPIMAGE_DETREND_READOUT", 3); + task->function = &ppImageThread_ppImageDetrendReadout; + psThreadTaskAdd (task); + } +# endif + + // *** Detrend Pattern Row + { + psThreadTask *task = psThreadTaskAlloc("PPIMAGE_PATTERN_ROW_CELL", 6); + task->function = &ppImageThread_ppImageDetrendPatternApplyCell; + psThreadTaskAdd(task); + psFree(task); + } return true; }