Changeset 42664
- Timestamp:
- Apr 23, 2024, 5:54:26 PM (2 years ago)
- Location:
- branches/2dbias/psModules
- Files:
-
- 4 edited
-
. (modified) (1 prop)
-
src/concepts/pmConcepts.c (modified) (1 diff)
-
src/detrend/pmOverscan.c (modified) (11 diffs)
-
src/detrend/pmOverscan.h (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/2dbias/psModules
- Property svn:mergeinfo changed
/branches/eam_branches/psModules.20240412 (added) merged: 42651-42654,42659
- Property svn:mergeinfo changed
-
branches/2dbias/psModules/src/concepts/pmConcepts.c
r42233 r42664 408 408 { 409 409 psList *biassecs = psListAlloc(NULL); // Blank biassecs 410 psMetadataItem *cellBiassec = psMetadataItemAllocPtr("CELL.BIASSEC", PS_DATA_LIST, 411 "Bias sections", biassecs); 410 psMetadataItem *cellBiassec = psMetadataItemAllocPtr("CELL.BIASSEC", PS_DATA_LIST, "Bias sections", biassecs); 412 411 psFree(biassecs); 413 412 pmConceptRegister(cellBiassec, p_pmConceptParse_CELL_BIASSEC, p_pmConceptFormat_CELL_BIASSEC, NULL, true, PM_FPA_LEVEL_CELL); -
branches/2dbias/psModules/src/detrend/pmOverscan.c
r42379 r42664 19 19 #define SMOOTH_NSIGMA 4.0 // Number of Gaussian sigma the smoothing kernel extends 20 20 21 static void pmOverscanOptionsFree(pmOverscanOptions *options) 22 { 23 psFree(options->stat); 24 psFree(options->poly); 25 psFree(options->spline); 26 } 27 28 pmOverscanOptions *pmOverscanOptionsAlloc(bool single, pmFit fitType, unsigned int order, psStats *stat, 29 int boxcar, float gauss) 30 { 31 pmOverscanOptions *opts = psAlloc(sizeof(pmOverscanOptions)); 32 psMemSetDeallocator(opts, (psFreeFunc)pmOverscanOptionsFree); 33 34 // Inputs 35 opts->single = single; 36 opts->constant = false; 37 opts->fitType = fitType; 38 opts->order = order; 39 opts->stat = psMemIncrRefCounter(stat); 40 41 opts->minValid = 0.0; // default value if not defined 42 opts->maxValid = (float) 0x10000; // default value if not defined 43 opts->maskVal = 0x0001; // default value if not defined 44 45 // Smoothing 46 opts->boxcar = boxcar; 47 opts->gauss = gauss; 48 49 // Outputs 50 opts->poly = NULL; 51 opts->spline = NULL; 52 53 return opts; 54 } 55 56 // Produce an overscan vector from an array of pixels 57 psVector *pmOverscanVector(float *chi2, // chi^2 from fit 58 pmOverscanOptions *overscanOpts, // Overscan options 59 const psArray *pixels, // Array of vectors containing the pixel values 60 psStats *myStats // Statistic to use in reducing the overscan 61 ) 62 { 63 assert(overscanOpts); 64 assert(pixels); 65 assert(myStats); 66 67 psStatsOptions statistic = psStatsSingleOption(myStats->options); // Statistic to use 68 assert(statistic != 0); 69 70 // Reduce the overscans 71 psVector *reduced = psVectorAlloc(pixels->n, PS_TYPE_F32); // Overscan for each row 72 psVector *ordinate = psVectorAlloc(pixels->n, PS_TYPE_F32); // Ordinate 73 psVector *mask = psVectorAlloc(pixels->n, PS_TYPE_VECTOR_MASK); // Mask for fitting 74 75 for (int i = 0; i < pixels->n; i++) { 76 psVector *values = pixels->data[i]; // Vector with overscan values 77 if (values->n > 0) { 78 mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0; 79 ordinate->data.F32[i] = 2.0*(float)i/(float)pixels->n - 1.0; // Scale to [-1,1] 80 if (!psVectorStats(myStats, values, NULL, NULL, 0)) { 81 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); 82 return false; 83 } 84 reduced->data.F32[i] = psStatsGetValue(myStats, statistic); 85 } else if (overscanOpts->fitType == PM_FIT_NONE) { 86 psError(PS_ERR_UNKNOWN, true, "The overscan is not supplied for all points on the " 87 "image, and no fit is requested.\n"); 88 return NULL; 89 } else { 90 // We'll fit this one out 91 mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 1; 92 } 93 } 94 95 // Smooth the reduced vector 96 if (overscanOpts->boxcar > 0) { 97 psVector *smoothed = psVectorBoxcar(NULL, reduced, overscanOpts->boxcar); // Smoothed vector 98 psFree(reduced); 99 reduced = smoothed; 100 } 101 if (isfinite(overscanOpts->gauss) && overscanOpts->gauss > 0) { 102 if (overscanOpts->boxcar > 0) { 103 psWarning("Gaussian smoothing the boxcar smoothed overscan --- you asked for it."); 104 } 105 psVector *smoothed = psVectorSmooth(NULL, reduced, overscanOpts->gauss, SMOOTH_NSIGMA); 106 psFree(reduced); 107 reduced = smoothed; 108 } 109 110 // Fit the overscan, if required 111 psVector *fitted = NULL; // Fitted overscan values 112 switch (overscanOpts->fitType) { 113 case PM_FIT_NONE: 114 // No fitting --- that's easy. 115 fitted = psMemIncrRefCounter(reduced); 116 break; 117 case PM_FIT_POLY_ORD: 118 if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || 119 overscanOpts->poly->type != PS_POLYNOMIAL_ORD)) { 120 psFree(overscanOpts->poly); 121 overscanOpts->poly = NULL; 122 } 123 if (! overscanOpts->poly) { 124 overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, overscanOpts->order); 125 } 126 psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); 127 fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); 128 break; 129 case PM_FIT_POLY_CHEBY: 130 if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || 131 overscanOpts->poly->type != PS_POLYNOMIAL_CHEB)) { 132 psFree(overscanOpts->poly); 133 overscanOpts->poly = NULL; 134 } 135 if (! overscanOpts->poly) { 136 overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_CHEB, overscanOpts->order); 137 } 138 psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); 139 fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); 140 break; 141 case PM_FIT_SPLINE: 142 143 // XXX I don't think psSpline1D is up to scratch yet --- it has no mask, and it assumes 144 // a knot for every input point. it needs an argument like 'number of knots' for the 145 // output spline. EAM: still true 2023.01.22 146 147 // overscanOpts->spline = psVectorFitSpline1D(reduced, ordinate); 148 // fitted = psSpline1DEvalVector(overscanOpts->spline, ordinate); 149 psError(PS_ERR_UNKNOWN, true, "Spline overscan fitting is broken\n"); 150 break; 151 default: 152 psError(PS_ERR_UNKNOWN, true, "Unknown value for the fitting type: %d\n", overscanOpts->fitType); 153 return NULL; 154 break; 155 } 156 157 if (chi2) { 158 *chi2 = 0.0; // chi^2 (sort of) 159 for (int i = 0; i < reduced->n; i++) { 160 *chi2 += PS_SQR(fitted->data.F32[i] - reduced->data.F32[i]); 161 } 162 } 163 164 psFree(reduced); 165 psFree(ordinate); 166 psFree(mask); 167 168 return fitted; 169 } 170 171 bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanOptions *overscanOpts, float chi2) { 172 173 psString comment = NULL; // Comment to add 174 175 switch (overscanOpts->fitType) { 176 case PM_FIT_POLY_ORD: 177 case PM_FIT_POLY_CHEBY: { 178 psStringAppend(&comment, "Overscan fit (chi2: %.2f): ", chi2); 179 psPolynomial1D *poly = overscanOpts->poly; // The polynomial 180 for (int i = 0; i < poly->nX; i++) { 181 psStringAppend(&comment, "%.1f ", poly->coeff[i]); 182 } 183 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); 184 psFree(comment); 185 comment = NULL; 186 187 // write metadata header value 188 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, 189 "Overscan value", poly->coeff[0]); 190 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, 191 "Overscan stdev", poly->coeffErr[0]); 192 break; 193 } 194 case PM_FIT_SPLINE: { 195 /* 196 psSpline1D *spline = overscanOpts->spline; // The spline 197 for (int i = 0; i < spline->n; i++) { 198 psStringAppend(&comment, "Overscan fit (chi2: %.2f) %d:", chi2, i); 199 psPolynomial1D *poly = spline->spline[i]; // i-th polynomial 200 for (int j = 0; j < poly->nX; j++) { 201 psStringAppend(&comment, "%.1f ", poly->coeff[i]); 202 } 203 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, 204 comment, ""); 205 psFree(comment); 206 comment = NULL; 207 } 208 */ 209 // write metadata header value 210 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, 211 "Overscan value", NAN); 212 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, 213 "Overscan stdev", NAN); 214 break; 215 } 216 case PM_FIT_NONE: 217 break; 218 default: 219 psAbort("Should never get here!!!\n"); 220 } 221 return true; 222 } 21 static void pmOverscanOptionsFree(pmOverscanOptions *options); 22 static void pmOverscanStatOptionsFree(pmOverscanStatOptions *options); 23 bool pmOverscanUpdateHeaderVector (pmReadout *input, pmHDU *hdu, pmOverscanOptions *overscanOpts, psVector *reduced); 223 24 224 25 bool pmOverscanSubtract (pmReadout *input, pmOverscanOptions *overscanOpts) { … … 234 35 if (overscanOpts->constant) { 235 36 // write metadata header value 236 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", 237 overscanOpts->value); 37 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", overscanOpts->value); 238 38 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", NAN); 239 39 … … 247 47 248 48 // Check for an unallowable pmFit. 249 if (overscanOpts->fitType != PM_FIT_NONE && overscanOpts->fitType != PM_FIT_POLY_ORD && 250 overscanOpts->fitType != PM_FIT_POLY_CHEBY && overscanOpts->fitType != PM_FIT_SPLINE) { 251 psError(PS_ERR_UNKNOWN, true, "Invalid fit type (%d). Returning original image.\n", 252 overscanOpts->fitType); 253 return false; 49 if (overscanOpts->primary) { 50 if (overscanOpts->primary->fitType != PM_FIT_NONE && 51 overscanOpts->primary->fitType != PM_FIT_POLY_ORD && 52 overscanOpts->primary->fitType != PM_FIT_POLY_CHEBY && 53 overscanOpts->primary->fitType != PM_FIT_SPLINE) { 54 psError(PS_ERR_UNKNOWN, true, "Invalid fit type (%d). Returning original image.\n", 55 overscanOpts->primary->fitType); 56 return false; 57 } 58 } 59 if (overscanOpts->secondary) { 60 if (overscanOpts->secondary->fitType != PM_FIT_NONE && 61 overscanOpts->secondary->fitType != PM_FIT_POLY_ORD && 62 overscanOpts->secondary->fitType != PM_FIT_POLY_CHEBY && 63 overscanOpts->secondary->fitType != PM_FIT_SPLINE) { 64 psError(PS_ERR_UNKNOWN, true, "Invalid fit type (2D) (%d). Returning original image.\n", 65 overscanOpts->secondary->fitType); 66 return false; 67 } 254 68 } 255 69 256 70 psList *overscans = input->bias; // List of the overscan images 257 258 psStatsOptions statistic = psStatsSingleOption(overscanOpts->stat->options); // Statistic to use259 if (statistic == 0) {260 psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Multiple or no statistics options set: %p\n",261 overscanOpts->stat);262 return false;263 }264 psStats *stats = psStatsAlloc(statistic); // A new psStats, to avoid clobbering original265 266 psString comment = NULL; // Comment to add267 psStringAppend(&comment, "Subtracting overscan (stat %x; type %x; order %d)",268 statistic, overscanOpts->fitType, overscanOpts->order);269 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK,270 comment, "");271 psTrace ("psModules.detrend", 4, "%s\n", comment);272 psFree(comment);273 71 274 72 // Reduce all overscan pixels to a single value 275 73 if (overscanOpts->single) { 74 75 // extract overscan pixels to a single vector 276 76 psVector *pixels = psVectorAlloc(0, PS_TYPE_F32); 277 77 psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator … … 289 89 psFree(iter); 290 90 91 // statistic to be calculated 92 psStatsOptions statistic = psStatsSingleOption(overscanOpts->primary->stat->options); // Statistic to use 93 if (!statistic) { 94 psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Multiple or no statistics options set: %p\n", 95 overscanOpts->primary->stat); 96 return false; 97 } 98 psStats *stats = psStatsAlloc(statistic); // A new psStats, to avoid clobbering original 99 291 100 if (!psVectorStats(stats, pixels, NULL, NULL, 0)) { 292 101 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); … … 302 111 303 112 // write metadata header value 304 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value",305 reduced);113 // XXX EAM : this could / should write the stdev of the overscan region 114 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", reduced); 306 115 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", NAN); 307 116 … … 313 122 overscan = NULL; // Overscan image from iterator 314 123 while ((overscan = psListGetAndIncrement(iter))) { 315 psBinaryOp(overscan, overscan, "-", psMemIncrRefCounter(reducedScalar)); // NOTE: psBinaryOp frees arg2 if it a scalar, so we need to bump to re-use124 psBinaryOp(overscan, overscan, "-", psMemIncrRefCounter(reducedScalar)); // NOTE: psBinaryOp frees arg2 if it a scalar, so we need to bump to re-use 316 125 } 317 126 psFree(iter); … … 347 156 348 157 // adjust operation depending on the read direction : need to re-org pixels for columns 349 if ( cellreaddir == 1) {158 if (!overscanOpts->TwoD && (cellreaddir == 1)) { 350 159 // The read direction is rows 351 160 psArray *pixels = psArrayAlloc(image->numRows); // Array of vectors containing pixels … … 369 178 values = psVectorRealloc(values, values->n + overscan->numCols); 370 179 values->n += overscan->numCols; 180 // XXX double-check the range of values->n here 371 181 memcpy(&values->data.F32[index], overscan->data.F32[j], 372 182 overscan->numCols * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); … … 378 188 379 189 // Reduce the overscans 380 psVector *reduced = pmOverscanVector(&chi2, overscanOpts , pixels, stats);190 psVector *reduced = pmOverscanVector(&chi2, overscanOpts->primary, pixels); 381 191 psFree(pixels); 382 192 if (! reduced) { 383 193 psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate overscan vector.\n"); 384 psFree(stats); 385 return false; 386 } 387 388 // generate stats of overscan vector for header 389 { 390 psString comment = NULL; // Comment to add 391 psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); 392 if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) { 393 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); 394 return false; 395 } 396 psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean); 397 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); 398 psFree(comment); 399 400 // write metadata header value 401 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean); 402 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev); 403 404 // EAM 2022.03.29 : if the calculated overscan value is below the threshold, 405 // declare the readout dead and mask 406 407 if ((vectorStats->sampleMean < overscanOpts->minValid) || (vectorStats->sampleMean > overscanOpts->maxValid)) { 408 fprintf (stderr, "bad overscan (2) %f, masking readout\n", vectorStats->sampleMean); 409 psImage *mask = input->mask; 410 for (int y = 0; y < mask->numRows; y++) { 411 for (int x = 0; x < mask->numCols; x++) { 412 mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal; 413 } 414 } 415 } 416 417 psFree (vectorStats); 194 return false; 195 } 196 197 if (!pmOverscanUpdateHeaderVector (input, hdu, overscanOpts, reduced)) { 198 psError(PS_ERR_UNKNOWN, false, "failure to update header"); 199 return false; 418 200 } 419 201 … … 427 209 // subtract from the overscan regions 428 210 { 429 psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator430 psImage *overscan = NULL; // Overscan image from iterator431 while ((overscan = psListGetAndIncrement(iter))) {432 // the overscan and image might not be aligned.433 int diff = overscan->row0 - image->row0; // Offset between the two regions434 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numRows, overscan->numRows + diff); i++) {435 int j = i - diff;436 // i is row on image437 // j is row on overscan438 for (int k = 0; k < overscan->numCols; k++) {439 overscan->data.F32[j][k] -= reduced->data.F32[j];440 }441 }442 }443 psFree(iter);211 psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator 212 psImage *overscan = NULL; // Overscan image from iterator 213 while ((overscan = psListGetAndIncrement(iter))) { 214 // the overscan and image might not be aligned. 215 int diff = overscan->row0 - image->row0; // Offset between the two regions 216 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numRows, overscan->numRows + diff); i++) { 217 int j = i - diff; 218 // i is row on image 219 // j is row on overscan 220 for (int k = 0; k < overscan->numCols; k++) { 221 overscan->data.F32[j][k] -= reduced->data.F32[j]; 222 } 223 } 224 } 225 psFree(iter); 444 226 } 445 227 psFree(reduced); 446 228 } 447 if (cellreaddir == 2) { 229 230 if (!overscanOpts->TwoD && (cellreaddir == 2)) { 448 231 // The read direction is columns 449 232 psArray *pixels = psArrayAlloc(image->numCols); // Array of vectors containing pixels … … 477 260 478 261 // Reduce the overscans 479 psVector *reduced = pmOverscanVector(&chi2, overscanOpts , pixels, stats);262 psVector *reduced = pmOverscanVector(&chi2, overscanOpts->primary, pixels); 480 263 psFree(pixels); 481 264 if (! reduced) { 482 265 psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate overscan vector.\n"); 483 psFree(stats); 484 return false; 485 } 486 487 // generate stats of overscan vector for header 488 { 489 psString comment = NULL; // Comment to add 490 psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); 491 if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) { 492 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); 493 return false; 266 return false; 267 } 268 269 if (!pmOverscanUpdateHeaderVector (input, hdu, overscanOpts, reduced)) { 270 psError(PS_ERR_UNKNOWN, false, "failure to update header"); 271 return false; 272 } 273 274 // Subtract column by column 275 for (int j = 0; j < image->numRows; j++) { 276 for (int i = 0; i < image->numCols; i++) { 277 image->data.F32[j][i] -= reduced->data.F32[i]; 278 } 279 } 280 281 // subtract from the overscan regions 282 { 283 psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator 284 psImage *overscan = NULL; // Overscan image from iterator 285 while ((overscan = psListGetAndIncrement(iter))) { 286 // the overscan and image might not be aligned. 287 int diff = overscan->col0 - image->col0; // Offset between the two regions 288 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numCols, overscan->numCols + diff); i++) { 289 int j = i - diff; 290 // i is col on image 291 // j is col on overscan 292 for (int k = 0; i < overscan->numRows; k++) { 293 overscan->data.F32[k][j] -= reduced->data.F32[j]; 294 } 295 } 296 } 297 psFree(iter); 298 } 299 300 psFree(reduced); 301 } 302 303 // 2D bias subtraction with x-dir readout direction: the 304 // bias is constructed by combining a 1D pattern in the 305 // readout direction from the top overscan region and a second 306 // 1D pattern in the cross direction from the overscan 307 if (overscanOpts->TwoD && (cellreaddir == 1)) { 308 psAssert (overscanOpts->secondary, "2D overscan subtraction requires OVERSCAN.2D parameters"); 309 // we require 2 overscan regions, and they must match these directions: 310 if (overscans->n != 2) { 311 psLogMsg (__func__, PS_LOG_ERROR, "OVERSCAN.2D requires 2 overscan regions but %d supplied", (int) overscans->n); 312 psLogMsg (__func__, PS_LOG_ERROR, "e.g.: CELL.BIASSEC STR [591:624,1:608],[1:590,598:608]"); 313 psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to generate overscan vector.\n"); 314 return false; 315 } 316 317 // the serial (fast readout) direction is columns (x-dir) 318 psImage *yscan = psListGet (overscans, 0); // overscan region spanning all rows 319 psImage *xscan = psListGet (overscans, 1); // overscan region spanning all columns 320 321 // Extract the y-dir overscan vector. The overscan and image might not be aligned: 322 // diff represents the offset between the rows in the image data and the overscan. 323 // pixels->data represents the image row pixels. For example, the image region may be 324 // inset in the y-direction but the overscan could cover the entire y-range 325 326 // The read direction is rows 327 psArray *yscanPixels = psArrayAlloc(image->numRows); // Array of vectors containing pixels 328 for (int i = 0; i < yscanPixels->n; i++) { 329 yscanPixels->data[i] = psVectorAlloc(0, PS_TYPE_F32); 330 } 331 332 // XXX this code allows multiple yscans to be appended, but this does not 333 // match the concept of how they are assigned above: biassec[0] = yscan 334 int yDiff = yscan->row0 - image->row0; // Offset between the two regions 335 for (int i = PS_MAX(0,yDiff); i < PS_MIN(image->numRows, yscan->numRows + yDiff); i++) { 336 int j = i - yDiff; 337 // i is row on image, j is row on yscan 338 psVector *values = yscanPixels->data[i]; 339 int index = values->n; // Index in the vector 340 values = psVectorRealloc(values, values->n + yscan->numCols); 341 values->n += yscan->numCols; 342 // XXX double-check the range of values->n here 343 memcpy(&values->data.F32[index], yscan->data.F32[j], 344 yscan->numCols * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); 345 index += yscan->numCols; 346 yscanPixels->data[i] = values; // Update the pointer in case it's moved 347 } 348 349 // Extract the x-dir overscan vector. The overscan and image might not be aligned: 350 // diff represents the offset between the rows in the image data and the overscan. 351 // pixels->data represents the image row pixels. For example, the image region may be 352 // inset in the x-direction but the overscan could cover the entire y-range 353 354 // Extract the top region as a vector of the columns 355 psArray *xscanPixels = psArrayAlloc(image->numCols); // Array of vectors containing pixels 356 for (int i = 0; i < xscanPixels->n; i++) { 357 xscanPixels->data[i] = psVectorAlloc(0, PS_TYPE_F32); 358 } 359 360 int xDiff = xscan->col0 - image->col0; // Offset between the two regions 361 for (int ix = PS_MAX(0,xDiff); ix < PS_MIN(image->numCols, xscan->numCols + xDiff); ix++) { 362 int jx = ix - xDiff; 363 // ix is row on image, jx is column on xscan 364 psVector *values = xscanPixels->data[ix]; 365 values = psVectorRealloc(values, xscan->numRows); 366 values->n = xscan->numRows; 367 for (int iy = 0; iy < xscan->numRows; iy++) { 368 values->data.F32[iy] = xscan->data.F32[iy][jx]; 369 } 370 xscanPixels->data[ix] = values; // Update the pointer in case it's moved 371 } 372 373 // Reduce the overscans 374 // XXX need to save 2 different chi-square values 375 psVector *yReduced = pmOverscanVector(&chi2, overscanOpts->primary, yscanPixels); 376 psFree(yscanPixels); 377 if (!yReduced) { 378 psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate y-dir overscan vector.\n"); 379 return false; 380 } 381 if (!pmOverscanUpdateHeaderVector (input, hdu, overscanOpts, yReduced)) { 382 psError(PS_ERR_UNKNOWN, false, "failure to update header"); 383 return false; 384 } 385 386 // Reduce the overscans 387 // XXX need to save 2 different chi-square values 388 psVector *xReduced = pmOverscanVector(&chi2, overscanOpts->secondary, xscanPixels); 389 psFree(xscanPixels); 390 if (!xReduced) { 391 psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate x-dir overscan vector.\n"); 392 return false; 393 } 394 395 // XXX apply the 2D bias correction here 396 for (int i = 0; i < image->numRows; i++) { 397 for (int j = 0; j < image->numCols; j++) { 398 image->data.F32[i][j] -= yReduced->data.F32[i]; 399 } 400 } 401 402 // subtract the y-dir vector from the y-dir overscan regions (why?) 403 { 404 // the overscan and image might not be aligned. 405 int diff = yscan->row0 - image->row0; // Offset between the two regions 406 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numRows, yscan->numRows + diff); i++) { 407 int j = i - diff; 408 // i is row on image (aligned with yReduced) 409 // j is row on yscan 410 for (int k = 0; k < yscan->numCols; k++) { 411 yscan->data.F32[j][k] -= yReduced->data.F32[i]; 412 } 413 } 414 } 415 416 // subtract the x-dir vector from the x-dir overscan regions (why?) 417 { 418 // the overscan and image might not be aligned. 419 int diff = xscan->col0 - image->col0; // Offset between the two regions 420 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numCols, xscan->numCols + diff); i++) { 421 int j = i - diff; 422 // i is column on image (aligned with xReduced) 423 // j is column on xscan 424 for (int k = 0; k < xscan->numRows; k++) { 425 xscan->data.F32[k][j] -= xReduced->data.F32[i]; 426 } 427 } 428 } 429 psFree(xReduced); 430 psFree(yReduced); 431 } 432 // pmOverscanUpdateHeader (hdu, overscanOpts, chi2); 433 return true; 434 435 } // End of overscan subtraction 436 437 static void pmOverscanOptionsFree(pmOverscanOptions *options) 438 { 439 psFree(options->primary); 440 psFree(options->secondary); 441 } 442 443 static void pmOverscanStatOptionsFree(pmOverscanStatOptions *options) 444 { 445 psFree(options->stat); 446 psFree(options->poly); 447 psFree(options->spline); 448 } 449 450 // Globally defined 451 pmOverscanStatOptions *pmOverscanStatOptionsAlloc(void) 452 { 453 pmOverscanStatOptions *opts = psAlloc(sizeof(pmOverscanStatOptions)); 454 psMemSetDeallocator(opts, (psFreeFunc)pmOverscanStatOptionsFree); 455 456 // Inputs 457 opts->fitType = PM_FIT_NONE; 458 opts->order = 0; 459 opts->stat = NULL; 460 461 // Smoothing 462 opts->boxcar = 0; 463 opts->gauss = 0.0; 464 465 // Outputs 466 opts->poly = NULL; 467 opts->spline = NULL; 468 469 return opts; 470 } 471 472 // Globally defined 473 pmOverscanOptions *pmOverscanOptionsAlloc(void) 474 { 475 pmOverscanOptions *opts = psAlloc(sizeof(pmOverscanOptions)); 476 psMemSetDeallocator(opts, (psFreeFunc)pmOverscanOptionsFree); 477 478 // Inputs 479 opts->single = false; 480 opts->constant = false; 481 opts->TwoD = false; 482 483 opts->value = 0.0; 484 485 opts->minValid = 0.0; // default value if not defined 486 opts->maxValid = (float) 0x10000; // default value if not defined 487 opts->maskVal = 0x0001; // default value if not defined 488 489 // stat options 490 opts->primary = NULL; 491 opts->secondary = NULL; 492 493 return opts; 494 } 495 496 // Produce an overscan vector from an array of pixels 497 psVector *pmOverscanVector(float *chi2, // chi^2 from fit 498 pmOverscanStatOptions *overscanOpts, // Overscan statistic options 499 const psArray *pixels // Array of vectors containing the pixel values 500 ) 501 { 502 assert(overscanOpts); 503 assert(pixels); 504 505 // statisctic to be calculated 506 psStatsOptions statistic = psStatsSingleOption(overscanOpts->stat->options); // Statistic to use 507 if (!statistic) { 508 psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Multiple or no statistics options set: %p\n", overscanOpts->stat); 509 return false; 510 } 511 psStats *stats = psStatsAlloc(statistic); // A new psStats, to avoid clobbering original 512 513 // Reduce the overscans 514 psVector *reduced = psVectorAlloc(pixels->n, PS_TYPE_F32); // Overscan for each row 515 psVector *ordinate = psVectorAlloc(pixels->n, PS_TYPE_F32); // Ordinate 516 psVector *mask = psVectorAlloc(pixels->n, PS_TYPE_VECTOR_MASK); // Mask for fitting 517 518 for (int i = 0; i < pixels->n; i++) { 519 psVector *values = pixels->data[i]; // Vector with overscan values 520 if (values->n > 0) { 521 mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0; 522 ordinate->data.F32[i] = 2.0*(float)i/(float)pixels->n - 1.0; // Scale to [-1,1] 523 if (!psVectorStats(stats, values, NULL, NULL, 0)) { 524 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); 525 goto escape; 526 } 527 reduced->data.F32[i] = psStatsGetValue(stats, statistic); 528 } else { 529 if (overscanOpts->fitType == PM_FIT_NONE) { 530 psError(PS_ERR_UNKNOWN, true, "The overscan is not supplied for all points on the image, and no fit is requested.\n"); 531 goto escape; 532 } else { 533 // We'll fit this one out 534 mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 1; 535 } 536 } 537 } 538 // Smooth the reduced vector 539 if (overscanOpts->boxcar > 0) { 540 psVector *smoothed = psVectorBoxcar(NULL, reduced, overscanOpts->boxcar); // Smoothed vector 541 psFree(reduced); 542 reduced = smoothed; 543 } 544 if (isfinite(overscanOpts->gauss) && overscanOpts->gauss > 0) { 545 if (overscanOpts->boxcar > 0) { 546 psWarning("Gaussian smoothing the boxcar smoothed overscan --- you asked for it."); 547 } 548 psVector *smoothed = psVectorSmooth(NULL, reduced, overscanOpts->gauss, SMOOTH_NSIGMA); 549 psFree(reduced); 550 reduced = smoothed; 551 } 552 553 // Fit the overscan, if required 554 psVector *fitted = NULL; // Fitted overscan values 555 switch (overscanOpts->fitType) { 556 case PM_FIT_NONE: 557 // No fitting --- that's easy. 558 fitted = psMemIncrRefCounter(reduced); 559 break; 560 case PM_FIT_POLY_ORD: 561 if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || 562 overscanOpts->poly->type != PS_POLYNOMIAL_ORD)) { 563 psFree(overscanOpts->poly); 564 overscanOpts->poly = NULL; 565 } 566 if (! overscanOpts->poly) { 567 overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, overscanOpts->order); 568 } 569 psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); 570 fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); 571 break; 572 case PM_FIT_POLY_CHEBY: 573 if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || 574 overscanOpts->poly->type != PS_POLYNOMIAL_CHEB)) { 575 psFree(overscanOpts->poly); 576 overscanOpts->poly = NULL; 577 } 578 if (! overscanOpts->poly) { 579 overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_CHEB, overscanOpts->order); 580 } 581 psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); 582 fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); 583 break; 584 case PM_FIT_SPLINE: 585 586 // XXX I don't think psSpline1D is up to scratch yet --- it has no mask, and it assumes 587 // a knot for every input point. it needs an argument like 'number of knots' for the 588 // output spline. EAM: still true 2023.01.22 589 590 // overscanOpts->spline = psVectorFitSpline1D(reduced, ordinate); 591 // fitted = psSpline1DEvalVector(overscanOpts->spline, ordinate); 592 psError(PS_ERR_UNKNOWN, true, "Spline overscan fitting is broken\n"); 593 break; 594 default: 595 psError(PS_ERR_UNKNOWN, true, "Unknown value for the fitting type: %d\n", overscanOpts->fitType); 596 goto escape; 597 } 598 599 if (chi2) { 600 *chi2 = 0.0; // chi^2 (sort of) 601 for (int i = 0; i < reduced->n; i++) { 602 *chi2 += PS_SQR(fitted->data.F32[i] - reduced->data.F32[i]); 603 } 604 } 605 606 psFree(reduced); 607 psFree(ordinate); 608 psFree(mask); 609 psFree(stats); 610 return fitted; 611 612 escape: 613 psFree(reduced); 614 psFree(ordinate); 615 psFree(mask); 616 psFree(stats); 617 return NULL; 618 } 619 620 // XXX EAM 2024.04.13 : this function seems poorly conceived. it is replacing the stats from 621 // the reduced overscan vector with the 0-order element from the polynomial fit. Not clear is 622 // adding any useful information 623 bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanStatOptions *overscanOpts, float chi2) { 624 625 psString comment = NULL; // Comment to add 626 627 switch (overscanOpts->fitType) { 628 case PM_FIT_POLY_ORD: 629 case PM_FIT_POLY_CHEBY: { 630 psStringAppend(&comment, "Overscan fit (chi2: %.2f): ", chi2); 631 psPolynomial1D *poly = overscanOpts->poly; // The polynomial 632 for (int i = 0; i < poly->nX; i++) { 633 psStringAppend(&comment, "%.1f ", poly->coeff[i]); 494 634 } 495 psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean);496 635 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); 497 636 psFree(comment); 637 comment = NULL; 498 638 499 639 // write metadata header value 500 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean); 501 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev); 502 503 // EAM 2022.03.29 : if the calculated overscan value is below the threshold, 504 // declare the readout dead and mask 640 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", poly->coeff[0]); 641 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", poly->coeffErr[0]); 642 break; 643 } 644 case PM_FIT_SPLINE: { 645 /* 646 psSpline1D *spline = overscanOpts->spline; // The spline 647 for (int i = 0; i < spline->n; i++) { 648 psStringAppend(&comment, "Overscan fit (chi2: %.2f) %d:", chi2, i); 649 psPolynomial1D *poly = spline->spline[i]; // i-th polynomial 650 for (int j = 0; j < poly->nX; j++) { 651 psStringAppend(&comment, "%.1f ", poly->coeff[i]); 652 } 653 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, 654 comment, ""); 655 psFree(comment); 656 comment = NULL; 657 } 658 */ 659 // write metadata header value 660 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, 661 "Overscan value", NAN); 662 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, 663 "Overscan stdev", NAN); 664 break; 665 } 666 case PM_FIT_NONE: 667 break; 668 default: 669 psAbort("Should never get here!!!\n"); 670 } 671 return true; 672 } 673 674 // generate stats of overscan vector for header 675 // reduced: 1D vector with overscan stats 676 bool pmOverscanUpdateHeaderVector (pmReadout *input, pmHDU *hdu, pmOverscanOptions *overscanOpts, psVector *reduced) { 677 678 psString comment = NULL; // Comment to add 679 psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); 680 if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) { 681 psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); 682 return false; 683 } 684 psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean); 685 psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); 686 psFree(comment); 687 688 // write metadata header value 689 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean); 690 psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev); 691 692 // EAM 2022.03.29 : if the calculated overscan value is below the threshold, 693 // declare the readout dead and mask 505 694 506 if ((vectorStats->sampleMean < overscanOpts->minValid) || (vectorStats->sampleMean > overscanOpts->maxValid)) { 507 fprintf (stderr, "bad overscan (3) %f, masking readout\n", vectorStats->sampleMean); 508 psImage *mask = input->mask; 509 for (int y = 0; y < mask->numRows; y++) { 510 for (int x = 0; x < mask->numCols; x++) { 511 mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal; 512 } 513 } 514 } 515 psFree (vectorStats); 516 } 517 518 // Subtract column by column 519 for (int j = 0; j < image->numRows; j++) { 520 for (int i = 0; i < image->numCols; i++) { 521 image->data.F32[j][i] -= reduced->data.F32[i]; 522 } 523 } 524 525 // subtract from the overscan regions 526 { 527 psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator 528 psImage *overscan = NULL; // Overscan image from iterator 529 while ((overscan = psListGetAndIncrement(iter))) { 530 // the overscan and image might not be aligned. 531 int diff = overscan->col0 - image->col0; // Offset between the two regions 532 for (int i = PS_MAX(0,diff); i < PS_MIN(image->numCols, overscan->numCols + diff); i++) { 533 int j = i - diff; 534 // i is col on image 535 // j is col on overscan 536 for (int k = 0; i < overscan->numRows; k++) { 537 overscan->data.F32[k][j] -= reduced->data.F32[j]; 538 } 539 } 540 } 541 psFree(iter); 542 } 543 544 psFree(reduced); 545 } 546 547 pmOverscanUpdateHeader (hdu, overscanOpts, chi2); 548 psFree(stats); 549 695 if ((vectorStats->sampleMean < overscanOpts->minValid) || (vectorStats->sampleMean > overscanOpts->maxValid)) { 696 fprintf (stderr, "bad overscan (2) %f, masking readout\n", vectorStats->sampleMean); 697 psImage *mask = input->mask; 698 for (int y = 0; y < mask->numRows; y++) { 699 for (int x = 0; x < mask->numCols; x++) { 700 mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= overscanOpts->maskVal; 701 } 702 } 703 } 704 705 psFree (vectorStats); 550 706 return true; 551 552 } // End of overscan subtraction 553 707 } 708 -
branches/2dbias/psModules/src/detrend/pmOverscan.h
r42379 r42664 35 35 { 36 36 // Inputs 37 bool single; ///< Reduce all overscan regions to a single value?38 bool constant; ///< use a supplied constant value (do not measure region)39 float value; ///< supplied value if needed (per above)40 37 pmFit fitType; ///< Type of fit to overscan 41 38 unsigned int order; ///< Order of polynomial, or number of spline pieces … … 43 40 int boxcar; ///< Boxcar smoothing radius 44 41 float gauss; ///< Gaussian smoothing sigma 42 43 // These are used to carry the fitted functions, but are only used to generate 44 // an updated reduced vector 45 psPolynomial1D *poly; ///< Result of polynomial fit 46 psSpline1D *spline; ///< Result of spline fit 47 } pmOverscanStatOptions; 48 49 /// Options for overscan subtraction 50 /// 51 /// The overscan subtraction may be performed by reducing all overscan regions to a single value (e.g., if 52 /// there is no structure); or the overscan may be fit perpendicular to the read direction (usually the 53 /// columns) with a particular functional form; or a single value may be subtracted for each read/scan without 54 /// fitting (if the structure defies characterisation). In any case, statistics are required to reduce 55 /// multiple values to a single value (either for the scan, or for the entire overscan regions). 56 typedef struct 57 { 58 // Inputs 59 bool single; ///< Reduce all overscan regions to a single value? 60 bool constant; ///< use a supplied constant value (do not measure region) 61 bool TwoD; ///< use a supplied constant value (do not measure region) 62 float value; ///< supplied value if needed (per above) 45 63 float minValid; ///< if overscan is too low, readout is dead : mask 46 64 float maxValid; ///< if overscan is too high, readout is dead : mask 47 65 psImageMaskType maskVal; ///< Mask value to give dead readouts 48 66 49 // Outputs 50 psPolynomial1D *poly; ///< Result of polynomial fit 51 psSpline1D *spline; ///< Result of spline fit 52 } 53 pmOverscanOptions; 67 pmOverscanStatOptions *primary; 68 pmOverscanStatOptions *secondary; 69 } pmOverscanOptions; 54 70 55 71 /// Allocator for overscan options 56 pmOverscanOptions *pmOverscanOptionsAlloc(bool single, ///< Reduce all overscan regions to a single value? 57 pmFit fitType, ///< Type of fit to overscan 58 unsigned int order, ///< Order of polynomial, or number of splines 59 psStats *stat, ///< Statistic to use 60 int boxcar, ///< Boxcar smoothing radius 61 float gauss ///< Gaussian smoothing sigma 62 ); 72 pmOverscanOptions *pmOverscanOptionsAlloc(void); 73 74 pmOverscanStatOptions *pmOverscanStatOptionsAlloc(void); 63 75 64 76 psVector *pmOverscanVector(float *chi2, // chi^2 from fit 65 pmOverscanOptions *overscanOpts, // Overscan options 66 const psArray *pixels, // Array of vectors containing the pixel values 67 psStats *myStats // Statistic to use in reducing the overscan 77 pmOverscanStatOptions *overscanOpts, // Overscan options 78 const psArray *pixels // Array of vectors containing the pixel values 68 79 ); 69 80 70 bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanOptions *overscanOpts, float chi2);81 // bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanOptions *overscanOpts, float chi2); 71 82 72 83 bool pmOverscanSubtract (pmReadout *input, pmOverscanOptions *overscanOpts);
Note:
See TracChangeset
for help on using the changeset viewer.
