- Timestamp:
- Mar 20, 2008, 5:08:17 PM (18 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/pap_branch_080320/ppMerge/src/ppMergeScaleZero.c
r16842 r17083 10 10 11 11 #include "ppMerge.h" 12 #include "ppMergeScaleZero.h"13 12 14 13 // Get the scale and zero for each chip of each input 15 bool ppMergeScaleZero(psImage **scales, // The scales for each integration/cell 16 psImage **zeros, // The zeroes for each integration/cell 17 psArray **shutters, // The shutter correction data for each cell 18 ppMergeData *data,// The data 19 const ppMergeOptions *options, // The options 20 const pmConfig *config // The configuration 21 ) 14 bool ppMergeScaleZero(pmConfig *config) 22 15 { 23 assert(data);24 assert(options);25 16 assert(config); 26 17 27 if (!options->scale && !options->zero && !options->shutter) { 28 return true; // We did everything we were asked for 18 ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame 19 int numInputs = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs 20 int numCells = psMetadataLookupS32(NULL, config->arguments, "INPUTS.CELLS"); // Number of cells 21 psStatsOptions meanStat = psMetadataLookupS32(NULL, config->arguments, "MEAN"); // Statistic for mean 22 psStatsOptions stdevStat = psMetadataLookupS32(NULL, config->arguments, "STDEV"); // Statistic for stdev 23 psMaskType maskVal = psMetadataLookupU8(NULL, config->arguments, "MASKVAL"); // Value to mask 24 int shutterSize = psMetadataLookupS32(NULL, config->arguments, "SHUTTER.SIZE"); // Size of shutter region 25 int sample = psMetadataLookupS32(NULL, config->arguments, "SAMPLE"); // Maximum size of sample 26 bool chipStats = false; // Do statistics on full chip instead of cell? 27 28 psVector *gains = NULL; // Gains for each cell 29 psArray *shutters = NULL; // Shutter data for each cell 30 psStats *stats = NULL; // Statistics for background 31 psImage *background = NULL; // Background measurements per cell per file 32 33 switch (type) { 34 case PPMERGE_TYPE_BIAS: 35 case PPMERGE_TYPE_DARK: 36 // Nothing to measure 37 return true; 38 case PPMERGE_TYPE_FLAT: 39 case PPMERGE_TYPE_FRINGE: 40 gains = psVectorAlloc(numCells, PS_TYPE_F32); 41 background = psImageAlloc(numCells, numInputs, PS_TYPE_F32); 42 psImageInit(background, NAN); 43 stats = psStatsAlloc(meanStat); 44 break; 45 case PPMERGE_TYPE_SHUTTER: 46 shutters = psArrayAlloc(numCells); 47 break; 48 case PPMERGE_TYPE_MASK: 49 stats = psStatsAlloc(meanStat | stdevStat); 50 default: 51 break; 29 52 } 30 31 assert(config->camera); // Need the camera configuration 32 assert(config->arguments); // Need the list of files 33 34 psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names 35 assert(filenames); // It should be here --- it's put here in ppMergeConfig 36 37 // Sanity checks 38 assert(!options->scale || scales); 39 assert(!scales || !*scales || ((*scales)->type.type == PS_TYPE_F32 && 40 (*scales)->numCols == data->numCells && 41 (*scales)->numRows == filenames->n)); 42 assert(!options->zero || zeros); 43 assert(!zeros || !*zeros || ((*zeros)->type.type == PS_TYPE_F32 && 44 (*zeros)->numCols == data->numCells && 45 (*zeros)->numRows == filenames->n)); 46 assert(!options->shutter || shutters); 47 assert(!shutters || !*shutters || (*shutters)->n == data->numCells); 48 49 // Allocate the outputs 50 if (options->scale) { 51 if (*scales) { 52 psFree(*scales); 53 } 54 *scales = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); 55 } 56 if (options->zero) { 57 if (*zeros) { 58 psFree(*zeros); 59 } 60 *zeros = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); 61 } 62 psRandom *rng = NULL; // Random number generator 63 if (options->shutter) { 64 if (*shutters) { 65 psFree(*shutters); 66 } 67 *shutters = psArrayAlloc(data->numCells); 68 rng = psRandomAlloc(PS_RANDOM_TAUS, 0); 69 } 70 71 bool fromConcepts = false; // Do we get the scale and zero points from the concepts? 72 bool first = true; // Are we on the first cell (that sets the standard for the rest)? 73 bool done = false; // Are we done going through the list? 74 bool mdok = true; // Status of MD lookup 75 for (long i = 0; i < data->in->n && !done; i++) { 76 pmFPA *fpa = data->in->data[i]; // The FPA 77 if (!fpa) { 78 continue; 79 } 80 long cellNum = -1; // Number of the cell 81 psArray *chips = fpa->chips; // The array of chips 82 for (long j = 0; j < chips->n && !done; j++) { 83 pmChip *chip = chips->data[j]; // The chip 84 if (!chip) { 85 continue; 86 } 87 psArray *cells = chip->cells; // The array of cells 88 for (long k = 0; k < cells->n && !done; k++) { 89 pmCell *cell = cells->data[k]; // The cell 90 if (!cell) { 53 psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator 54 pmFPAview *view = NULL; // View into FPA 55 56 psVector *values = psVectorAlloc(sample, PS_TYPE_F32); // Values for statistics 57 58 for (int i = 0; i < numInputs; i++) { 59 pmFPAfileActivate(config->files, false, NULL); 60 psArray *files = ppMergeFileActivateSingle(config, PPMERGE_FILES_INPUT, true, i); // Activated files 61 pmFPAfile *input = files->data[0]; // Representative file; should be the image (not mask or weight) 62 pmFPA *fpa = input->fpa; // FPA of interest 63 view = pmFPAviewAlloc(0); // View to component of interest 64 int cellNum = 0; // Index for cell 65 if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { 66 goto ERROR; 67 } 68 pmChip *chip; // Chip of interest 69 while ((chip = pmFPAviewNextChip(view, fpa, 1))) { 70 if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { 71 goto ERROR; 72 } 73 74 int valueIndex = 0; // Index into values vector 75 76 pmCell *cell; // Cell of interest 77 while ((cell = pmFPAviewNextCell(view, fpa, 1))) { 78 if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { 79 goto ERROR; 80 } 81 82 if (!cell->data_exists) { 91 83 continue; 92 84 } 85 86 if (cell->readouts->n > 1) { 87 psError(PS_ERR_BAD_PARAMETER_VALUE, true, 88 "File %d chip %d cell %d contains more than one readout (%ld)", 89 i, view->chip, view->cell, cell->readouts->n); 90 goto ERROR; 91 } 92 pmReadout *readout = cell->readouts->data[0]; // Readout of interest 93 94 switch (type) { 95 case PPMERGE_TYPE_FLAT: 96 case PPMERGE_TYPE_FRINGE: { 97 // Extract the gain 98 float gain = psMetadataLookupF32(NULL, cell->concepts, "CELL.GAIN"); // Cell gain 99 if (!isfinite(gain)) { 100 psError(PS_ERR_BAD_PARAMETER_VALUE, false, 101 "CELL.GAIN for file %d chip %d cell %d is not set.", 102 i, view->chip, view->cell); 103 goto ERROR; 104 } 105 gains->data.F32[cellNum] = gain; 106 107 // Measure the background 108 if (!psImageBackground(stats, NULL, readout->image, readout->mask, maskVal, rng)) { 109 psError(PS_ERR_UNKNOWN, false, 110 "Unable to get statistics for file %d chip %d cell %d", 111 i, view->chip, view->cell); 112 goto ERROR; 113 } 114 background->data.F32[cellNum][i] = psStatsGetValue(stats, meanStat); 115 break; 116 } 117 case PPMERGE_TYPE_SHUTTER: { 118 pmShutterCorrectionData *shutter = shutters->data[cellNum]; // Shutter correction data 119 if (!shutter) { 120 shutter = pmShutterCorrectionDataAlloc(readout->image->numCols, 121 readout->image->numRows, 122 shutterSize); 123 shutters->data[cellNum] = shutter; 124 } 125 if (!pmShutterCorrectionAddReadout(shutter, readout, meanStat, stdevStat, 126 maskVal, rng)) { 127 psError(PS_ERR_UNKNOWN, false, 128 "Can't add file %d chip %d cell %d to shutter correction.", 129 i, view->chip, view->cell); 130 goto ERROR; 131 } 132 break; 133 } 134 case PPMERGE_TYPE_MASK: { 135 psImage *image = readout->image, *mask = readout->mask; // Image and mask 136 int numCols = readout->image->numCols, numRows = readout->image->numRows; // Image size 137 int numPix = numCols * numRows; // Number of pixels 138 int numCells = chipStats ? readout->parent->parent->cells->n : 1; // Number of cells 139 int num = PS_MIN(numPix, sample / numCells); // Number of values to add 140 if (!chipStats) { 141 valueIndex = 0; 142 } 143 for (int i = 0; i < num; i++) { 144 int pixel = numPix * psRandomUniform(rng); 145 int x = pixel % numCols; 146 int y = pixel / numCols; 147 if ((mask && (mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal)) || 148 !isfinite(image->data.F32[y][x])) { 149 continue; 150 } 151 152 values->data.F32[valueIndex++] = image->data.F32[y][x]; 153 } 154 155 if (!chipStats) { 156 if (!psVectorStats(stats, values, NULL, NULL, 0)) { 157 psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on readout."); 158 goto ERROR; 159 } 160 161 psMetadataAddF32(cell->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_MEAN, 0, 162 "Mean value of readout", psStatsGetValue(stats, meanStat)); 163 psMetadataAddF32(cell->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_STDEV, 0, 164 "Stdev value of readout", psStatsGetValue(stats, stdevStat)); 165 } 166 break; 167 } 168 default: 169 psAbort("Should never get here."); 170 } 171 93 172 cellNum++; 94 173 95 if (options->scale) { 96 float scale = psMetadataLookupF32(&mdok, cell->concepts, "PPMERGE.SCALE"); // The scale 97 if (mdok && !isnan(scale)) { 98 if (!first && !fromConcepts) { 99 psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " 100 "for some, but not all cells --- we will re-measure it for all cells."); 101 done = true; 102 continue; 103 } 104 fromConcepts = true; 105 (*scales)->data.F32[i][cellNum] = scale; 106 psTrace("ppMerge", 9, "Scale for input %ld, chip %ld, cell %ld: %f\n", 107 i, j, k, scale); 108 } else if (!first && fromConcepts) { 109 psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " 110 "for some, but not all cells --- we will re-measure it for all cells."); 111 fromConcepts = false; 112 done = true; 174 if ((type != PPMERGE_TYPE_MASK || !chipStats) && 175 !pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { 176 goto ERROR; 177 } 178 } 179 180 // Additional run through cells if we want chip-level statistics for masks 181 if (type == PPMERGE_TYPE_MASK && chipStats) { 182 if (!psVectorStats(stats, values, NULL, NULL, 0)) { 183 psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on chip."); 184 goto ERROR; 185 } 186 while ((cell = pmFPAviewNextCell(view, fpa, 1))) { 187 if (!cell->data_exists) { 113 188 continue; 114 189 } 115 } 116 117 if (options->zero) { 118 float zero = psMetadataLookupF32(&mdok, cell->concepts, "PPMERGE.ZERO"); // The zero 119 if (mdok && !isnan(zero)) { 120 if (!first && !fromConcepts) { 121 psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " 122 "for some, but not all cells --- we will re-measure it for all cells."); 123 done = true; 124 continue; 125 } 126 fromConcepts = true; 127 (*zeros)->data.F32[i][cellNum] = zero; 128 psTrace("ppMerge", 9, "Zero for input %ld, chip %ld, cell %ld: %f\n", i, j, k, zero); 129 } else if (!first && fromConcepts) { 130 psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " 131 "for some, but not all cells --- we will re-measure it for all cells."); 132 fromConcepts = false; 133 done = true; 134 continue; 190 psMetadataAddF32(cell->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_MEAN, 0, 191 "Mean value of chip", psStatsGetValue(stats, meanStat)); 192 psMetadataAddF32(cell->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_STDEV, 0, 193 "Stdev value of chip", psStatsGetValue(stats, stdevStat)); 194 if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { 195 goto ERROR; 135 196 } 136 197 } 137 138 first = false; 139 } 140 } 198 } 199 200 if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { 201 goto ERROR; 202 } 203 } 204 205 if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { 206 goto ERROR; 207 } 208 209 psFree(view); 210 211 #if 0 212 // Reset files for reading again 213 for (int i = 0; i < files->n; i++) { 214 pmFPAfile *file = files->data[i]; // File of interest 215 } 216 #endif 217 psFree(files); 141 218 } 142 219 143 if (fromConcepts) { 144 // We've already done everything we need to 145 psFree(rng); 146 return true; 220 psFree(rng); rng = NULL; 221 psFree(stats); stats = NULL; 222 223 // Store results 224 switch (type) { 225 case PPMERGE_TYPE_FRINGE: 226 psMetadataAddImage(config->arguments, PS_LIST_TAIL, "ZEROS", 0, 227 "Zero to subtract from each input cell", background); 228 // Flow through 229 case PPMERGE_TYPE_FLAT: { 230 // Need to normalize over the focal plane 231 if (psTraceGetLevel("ppMerge") > 9) { 232 for (int i = 0; i < gains->n; i++) { 233 psTrace("ppMerge", 10, "Gain for cell %d is %f\n", i, gains->data.F32[i]); 234 } 235 } 236 psVector *fluxes = NULL; // Solution to fluxes 237 if (!pmFlatNormalize(&fluxes, &gains, background)) { 238 psError(PS_ERR_UNKNOWN, false, "Normalisation failed to converge --- continuing anyway."); 239 psFree(fluxes); 240 goto ERROR; 241 } 242 243 psMetadataAddVector(config->arguments, PS_LIST_TAIL, "SCALES", 0, 244 "Scale to divide into each input file", fluxes); 245 psFree(fluxes); // Drop reference 246 break; 247 } 248 case PPMERGE_TYPE_SHUTTER: 249 psMetadataAddArray(config->arguments, PS_LIST_TAIL, "SHUTTER", 0, 250 "Shutter data", shutters); 251 break; 252 default: 253 psAbort("Should never get here."); 147 254 } 148 255 149 psImage *background = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); // Background measurements 150 psImageInit(background, NAN); 151 psStats *bgStats = psStatsAlloc(options->mean); // Statistic to measure the background 152 psVector *gains = NULL; // The gains for each cell 153 if (options->scale) { 154 gains = psVectorAlloc(data->numCells, PS_TYPE_F32); 155 } 156 157 pmFPAview *view = pmFPAviewAlloc(0); 158 159 bool status = true; // Status of getting the scale and zero --- did everything go right? 160 for (int i = 0; i < filenames->n; i++) { 161 psString name = filenames->data[i]; // The name of the file 162 if (!name || strlen(name) == 0) { 163 continue; 164 } 165 psTrace("ppMerge", 9, "Opening %s to get background...\n", name); 166 psFits *inFile = data->files->data[i]; // The FITS file to read 167 pmFPA *fpa = data->in->data[i]; // The FPA for this input 168 int cellNum = -1; // Number of the cell 169 psArray *chips = fpa->chips; // Array of chips 170 for (int j = 0; j < chips->n; j++) { 171 pmChip *chip = chips->data[j]; // The chip of interest 172 if (!chip) { 173 continue; 174 } 175 psArray *cells = chip->cells; // Array of cells 176 for (int k = 0; k < cells->n; k++) { 177 pmCell *cell = cells->data[k]; // The cell of interest 178 if (!cell) { 179 continue; 180 } 181 cellNum++; 182 183 if (!pmCellReadHeader(cell, inFile)) { 184 continue; 185 } 186 187 // Scaling by the background 188 if (options->scale || options->zero) { 189 if (!pmCellRead(cell, inFile, config->database)) { 190 // Nothing here 191 pmCellFreeData(cell); 192 continue; 193 } 194 195 if (cell->readouts->n > 1) { 196 psWarning("File %s chip %d cell %d contains more than one " 197 "readout --- ignoring all but the first.\n", name, j, k); 198 status = false; 199 } 200 201 pmReadout *readout = cell->readouts->data[0]; // The readout of interest 202 psImage *image = readout->image; // The pixels of interest 203 if (!image) { 204 pmCellFreeData(cell); 205 continue; 206 } 207 208 // Get the gain 209 if (options->scale) { 210 bool mdok = true; // Status of MD lookup 211 gains->data.F32[cellNum] = psMetadataLookupF32(&mdok, cell->concepts, "CELL.GAIN"); 212 if (!mdok || isnan(gains->data.F32[cellNum])) { 213 psWarning("CELL.GAIN for file %s chip %d cell %d is not " 214 "set.\n", name, j, k); 215 gains->data.F32[cellNum] = NAN; 216 status = false; 217 } 218 } 219 220 // Get the background 221 int sampleSize = (image->numCols * image->numRows) / options->sample; // Size of sample 222 psVector *sample = psVectorAlloc(sampleSize, PS_TYPE_F32); // Sample of the image 223 psVector *sampleMask = NULL; // Mask for sample 224 if (readout->mask) { 225 sampleMask = psVectorAlloc(sampleSize, PS_TYPE_U8); 226 } 227 psImage *mask = readout->mask; // The mask image 228 for (long i = 0; i < sampleSize; i++) { 229 int j = i * options->sample; // Index into image 230 int x = j % image->numCols; // x index 231 int y = j / image->numCols; // y index 232 sample->data.F32[i] = image->data.F32[y][x]; 233 if (readout->mask) { 234 sampleMask->data.U8[i] = mask->data.U8[y][x]; 235 } 236 } 237 status = psVectorStats(bgStats, sample, sampleMask, NULL, options->combine->maskVal); 238 if (!status) { 239 psTrace("ppMerge", 3, "failed to get stats for for %s, cell %d is %f\n", name, cellNum, 240 background->data.F32[i][cellNum]); 241 psErrorClear(); 242 } 243 psFree(sample); 244 psFree(sampleMask); 245 background->data.F32[i][cellNum] = psStatsGetValue(bgStats, options->mean); 246 psTrace("ppMerge", 3, "Background for %s, cell %d is %f\n", name, cellNum, 247 background->data.F32[i][cellNum]); 248 } 249 250 // Shutter correction 251 if (options->shutter) { 252 if (!pmCellRead(cell, inFile, config->database)) { 253 // Nothing here 254 pmCellFreeData(cell); 255 continue; 256 } 257 258 if (cell->readouts->n > 1) { 259 psWarning("File %s chip %d cell %d contains more than one " 260 "readout --- ignoring all but the first.\n", name, j, k); 261 status = false; 262 } 263 pmReadout *readout = cell->readouts->data[0]; // The readout of interest 264 265 pmShutterCorrectionData *shutter = (*shutters)->data[cellNum]; // Shutter correction data 266 if (!shutter) { 267 shutter = pmShutterCorrectionDataAlloc(readout->image->numCols, 268 readout->image->numRows, 269 options->shutterSize); 270 (*shutters)->data[cellNum] = shutter; 271 } 272 if (!pmShutterCorrectionAddReadout(shutter, readout, options->mean, options->stdev, 273 options->combine->maskVal, rng)) { 274 psWarning("Can't add file %s chip %d cell %d to shutter correction --- ignored.", 275 name, j, k); 276 status = false; 277 } 278 } 279 280 281 pmCellFreeData(cell); 282 } 283 pmChipFreeData(chip); 284 } 285 pmFPAFreeData(fpa); 286 } 256 psFree(background); 257 psFree(shutters); 258 return true; 259 260 ERROR: 261 // Common path for errors 262 psFree(gains); 263 psFree(background); 264 psFree(shutters); 287 265 psFree(rng); 288 psFree( bgStats);266 psFree(stats); 289 267 psFree(view); 290 291 if (options->scale) { 292 // Need to normalize over the focal plane 293 if (psTraceGetLevel("ppMerge") > 9) { 294 for (int i = 0; i < gains->n; i++) { 295 psTrace("ppMerge", 10, "Gain for cell %d is %f\n", i, gains->data.F32[i]); 296 } 297 } 298 psVector *fluxes = NULL; // Solution to fluxes 299 if (!pmFlatNormalize(&fluxes, &gains, background)) { 300 psWarning("Normalisation failed to converge --- continuing anyway.\n"); 301 status = false; 302 } 303 psFree(gains); 304 305 psImage *scalesDeref = *scales; // Dereference the pointer 306 307 for (int i = 0; i < scalesDeref->numRows; i++) { 308 psF32 bg = fluxes->data.F32[i]; 309 for (int j = 0; j < scalesDeref->numCols; j++) { 310 scalesDeref->data.F32[i][j] = bg; 311 } 312 } 313 } 314 315 if (options->zero) { 316 if (!*zeros) { 317 // This is much faster than copying! 318 *zeros = psMemIncrRefCounter(background); 319 } else { 320 *zeros = psImageCopy(*zeros, background, PS_TYPE_F32); 321 } 322 } 323 324 // Diagnostic stuff 325 if (scales && *scales && psTraceGetLevel("ppMerge") > 9) { 326 psImage *scalesDeref = *scales; // Dereference the pointer 327 for (int i = 0; i < scalesDeref->numRows; i++) { 328 for (int j = 0; j < scalesDeref->numCols; j++) { 329 psTrace("ppMerge", 9, "Scale for exposure %d, cell %d is: %f\n", i, j, 330 scalesDeref->data.F32[i][j]); 331 } 332 } 333 } 334 if (zeros && *zeros && psTraceGetLevel("ppMerge") > 9) { 335 psImage *zerosDeref = *zeros; // Dereference the pointer 336 for (int i = 0; i < zerosDeref->numRows; i++) { 337 for (int j = 0; j < zerosDeref->numCols; j++) { 338 psTrace("ppMerge", 9, "Zero for exposure %d, cell %d is: %f\n", i, j, 339 zerosDeref->data.F32[i][j]); 340 } 341 } 342 } 343 344 psFree(background); 345 346 return status; 268 return false; 347 269 } 348 270
Note:
See TracChangeset
for help on using the changeset viewer.
