Index: /tags/ipp-20130307/ippconfig/gpc1/ppImage.config =================================================================== --- /tags/ipp-20130307/ippconfig/gpc1/ppImage.config (revision 35531) +++ /tags/ipp-20130307/ippconfig/gpc1/ppImage.config (revision 35532) @@ -30,4 +30,7 @@ USE.VIDEO.MASK BOOL TRUE # Use a video mask if we have video data? +# aditional auxillary mask applied to corners of all chips in video cells. +# Is only applied if MASK.AUXMASK and USE.VIDEO.MASK are both true +AUX.VIDEO.MASK STR neb://any/gpc1/auxmask/mpg.20130506/mask_video_cell_corners.fits @@ -150,5 +153,5 @@ FRINGE.FILTERS STR y.00000 -# Standard chip processing +# Standard chip processing (NOTE: when changing this be sure to check whether changes should be applied toCHIP_BGPRESEVE and/or CHIP_AUXMASK CHIP METADATA BASE.FITS BOOL FALSE # Save base detrended image? @@ -169,4 +172,5 @@ MASK.BUILD BOOL TRUE # Build internal mask? MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL FALSE # augment with more restrictive auxillary mask PATTERN.ROW BOOL TRUE # Row pattern correction PATTERN.CELL BOOL FALSE # Cell pattern correction @@ -1220,6 +1224,6 @@ END -# Standard chip processing without any background removal -CHIP_PRESERVE_BG METADATA +# Standard chip processing + auxillary mask +CHIP_AUXMASK METADATA BASE.FITS BOOL FALSE # Save base detrended image? BASE.MASK.FITS BOOL FALSE # Save base detrended image? @@ -1232,4 +1236,5 @@ BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE REMNANCE BOOL FALSE # Remnance masking SHUTTER BOOL FALSE # Shutter correction @@ -1238,4 +1243,38 @@ MASK.BUILD BOOL TRUE # Build internal mask? MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL TRUE # augment with more restrictive auxillary mask + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END + +# Standard chip processing without any background removal +CHIP_PRESERVE_BG METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL TRUE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL FALSE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL TRUE # augment with more restrictive auxillary mask PATTERN.ROW BOOL FALSE # Row pattern correction PATTERN.CELL BOOL FALSE # Cell pattern correction Index: /tags/ipp-20130307/ippconfig/recipes/ppImage.config =================================================================== --- /tags/ipp-20130307/ippconfig/recipes/ppImage.config (revision 35531) +++ /tags/ipp-20130307/ippconfig/recipes/ppImage.config (revision 35532) @@ -17,4 +17,6 @@ MASK.LOW BOOL TRUE # Mask pixels below valid range MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails +MASK.AUXMASK BOOL FALSE # augment mask with auxillary mask +AUX.VIDEO.MASK STR NULL VARIANCE.BUILD BOOL FALSE # Build internal variance image ADDNOISE BOOL FALSE # Add noise to degrade an MD image to a 3pi image? @@ -168,4 +170,29 @@ # Standard chip processing CHIP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics. +END + +# Standard chip processing +CHIP_AUXMASK METADATA BASE.FITS BOOL FALSE # Save base detrended image? BASE.MASK.FITS BOOL FALSE # Save base detrended image? Index: /tags/ipp-20130307/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /tags/ipp-20130307/ippconfig/recipes/reductionClasses.mdc (revision 35531) +++ /tags/ipp-20130307/ippconfig/recipes/reductionClasses.mdc (revision 35532) @@ -538,4 +538,11 @@ END +THREEPI_STACK_ALT METADATA + STACK_PPSTACK STR STACK_THREEPI_ALT + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK +END + + # quick stacks QUICKSTACK METADATA @@ -632,4 +639,23 @@ PRESERVE_BG METADATA CHIP_PPIMAGE STR CHIP_PRESERVE_BG + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND +END + +# Background removal, but then add it back in +RESTORE_BG METADATA + CHIP_PPIMAGE STR CHIP_RESTORE_BG CHIP_PSPHOT STR CHIP WARP_PSWARP STR WARP @@ -981,5 +1007,5 @@ STS_DATASET METADATA - CHIP_PPIMAGE STR CHIP + CHIP_PPIMAGE STR CHIP_AUXMASK CHIP_PSPHOT STR CHIP WARP_PSWARP STR WARP @@ -997,4 +1023,28 @@ END +M31_DATASET METADATA + CHIP_PPIMAGE STR CHIP_AUXMASK + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + CNP_DATASET METADATA CHIP_PPIMAGE STR CHIP Index: /tags/ipp-20130307/ppImage/src/Makefile.am =================================================================== --- /tags/ipp-20130307/ppImage/src/Makefile.am (revision 35531) +++ /tags/ipp-20130307/ppImage/src/Makefile.am (revision 35532) @@ -65,5 +65,6 @@ ppImageMemory.c \ ppImageAddNoise.c \ - ppImageRandomGaussian.c + ppImageRandomGaussian.c \ + ppImageAuxiliaryMask.c CLEANFILES = *~ Index: /tags/ipp-20130307/ppImage/src/ppImage.h =================================================================== --- /tags/ipp-20130307/ppImage/src/ppImage.h (revision 35531) +++ /tags/ipp-20130307/ppImage/src/ppImage.h (revision 35532) @@ -30,4 +30,5 @@ bool doMaskLow; // mask low pixels bool doMask; // Mask bad pixels + bool doAuxMask; // apply auxillary mask bool doNonLin; // Non-linearity correction bool doOverscan; // Overscan subtraction @@ -126,4 +127,6 @@ psU16 maskstat_magic; psU16 maskstat_advisory; + + psString auxVideoMask; // auxillary video mask file } ppImageOptions; @@ -312,4 +315,5 @@ void ppImageRandomGaussianFree(void); +bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options); #endif Index: /tags/ipp-20130307/ppImage/src/ppImageAuxiliaryMask.c =================================================================== --- /tags/ipp-20130307/ppImage/src/ppImageAuxiliaryMask.c (revision 35532) +++ /tags/ipp-20130307/ppImage/src/ppImageAuxiliaryMask.c (revision 35532) @@ -0,0 +1,198 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +psImage *readAuxiliaryMask(pmConfig *config, psString fileName) +{ + psString realName = pmConfigConvertFilename(fileName, config, false, false); + if (!realName) { + psError (psErrorCodeLast(), false, "unable to resolve %s", fileName); + return NULL; + } + + psFits *fits = psFitsOpen(realName, "r"); + if (!fits) { + psError (psErrorCodeLast(), false, "psFitsOpen failed for %s", realName); + psFree(realName); + return NULL; + } + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psFree(fits); + psError (psErrorCodeLast(), false, "psFitsReadHeader failed for %s", realName); + psFree(realName); + return NULL; + } + psRegion region = {0, 0, 0, 0}; + psImage *image = psFitsReadImage(fits, region, 0); + psFree(fits); + psFree(header); + if (!image) { + psError (psErrorCodeLast(), false, "psFitsReadImage failed for %s", realName); + psFree(realName); + return NULL; + } + psFree(realName); + + return image; +} + +bool recordFileInHeader(pmChip *chip, psString tag, psString desc, psString filename) +{ + pmHDU *hdu = pmHDUGetHighest(chip->parent, chip, NULL); + + // strip off the directories and nebulous bits + char *base = filename; + for (char *new = base; (new = strpbrk(base, "/:")); base = new + 1); + + psMetadataAddStr(hdu->header, PS_LIST_TAIL, tag, PS_META_DUPLICATE_OK, desc, base); + return true; +} + +// In this function, we augment the mask with the more conservative auxiliary mask +bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options) +{ + psAssert(config, "Need configuration"); + psAssert(view, "Need view to chip"); + psAssert(options, "Need options"); + + if (!options->doAuxMask) { + return true; + } + + bool status; + // Our target chip + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.CHIP"); // File to correct + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return false; + } + + // Find a suitable detRun with type AUXMASK + + psTime *time = psMetadataLookupPtr(NULL, input->fpa->concepts, "FPA.TIME"); + if (time->sec == 0 && time->nsec == 0) { + psLogMsg ("psModules.camera", PS_LOG_WARN, "FPA.TIME has not been set.\n"); + } + + char *cameraName = psMetadataLookupStr(NULL, input->fpa->concepts, "FPA.CAMERA"); + pmDetrendSelectOptions *detrendOptions = pmDetrendSelectOptionsAlloc(cameraName, *time, PM_DETREND_TYPE_AUXMASK); + + pmDetrendSelectResults *results = pmDetrendSelect(detrendOptions, config); + psFree(detrendOptions); + if (!results) { + psError (psErrorCodeLast(), false, "no matching auxiliary mask found"); + return false; + } + + pmFPALevel fileLevel = pmFPALevelFromName(results->level); + if (fileLevel != PM_FPA_LEVEL_CHIP) { + psError (PM_ERR_CONFIG, false, "invalid file level %d for selected auxiliary mask", fileLevel); + return false; + } + + // Go find the readout + // code to find the readouts was adapted from ppImageSubtractBackground + // XXX: shouldn't most of these psWarnings be psAsserts? + + // Since we are working on a chip-mosaicked image, there should only be a single cell and readout + pmChip *chip = pmFPAviewThisChip(view, input->fpa); // Chip of interest + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip"); + return false; + } + if (chip->cells->n == 0) { + psWarning("Chip has no cells"); + return true; + } + if (chip->cells->n > 1) { + psWarning("Chip has %ld cells; only the first will be processed", chip->cells->n); + } + pmCell *cell = chip->cells->data[0]; // Cell of interest + if (!cell || !cell->process || !cell->file_exists) { + // Nothing to process + return true; + } + if (cell->readouts->n == 0) { + psWarning("Cell has no readouts"); + return true; + } + if (cell->readouts->n > 1) { + psWarning("Cell has %ld readouts; only the first will be processed", cell->readouts->n); + } + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + if (!ro || !ro->data_exists) { + // Nothing to process + return true; + } + // not needed psImage *image = ro->image; + psImage *mask = ro->mask; + + // now read the auxiliary mask file + psString class_id = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + + char *auxMaskFileName = pmDetrendFile(results->detID, class_id, config); + if (!auxMaskFileName) { + psError (PM_ERR_CONFIG, false, "unable to find auxiliary mask file name for %s %s", results->detID, class_id); + return false; + } + + psFree(results); + // record that we read this file in the config dump file. + // Note: this value isn't used during updates though. + pmConfigRunFilenameAddRead(config, "PPIMAGE.AUXMASK", auxMaskFileName); + recordFileInHeader(chip, "DETREND.AUXMASK", "auxiliary mask", auxMaskFileName); + + psImage *auxMask = readAuxiliaryMask(config, auxMaskFileName); + psFree(auxMaskFileName); + if (!auxMask) { + psError(PS_ERR_UNKNOWN, false, "failed to read auxiliary mask file"); + return false; + } + + // if the cell has video and the recipe value + if (options->hasVideo && options->auxVideoMask && strcmp(options->auxVideoMask, "NULL")) { + psImage *videoMask = readAuxiliaryMask(config, options->auxVideoMask); + if (!videoMask) { + psError(PS_ERR_UNKNOWN, false, "failed to read auxiliary video mask file"); + return false; + } + pmConfigRunFilenameAddRead(config, "PPIMAGE.AUXVIDEOMASK", options->auxVideoMask); + recordFileInHeader(chip, "DETREND.AUXVIDEOMASK", "auxiliary video mask", options->auxVideoMask); + + // compute auxMask *= videoMask + if (!psBinaryOp(auxMask, auxMask, "*", videoMask)) { + psError(PS_ERR_UNKNOWN, false, "mulitplication of auxiliary mask and auxiliary video mask failed"); + return false; + } + psFree(videoMask); + } + + if ((mask->numRows != auxMask->numRows) || (mask->numCols != auxMask->numCols) || + (mask->row0 != auxMask->row0) || (mask->col0 != auxMask->col0)) { + psError(PS_ERR_IO, false, "structure of auxiliary mask does not match this chip"); + return false; + } + + psImageMaskType maskDetector = pmConfigMaskGet("DETECTOR", config); + + int numCols = mask->numCols, numRows = mask->numRows; // Size of image + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + // auxMask is floating point type image with pixel value 0 meaning mask it + // XXX: support other types and get the mask value from the header + if (auxMask->data.F32[y][x] == 0) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskDetector; + // I don't need to do this. + // The background subtraction code runs after us and will do it there + // image->data.F32[y][x] = 0.0; + } + } + } + + psFree(auxMask); + + return true; +} Index: /tags/ipp-20130307/ppImage/src/ppImageDetrendReadout.c =================================================================== --- /tags/ipp-20130307/ppImage/src/ppImageDetrendReadout.c (revision 35531) +++ /tags/ipp-20130307/ppImage/src/ppImageDetrendReadout.c (revision 35532) @@ -50,5 +50,5 @@ char *Vptr = strchr(psMetadataLookupStr(NULL,input->parent->parent->hdu->header,"CELLMODE"),'V'); if (Vptr) { - hasVideo = true; + hasVideo = options->hasVideo = true; psLogMsg ("ppImage.detrend", PS_LOG_INFO, "VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); } Index: /tags/ipp-20130307/ppImage/src/ppImageLoop.c =================================================================== --- /tags/ipp-20130307/ppImage/src/ppImageLoop.c (revision 35531) +++ /tags/ipp-20130307/ppImage/src/ppImageLoop.c (revision 35532) @@ -188,4 +188,9 @@ ESCAPE("Unable to mosaic chip"); } + + if (!ppImageAuxiliaryMask(config, view, options)) { + ESCAPE("Unable to apply auxiliary mask"); + } + timeDetrend += psTimerClear(TIMER_DETREND); Index: /tags/ipp-20130307/ppImage/src/ppImageOptions.c =================================================================== --- /tags/ipp-20130307/ppImage/src/ppImageOptions.c (revision 35531) +++ /tags/ipp-20130307/ppImage/src/ppImageOptions.c (revision 35532) @@ -10,4 +10,5 @@ // psFree(options->nonLinearData); // psFree(options->nonLinearSource); + psFree(options->auxVideoMask); } @@ -24,4 +25,5 @@ options->doVarianceBuild = false; // Build internal variance options->doMask = false; // Mask bad pixels + options->doAuxMask = false; // apply auxillary mask options->doNonLin = false; // Non-linearity correction options->doOverscan = false; // Overscan subtraction @@ -116,4 +118,6 @@ options->normClass = NULL; // per-class normalizations refer to this class + options->auxVideoMask = NULL; // auxillary video mask file name + return options; } @@ -228,4 +232,14 @@ options->doMaskBurntool = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL"); options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD"); + options->doAuxMask = psMetadataLookupBool(NULL, recipe, "MASK.AUXMASK"); + if (options->doAuxMask) { + // if we are applying an auxiliary mask we can optionally apply another + // mask to video cells only. + psString auxVideoMask = psMetadataLookupStr(NULL, recipe, "AUX.VIDEO.MASK"); + // save the value if defined and not the value "NULL" + if (auxVideoMask && strcmp(auxVideoMask, "NULL")) { + options->auxVideoMask = psStringCopy(auxVideoMask); + } + } // Mask recipe options (note that mask bit values are set in ppImageSetMaskBits.c) Index: /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.c =================================================================== --- /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.c (revision 35531) +++ /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.c (revision 35532) @@ -110,4 +110,5 @@ DETREND_STRING_CASE(VIDEODARK); DETREND_STRING_CASE(LINEARITY); + DETREND_STRING_CASE(AUXMASK); default: return NULL; Index: /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.h =================================================================== --- /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.h (revision 35531) +++ /tags/ipp-20130307/psModules/src/detrend/pmDetrendDB.h (revision 35532) @@ -39,4 +39,5 @@ PM_DETREND_TYPE_VIDEODARK, PM_DETREND_TYPE_LINEARITY, + PM_DETREND_TYPE_AUXMASK, } pmDetrendType;