Index: trunk/ppSub/src/Makefile.am =================================================================== --- trunk/ppSub/src/Makefile.am (revision 23642) +++ trunk/ppSub/src/Makefile.am (revision 23688) @@ -19,4 +19,5 @@ ppSubBackground.c \ ppSubCamera.c \ + ppSubData.c \ ppSubLoop.c \ ppSubReadout.c \ Index: trunk/ppSub/src/ppSub.h =================================================================== --- trunk/ppSub/src/ppSub.h (revision 23642) +++ trunk/ppSub/src/ppSub.h (revision 23688) @@ -22,4 +22,20 @@ #define PPSUB_RECIPE "PPSUB" /// Name of the recipe to use +// Output files, for activation/deactivation +typedef enum { + PPSUB_FILES_IMAGE = 0x01, // Image files + PPSUB_FILES_PHOT = 0x02, // Photometry files + PPSUB_FILES_ALL = 0xFF, // All files +} ppSubFiles; + +/// Data for processing +typedef struct { + psErrorCode quality; /// Quality code; 0 for no problem + psMetadata *stats; /// Statistics +} ppSubData; + +/// Constructor +ppSubData *ppSubDataAlloc(void); + /// Setup the arguments parsing bool ppSubArgumentsSetup(int argc, char *argv[], ///< Command-line arguments @@ -41,5 +57,5 @@ /// Perform PSF-matched image subtraction on the readout bool ppSubReadout(pmConfig *config, ///< Configuration - psMetadata *stats, ///< Statistics, for output + ppSubData *data, ///< Processing data const pmFPAview *view ///< View of readout to subtract ); @@ -53,4 +69,5 @@ /// looking up the parameters in the recipe and supplying them to the function pmSubtractionMatch() bool ppSubMatchPSFs(pmConfig *config, ///< Configuration + ppSubData *data, ///< Processing data const pmFPAview *view ///< View of active readout ); @@ -63,4 +80,5 @@ /// Photometry stage 1: measure the PSF from the minuend image bool ppSubMakePSF(pmConfig *config, ///< Configuration + ppSubData *data, ///< Processing data const pmFPAview *view ///< View of active readout ); @@ -74,5 +92,5 @@ /// Photometry stage 2: find and measure sources on the subtracted image bool ppSubReadoutPhotometry(pmConfig *config, ///< Configuration - psMetadata *stats, ///< Statistics, for output + ppSubData *data, ///< Processing data const pmFPAview *view ///< View of active readout ); @@ -80,5 +98,5 @@ /// Renormalize, update headers and generate JPEGs bool ppSubReadoutUpdate(pmConfig *config, ///< Configuration - psMetadata *stats, ///< Statistics for output, or NULL + ppSubData *data, ///< Processing data const pmFPAview *view ///< View of active readout ); @@ -96,4 +114,10 @@ void ppSubVersionPrint(void); +/// Mark the data quality as bad and prepare to suspend processing +void ppSubDataQuality(pmConfig *config, ///< Configuration + ppSubData *data, ///< Processing data + psErrorCode error,///< Error code + ppSubFiles files ///< Files to deactivate + ); // Copy every instance of a single keyword from one metadata to another Index: trunk/ppSub/src/ppSubCamera.c =================================================================== --- trunk/ppSub/src/ppSubCamera.c (revision 23642) +++ trunk/ppSub/src/ppSubCamera.c (revision 23688) @@ -285,2 +285,4 @@ return true; } + + Index: trunk/ppSub/src/ppSubData.c =================================================================== --- trunk/ppSub/src/ppSubData.c (revision 23688) +++ trunk/ppSub/src/ppSubData.c (revision 23688) @@ -0,0 +1,63 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppSub.h" + + +// Image files to activate/deactivate +static const char *imageFiles[] = { "PPSUB.OUTPUT", "PPSUB.OUTPUT.MASK", "PPSUB.OUTPUT.VARIANCE", + "PPSUB.OUTPUT.KERNELS", "PPSUB.OUTPUT.JPEG1", "PPSUB.OUTPUT.JPEG2", + "PPSUB.INPUT.CONV", "PPSUB.INPUT.CONV.MASK", "PPSUB.INPUT.CONV.VARIANCE", + "PPSUB.REF.CONV", "PPSUB.REF.CONV.MASK", "PPSUB.REF.CONV.VARIANCE", + NULL }; + + + +static void subOptionsFree(ppSubData *options) +{ + psFree(options->stats); + return; +} + +ppSubData *ppSubDataAlloc(void) +{ + ppSubData *options = psAlloc(sizeof(ppSubData)); // Processing data, to return + psMemSetDeallocator(options, (psFreeFunc)subOptionsFree); + + options->quality = 0; + options->stats = psMetadataAlloc(); + psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", 0, "Data quality", 0); + + return options; +} + + +void ppSubDataQuality(pmConfig *config, ppSubData *data, psErrorCode error, ppSubFiles files) +{ + psAssert(config, "Require configuration"); + psAssert(data, "Require processing data"); + + if (psMetadataLookupS32(NULL, data->stats, "QUALITY") == 0) { + data->quality = error; + psMetadataAddS32(data->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "Data quality", error); + } + + if (files & PPSUB_FILES_IMAGE) { + for (int i = 0; imageFiles[i]; i++) { + pmFPAfileActivate(config->files, imageFiles[i], false); + } + } + if (files & PPSUB_FILES_PHOT) { + psphotFilesActivate(config, false); + } + + psErrorClear(); + + return; +} Index: trunk/ppSub/src/ppSubLoop.c =================================================================== --- trunk/ppSub/src/ppSubLoop.c (revision 23642) +++ trunk/ppSub/src/ppSubLoop.c (revision 23688) @@ -28,7 +28,8 @@ pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,MASKS,JPEG"); + ppSubData *data = ppSubDataAlloc(); // Processing data + bool mdok; // Status of MD lookup const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics - psMetadata *stats = NULL; // Container for statistics FILE *statsFile = NULL; // File stream for statistics if (statsName && strlen(statsName) > 0) { @@ -38,7 +39,5 @@ psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); psFree(resolved); - return false; - } else { - stats = psMetadataAlloc(); + goto ERROR; } psFree(resolved); @@ -48,5 +47,5 @@ if (!input) { psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find input data!\n"); - return false; + goto ERROR; } @@ -54,5 +53,5 @@ if (!reference) { psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find reference data!\n"); - return false; + goto ERROR; } @@ -60,5 +59,5 @@ if (!output) { psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!\n"); - return false; + goto ERROR; } @@ -67,5 +66,5 @@ // Iterate over the FPA hierarchy if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; + goto ERROR; } @@ -77,5 +76,5 @@ psError(PS_ERR_BAD_PARAMETER_VALUE, true, "FPA format discrepency between input and reference"); psFree(view); - return false; + goto ERROR; } @@ -85,5 +84,5 @@ if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; + goto ERROR; } @@ -96,5 +95,5 @@ "FPA format discrepency between input and reference"); psFree(view); - return false; + goto ERROR; } if (!inCell->file_exists) { @@ -102,5 +101,5 @@ } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; + goto ERROR; } @@ -108,5 +107,5 @@ while ((inRO = pmFPAviewNextReadout(view, input->fpa, 1))) { if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; + goto ERROR; } pmReadout *refRO = pmFPAviewThisReadout(view, reference->fpa);// Reference readout of interest @@ -116,5 +115,5 @@ "FPA format discrepency between input and reference"); psFree(view); - return false; + goto ERROR; } if (!inRO->data_exists) { @@ -123,37 +122,37 @@ // Perform the analysis - if (!ppSubReadout(config, stats, view)) { + if (!ppSubReadout(config, data, view)) { psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.\n"); - return false; + goto ERROR; } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; + goto ERROR; } } // Perform statistics on the cell - if (stats) { + if (statsFile) { pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); // Output file if (!output) { psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PPSUB.OUTPUT.\n"); - return false; + goto ERROR; } psImageMaskType maskValue = pmConfigMaskGet("MASK.VALUE", config); - ppStatsFPA(stats, output->fpa, view, maskValue, config); + ppStatsFPA(data->stats, output->fpa, view, maskValue, config); } if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { - return false; + goto ERROR; } } if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { - return false; + goto ERROR; } } if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { - return false; + goto ERROR; } @@ -161,9 +160,9 @@ // Write out summary statistics - if (stats) { - psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_SUB", 0, "Time for subtraction completion", + if (statsFile) { + psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_SUB", 0, "Time for subtraction completion", psTimerMark("ppSub")); - const char *statsMDC = psMetadataConfigFormat(stats); + const char *statsMDC = psMetadataConfigFormat(data->stats); if (!statsMDC || strlen(statsMDC) == 0) { psWarning("Unable to generate statistics MDC file.\n"); @@ -173,15 +172,17 @@ psFree((void *)statsMDC); fclose(statsFile); - - psFree(stats); } psString dump_file = psMetadataLookupStr(&mdok, config->arguments, "-dumpconfig"); if (dump_file) { - pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file pmConfigDump(config, input->fpa, dump_file); } + psFree(data); return true; + +ERROR: + psFree(data); + return false; } Index: trunk/ppSub/src/ppSubMakePSF.c =================================================================== --- trunk/ppSub/src/ppSubMakePSF.c (revision 23642) +++ trunk/ppSub/src/ppSubMakePSF.c (revision 23688) @@ -22,5 +22,5 @@ #include "ppSub.h" -bool ppSubMakePSF(pmConfig *config, const pmFPAview *view) +bool ppSubMakePSF(pmConfig *config, ppSubData *data, const pmFPAview *view) { psAssert(config, "Require configuration"); @@ -83,6 +83,10 @@ psArray *sources = psMetadataLookupPtr(&mdok, minuend->analysis, "PSPHOT.SOURCES"); if (!psphotReadoutFindPSF(config, view, sources)) { - psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on subtracted image."); - return false; + // This is likely a data quality issue + // XXX Split into multiple cases using error codes? + psErrorStackPrint(stderr, "Unable to determine PSF"); + psWarning("Unable to determine PSF --- suspect bad data quality."); + ppSubDataQuality(config, data, PSPHOT_ERR_PSF, PPSUB_FILES_PHOT); + return true; } @@ -98,21 +102,2 @@ return true; } - -// XXX we used to need this, is it still needed? - -// pmCell *photCell = pmFPAfileThisCell(config->files, view, "PSPHOT.INPUT"); -// pmCellFreeReadouts(photCell); - -// if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") || -// !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") || -// !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) { -// psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files."); -// return false; -// } - -// Blow away the sources psphot found --- they're irrelevant for the subtraction -// XXX is this still needed? These are now probably not being set -// pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with sources -// psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES"); -// psMetadataRemoveKey(photRO->analysis, "PSPHOT.HEADER"); - Index: trunk/ppSub/src/ppSubMatchPSFs.c =================================================================== --- trunk/ppSub/src/ppSubMatchPSFs.c (revision 23642) +++ trunk/ppSub/src/ppSubMatchPSFs.c (revision 23688) @@ -22,5 +22,5 @@ #include "ppSub.h" -bool ppSubMatchPSFs(pmConfig *config, const pmFPAview *view) +bool ppSubMatchPSFs(pmConfig *config, ppSubData *data, const pmFPAview *view) { psAssert(config, "Require configuration"); @@ -136,15 +136,27 @@ // Match the PSFs + bool success = false; // Operation was successful? if (kernelRO) { - if (!pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis, - stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac)) { - psError(PS_ERR_UNKNOWN, false, "Unable to convolve images."); - return false; - } + success = pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis, + stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac); } else { - if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing, - threshold, sources, stampsName, type, size, order, widths, orders, inner, - ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter, - rej, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) { + success = pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, + spacing, threshold, sources, stampsName, type, size, order, + widths, orders, inner, ringsOrder, binning, penalty, optimum, + optWidths, optOrder, optThresh, iter, rej, sys, maskVal, + maskBad, maskPoor, poorFrac, badFrac, subMode); + } + + psFree(optWidths); + pmSubtractionThreadsFinalize(inRO, refRO); + + if (!success) { + psErrorCode error = psErrorCodeLast(); // Error code + if (error == PM_ERR_STAMPS) { + psErrorStackPrint(stderr, "Unable to find stamps"); + psWarning("Unable to find stamps --- suspect bad data quality."); + ppSubDataQuality(config, data, error, PPSUB_FILES_ALL); + return true; + } else { psError(PS_ERR_UNKNOWN, false, "Unable to match images."); return false; @@ -152,19 +164,7 @@ } - psFree(optWidths); - - pmSubtractionThreadsFinalize(inRO, refRO); - psImageCovarianceTransfer(inConv->variance, inConv->covariance); psImageCovarianceTransfer(refConv->variance, refConv->covariance); - // XXX drop the pixels associated with inRO and refRO (now that we have inConv and refConf) -#ifdef TESTING - psphotSaveImage (NULL, inRO->image, "inRO.fits"); - psphotSaveImage (NULL, refRO->image, "refRO.fits"); - psphotSaveImage (NULL, inConv->image, "inConv.fits"); - psphotSaveImage (NULL, refConv->image, "refConv.fits"); -#endif - return true; } Index: trunk/ppSub/src/ppSubReadout.c =================================================================== --- trunk/ppSub/src/ppSubReadout.c (revision 23642) +++ trunk/ppSub/src/ppSubReadout.c (revision 23688) @@ -21,5 +21,5 @@ #include "ppSub.h" -bool ppSubReadout(pmConfig *config, psMetadata *stats, const pmFPAview *view) +bool ppSubReadout(pmConfig *config, ppSubData *data, const pmFPAview *view) { psTimerStart("PPSUB_MATCH"); @@ -30,7 +30,10 @@ } - if (!ppSubMatchPSFs(config, view)) { + if (!ppSubMatchPSFs(config, data, view)) { psError(PS_ERR_UNKNOWN, false, "Unable to match PSFs."); return false; + } else if (data->quality) { + // Can't do anything at all + return true; } @@ -40,5 +43,5 @@ } - if (!ppSubMakePSF(config, view)) { + if (!data->quality && !ppSubMakePSF(config, data, view)) { psError(PS_ERR_UNKNOWN, false, "Unable to generate PSF."); return false; @@ -56,10 +59,10 @@ } - if (!ppSubReadoutPhotometry(config, stats, view)) { + if (!data->quality && !ppSubReadoutPhotometry(config, data, view)) { psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry."); return false; } - if (!ppSubReadoutUpdate(config, stats, view)) { + if (!ppSubReadoutUpdate(config, data, view)) { psError(PS_ERR_UNKNOWN, false, "Unable to update."); return false; Index: trunk/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- trunk/ppSub/src/ppSubReadoutPhotometry.c (revision 23642) +++ trunk/ppSub/src/ppSubReadoutPhotometry.c (revision 23688) @@ -22,5 +22,5 @@ #include "ppSub.h" -bool ppSubReadoutPhotometry (pmConfig *config, psMetadata *stats, const pmFPAview *view) +bool ppSubReadoutPhotometry (pmConfig *config, ppSubData *data, const pmFPAview *view) { psAssert(config, "Require configuration"); @@ -45,6 +45,8 @@ pmPSF *psf = psMetadataLookupPtr(NULL, psfInputChip->analysis, "PSPHOT.PSF"); // PSF for photometry if (!psf) { - psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF from psphot"); - return false; + psErrorStackPrint(stderr, "No PSF available"); + psWarning("No PSF available --- suspect bad data quality."); + ppSubDataQuality(config, data, psErrorCodeLast(), PPSUB_FILES_PHOT); + return true; } psMetadataAddPtr(psfLoadChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE, @@ -80,25 +82,21 @@ } -#if 0 - psMetadataAddPtr(config->files, PS_LIST_TAIL, "PSPHOT.INPUT", PS_DATA_UNKNOWN | PS_META_REPLACE, - "psphot input: view on another pmFPAfile", photFile); -#endif + if (!psphotReadoutMinimal(config, view)) { + // This is likely a data quality issue + // XXX Split into multiple cases using error codes? + psErrorStackPrint(stderr, "Unable to perform photometry on image"); + psWarning("Unable to perform photometry on image --- suspect bad data quality."); + ppSubDataQuality(config, data, psErrorCodeLast(), PPSUB_FILES_PHOT); + } - if (!psphotReadoutMinimal(config, view)) { - psWarning("Unable to perform photometry on subtracted image."); - psErrorStackPrint(stderr, "Error stack from photometry:"); - psErrorClear(); - } -#if 1 photRO->data_exists = true; photRO->parent->data_exists = true; photRO->parent->parent->data_exists = true; -#endif - if (stats) { + if (data->stats) { psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources - psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", + psMetadataAddS32(data->stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", sources ? sources->n : 0); - psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry", + psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry", psTimerClear("PPSUB_PHOT")); } Index: trunk/ppSub/src/ppSubReadoutUpdate.c =================================================================== --- trunk/ppSub/src/ppSubReadoutUpdate.c (revision 23642) +++ trunk/ppSub/src/ppSubReadoutUpdate.c (revision 23688) @@ -21,5 +21,5 @@ #include "ppSub.h" -bool ppSubReadoutUpdate(pmConfig *config, psMetadata *stats, const pmFPAview *view) +bool ppSubReadoutUpdate(pmConfig *config, ppSubData *data, const pmFPAview *view) { psAssert(config, "Require configuration"); @@ -47,18 +47,18 @@ // Statistics on the matching - if (stats) { - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY); - psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY); + if (data->stats) { + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY); + psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY); - psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs", + psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs", psTimerClear("PPSUB_MATCH")); }