Index: /trunk/ppImage/src/Makefile.am =================================================================== --- /trunk/ppImage/src/Makefile.am (revision 6816) +++ /trunk/ppImage/src/Makefile.am (revision 6817) @@ -12,19 +12,18 @@ ppImage_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) ppImage_SOURCES = \ - ppFile.c \ ppImage.c \ ppImageConfig.c \ - ppImageData.c \ - ppImageDetrendBias.c \ + ppImageOptions.c \ + ppImageParseDetrend.c \ + ppImageLoop.c \ ppImageDetrendCell.c \ ppImageDetrendMask.c \ + ppImageDetrendBias.c \ ppImageDetrendNonLinear.c \ - ppImageLoop.c \ - ppImageOptions.c \ - ppImageParseCamera.c \ - ppImageWeights.c \ ppMem.c +# ppImageParseCamera.c \ +# ppDetrendFlat.c \ +# ppImageWeights.c \ # ppImageLoadPixels.c -# ppDetrendFlat.c # ppImageOutput.c # ppImageDetrendPedestal.c Index: unk/ppImage/src/ppConfig.c =================================================================== --- /trunk/ppImage/src/ppConfig.c (revision 6816) +++ (revision ) @@ -1,27 +1,0 @@ -#include -#include "pslib.h" -#include "ppConfig.h" - -static void configFree(ppConfig *config) -{ - psFree(config->site); - psFree(config->camera); - psFree(config->recipe); - psFree(config->arguments); - psFree(config->database); -} - -ppConfig *ppConfigAlloc(void) -{ - ppConfig *config = psAlloc(sizeof(ppConfig)); - psMemSetDeallocator(config, (psFreeFunc)configFree); - - config->site = NULL; - config->camera = NULL; - config->recipe = NULL; - config->arguments = NULL; - config->database = NULL; - - return config; -} - Index: unk/ppImage/src/ppConfig.h =================================================================== --- /trunk/ppImage/src/ppConfig.h (revision 6816) +++ (revision ) @@ -1,16 +1,0 @@ -#ifndef PP_CONFIG_H -#define PP_CONFIG_H - -#include "pslib.h" - -typedef struct { - psMetadata *site; // Site configuration - psMetadata *camera; // Camera specification - psMetadata *recipe; // Recipe for processing - psMetadata *arguments; // Command-line arguments - psDB *database; // Database handle -} ppConfig; - -ppConfig *ppConfigAlloc(void); - -#endif Index: unk/ppImage/src/ppFile.c =================================================================== --- /trunk/ppImage/src/ppFile.c (revision 6816) +++ (revision ) @@ -1,52 +1,0 @@ -#include -#include "pslib.h" -#include "ppFile.h" - -// Free a ppFile -static void fileFree(ppFile *file) -{ - psFree(file->filename); - psFree(file->fpa); - psFree(file->phu); - if (file->fits) { - psFitsClose(file->fits); - } - psFree(file->view); - return; -} - -ppFile *ppFileAlloc(void) -{ - - ppFile *file = psAlloc(sizeof(ppFile)); - psMemSetDeallocator(file, (psFreeFunc)fileFree); - - file->filename = NULL; - file->fpa = NULL; - file->fits = NULL; - file->phu = NULL; - file->view = NULL; - - return file; -} - - -bool ppFileOpen(ppFile *file, const char *name, pmConfig *config, psMetadata *camera, psMetadata *format) -{ - file->fpa = pmFPAConstruct(camera); - psString filename = psMetadataLookupStr(NULL, config->arguments, name); - if (strlen(filename) == 0) { - return false; - } - file->filename = psMemIncrRefCounter(filename); - file->fits = psFitsOpen(filename, "r"); - file->phu = psFitsReadHeader(NULL, file->fits); - if (!pmConfigValidateCameraFormat(format, file->phu)) { - psLogMsg(__func__, PS_LOG_WARN, "File %s doesn't match camera format --- ignored.\n", filename); - psFitsClose(file->fits); - return false; - } - file->view = pmFPAAddSource(file->fpa, file->phu, format); - return true; -} - Index: unk/ppImage/src/ppFile.h =================================================================== --- /trunk/ppImage/src/ppFile.h (revision 6816) +++ (revision ) @@ -1,22 +1,0 @@ -#ifndef PP_FILE_H -#define PP_FILE_H - -#include "pslib.h" -#include "psmodules.h" - -// A file to process -typedef struct { - char *filename; // File name - psFits *fits; // The FITS file handle - psMetadata *phu; // The FITS header - pmFPA *fpa; // The FPA, with pixels and extensions - pmFPAview *view; // The view -} ppFile; - - -// Allocator -ppFile *ppFileAlloc(void); - -bool ppFileOpen(ppFile *file, const char *name, pmConfig *config, psMetadata *camera, psMetadata *format); - -#endif Index: /trunk/ppImage/src/ppImage.c =================================================================== --- /trunk/ppImage/src/ppImage.c (revision 6816) +++ /trunk/ppImage/src/ppImage.c (revision 6817) @@ -1,40 +1,24 @@ -#include -#include "pslib.h" -#include "psmodules.h" #include "ppImage.h" -#include "ppMem.h" -int main(int argc, char **argv) -{ -// psErrorRegister(NULL, -1); - - ppImageData *data = ppImageDataAlloc(); - ppImageOptions *options = ppImageOptionsAlloc(); +int main(int argc, char **argv) { psTimerStart(TIMERNAME); // Parse the configuration and arguments + // Open the input image(s) + // Determine camera, format from header if not already defined + // Construct camera in preparation for reading pmConfig *config = ppImageConfig(argc, argv); - // Open the input image, output image, output mask - // Get camera configuration from header if not already defined - // Construct camera in preparation for reading - ppImageParseCamera(data, config); + // Set various tasks (define optional operations) + ppImageOptions *options = ppImageOptionsParse(config); - // Set various tasks (define optional operations) - ppImageOptionsParse(data, options, config); - -#if 0 - // XXX I think we do this in ppImageParseCamera - - // open detrend images, load headers, optionally load pixels - ppImageParseDetrend(data, options, config); -#endif + // define the active I/O files + ppImageParseDetrend(options, config); // Image Arithmetic Loop - ppImageLoop(data, options, config); + ppImageLoop(options, config); // Cleaning up - psFree(data); psFree(options); psFree(config); @@ -43,6 +27,6 @@ pmConceptsDone(); pmConfigDone(); - ppMemCheck(); - + // ppMemCheck(); + fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "ppImage"); return EXIT_SUCCESS; } Index: /trunk/ppImage/src/ppImage.h =================================================================== --- /trunk/ppImage/src/ppImage.h (revision 6816) +++ /trunk/ppImage/src/ppImage.h (revision 6817) @@ -2,9 +2,9 @@ #define PP_IMAGE_H +#include #include "pslib.h" #include "psmodules.h" - -#include "ppImageData.h" #include "ppImageOptions.h" +#include "ppMem.h" #define RECIPE_NAME "PHASE2" // Name of the recipe to use @@ -15,30 +15,18 @@ // Determine what type of camera, and initialise -bool ppImageParseCamera(ppImageData *data, // The data to be processed - pmConfig *config // Configuration - ); - +bool ppImageParseDetrend(ppImageOptions *options, pmConfig *config); // Loop over the input -bool ppImageLoop(ppImageData *data, ppImageOptions *options, pmConfig *config); +bool ppImageLoop(ppImageOptions *options, pmConfig *config); -// Load the pixels for the given file -bool ppImageLoadPixels(ppFile *input, // File for which to load the pixels - psDB *db, // Database handle (for reading concepts) - int nChip, // Chip number to load - int nCell // Cell number to load - ); +// perform the detrend analysis on the current readout +bool ppImageDetrendReadout (ppImageOptions *options, pmConfig *config, pmFPAview *view); -bool ppImageParseDetrend(ppImageData *data, ppImageOptions *options, pmConfig *config); +bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options); +bool ppImageDetrendMask(pmReadout *input, pmReadout *mask); -bool ppReadoutWeights(pmReadout *readout); - -#if 0 -// These functions are not implemented, or not needed -pmReadout* ppDetrendPedestal(pmReadout *pedestal, pmCell *input, pmReadout *bias, pmReadout *dark, float darkTime, ppImageOptions *options); -bool ppImageOutput(ppImageData *data, pmConfig *config); -bool ppImagePhot(ppImageData *data, ppImageOptions *options, pmConfig *config); -bool ppFileOpen(ppFile *fpa, char *name, bool doThis); -#endif +bool ppImageDetrendNonLinear(pmReadout *input, ppImageOptions *options); +bool ppImageDetrendNonLinearLookup(pmReadout *input, psMetadataItem *dataItem); +bool ppImageDetrendNonLinearPolynomial(pmReadout *input, psMetadataItem *dataItem); #endif // Pau. Index: /trunk/ppImage/src/ppImageConfig.c =================================================================== --- /trunk/ppImage/src/ppImageConfig.c (revision 6816) +++ /trunk/ppImage/src/ppImageConfig.c (revision 6817) @@ -1,9 +1,17 @@ -#include -#include "pslib.h" -#include "psmodules.h" #include "ppImage.h" + +static void usage (void) { + fprintf (stderr, "USAGE: ppImage [-file INPUT.fits] [-list INPUT.txt] OUTPUT\n"); + exit (2); +} pmConfig *ppImageConfig(int argc, char **argv) { + + bool status; + + if (argc == 1) usage (); + + // load the site-wide configuration information pmConfig *config = pmConfigRead(&argc, argv); if (! config) { @@ -12,25 +20,38 @@ } - // Parse other command-line arguments + // Parse other command-line arguments, save for future use config->arguments = psMetadataAlloc(); // The arguments, with default values - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-key", 0, "exposure ID", ""); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-bias", 0, "Name of the bias image", ""); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-dark", 0, "Name of the dark image", ""); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-flat", 0, "Name of the flat-field image", ""); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-mask", 0, "Name of the mask image", ""); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-fringe", 0, "Name of the fringe image", ""); - psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-chip", 0, "Chip number to process (if positive)", -1); - if (! psArgumentParse(config->arguments, &argc, argv) || argc != 3) { - printf("\nPan-STARRS Phase 2 processing\n\n"); - printf("Usage: %s INPUT.fits OUTPUT.fits\n\n", argv[0]); - psArgumentHelp(config->arguments); - psFree(config->arguments); - exit(EXIT_FAILURE); + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "BIAS", "-bias", "-biaslist"); + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "DARK", "-dark", "-darklist"); + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "FLAT", "-flat", "-flatlist"); + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist"); + + // other command-line options + // XXX not sure how these are to be used at the moment + // psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-key", 0, "exposure ID", ""); + // psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-chip", 0, "Chip number to process (if positive)", -1); + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { usage ();} + + if (! psArgumentParse(config->arguments, &argc, argv) || argc != 2) { + usage (); } // Add the input and output images (which remain on the command-line) to the arguments list - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-input", 0, "Name of the input image", argv[1]); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-output", 0, "Name of the output image", argv[2]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]); + + // the input image(s) are required arguments + // the first one defines the camera + status = false; + pmFPAfileFromArgs (&status, config, "PPIMAGE.INPUT", "INPUT"); + if (!status) { + psAbort (__func__, "missing INPUT entry"); + } // Define database handle, if used Index: unk/ppImage/src/ppImageData.c =================================================================== --- /trunk/ppImage/src/ppImageData.c (revision 6816) +++ (revision ) @@ -1,32 +1,0 @@ -#include -#include "pslib.h" -#include "ppImageData.h" - -static void imageDataFree(ppImageData *data) -{ - psFree(data->input); - psFree(data->mask); - psFree(data->bias); - psFree(data->dark); - psFree(data->flat); - psFree(data->fringe); - if (data->output) { - psFitsClose(data->output); - } -} - -ppImageData *ppImageDataAlloc(void) -{ - ppImageData *data = psAlloc(sizeof(ppImageData)); - psMemSetDeallocator(data, (psFreeFunc)imageDataFree); - - data->input = NULL; - data->mask = NULL; - data->bias = NULL; - data->dark = NULL; - data->flat = NULL; - data->fringe = NULL; - data->output = NULL; - - return data; -} Index: unk/ppImage/src/ppImageData.h =================================================================== --- /trunk/ppImage/src/ppImageData.h (revision 6816) +++ (revision ) @@ -1,19 +1,0 @@ -#ifndef PP_IMAGE_DATA_H -#define PP_IMAGE_DATA_H - -#include "ppFile.h" - -// The data to be processed -typedef struct { - ppFile *input; // The input, to be operated upon - ppFile *mask; // The bad pixel mask image - ppFile *bias; // The bias correction image - ppFile *dark; // The dark correction image - ppFile *flat; // The flat-field correction image - ppFile *fringe; // The fringe correction image - psFits *output; // The output file -} ppImageData; - -ppImageData *ppImageDataAlloc(void); - -#endif Index: /trunk/ppImage/src/ppImageDetrendBias.c =================================================================== --- /trunk/ppImage/src/ppImageDetrendBias.c (revision 6816) +++ /trunk/ppImage/src/ppImageDetrendBias.c (revision 6817) @@ -1,7 +1,3 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImageOptions.h" -#include "ppImageDetrend.h" +#include "ppImage.h" bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options) Index: /trunk/ppImage/src/ppImageDetrendCell.c =================================================================== --- /trunk/ppImage/src/ppImageDetrendCell.c (revision 6816) +++ /trunk/ppImage/src/ppImageDetrendCell.c (revision 6817) @@ -1,63 +1,62 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImageDetrend.h" -#include "ppImageOptions.h" +#include "ppImage.h" -// mask, bias, dark, flat are defined per Cell +bool ppImageDetrendReadout (ppImageOptions *options, pmConfig *config, pmFPAview *view) { -pmReadout *ppImageDetrendSelectFirst(pmCell *cell, char *name, bool doThis) -{ - if (!doThis) { - return NULL; + bool status; + pmFPAfile *file; + + // construct a view for the detrend images (which have only one readout) + pmFPAview *detview = pmFPAviewAlloc (0); + *detview = *view; + detview->readout = 0; + + // find the currently selected readout + file = psMetadataLookupPtr (&status, config->files, "PPIMAGE.INPUT"); + pmReadout *input = pmFPAviewThisReadout (view, file->fpa); + + pmReadoutSetWeights (input); + + // Mask bad pixels + if (options->doMask) { + file = psMetadataLookupPtr (&status, config->files, "PPIMAGE.MASK"); + pmReadout *mask = pmFPAviewThisReadout (detview, file->fpa); + ppImageDetrendMask(input, mask); } - if (cell->readouts->n > 1) { - psLogMsg("phase2", PS_LOG_WARN, "%s contains multiple readouts: only the first will be used.", name); +# if 0 + // Non-linearity correction + if (options->doNonLin) { + ppImageDetrendNonLinear(detrend->input, input, options); + } +# endif + + // set up the readouts for dark and bias + pmReadout *dark = NULL; + pmReadout *bias = NULL; + if (options->doBias) { + file = psMetadataLookupPtr (&status, config->files, "PPIMAGE.BIAS"); + bias = pmFPAviewThisReadout (detview, file->fpa); + } + if (options->doDark) { + file = psMetadataLookupPtr (&status, config->files, "PPIMAGE.DARK"); + dark = pmFPAviewThisReadout (detview, file->fpa); } - pmReadout *readout = cell->readouts->data[0]; // Readout of interest in this cell + // Bias, dark and overscan subtraction are all merged. + ppImageDetrendBias(input, bias, dark, options); - return readout; + // Flat-field correction (no options used?) + if (options->doFlat) { + file = psMetadataLookupPtr (&status, config->files, "PPIMAGE.FLAT"); + pmReadout *flat = pmFPAviewThisReadout (detview, file->fpa); + pmFlatField(input, flat); + } + + psFree (detview); + return true; } -bool ppImageDetrendCell(ppImageDetrend *detrend, ppImageOptions *options, pmConfig *config) -{ -#if 0 - pmCellSetWeights(detrend->input); -#endif + // XXX in psphot, the recipe options are parsed at this depth + // XXX consider if we move ppImageOptions in here + // psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, "PSPHOT"); - pmReadout *mask = ppImageDetrendSelectFirst(detrend->mask, "mask", options->doMask); - pmReadout *bias = ppImageDetrendSelectFirst(detrend->bias, "bias", options->doBias); - pmReadout *dark = ppImageDetrendSelectFirst(detrend->dark, "dark", options->doDark); - pmReadout *flat = ppImageDetrendSelectFirst(detrend->flat, "flat", options->doFlat); - -#if 0 - printf("Flat type: %x\n", flat->image->type.type); - printf("Flat value: %f\n", flat->image->data.F32[flat->image->numRows/2][flat->image->numCols/2]); -#endif - - for (int k = 0; k < detrend->input->readouts->n; k++) { - - pmReadout *input = detrend->input->readouts->data[k]; // Readout of interest in input image - - // Mask bad pixels - if (options->doMask) { - ppImageDetrendMask(detrend->input, input, mask); - } - - // Non-linearity correction - if (options->doNonLin) { - ppImageDetrendNonLinear(detrend->input, input, options); - } - - // Bias, dark and overscan subtraction are all merged. - ppImageDetrendBias(input, bias, dark, options); - - // Flat-field correction (no options used?) - if (options->doFlat) { - pmFlatField(input, flat); - } - } - return true; -} Index: /trunk/ppImage/src/ppImageDetrendMask.c =================================================================== --- /trunk/ppImage/src/ppImageDetrendMask.c (revision 6816) +++ /trunk/ppImage/src/ppImageDetrendMask.c (revision 6817) @@ -1,5 +1,3 @@ -#include -#include "psmodules.h" -#include "ppImageDetrend.h" +#include "ppImage.h" # define MASK_MODE (PM_MASK_TRAP | PM_MASK_BADCOL | PM_MASK_SAT) @@ -7,13 +5,29 @@ // XXX pass 'concepts' not 'cell' to this function? -bool ppImageDetrendMask(pmCell *cell, pmReadout *input, pmReadout *mask) { +bool ppImageDetrendMask(pmReadout *input, pmReadout *mask) { - float saturation = psMetadataLookupF32(NULL, cell->concepts, "CELL.SATURATION"); + psImage *image = NULL; + + float saturation = psMetadataLookupF32(NULL, input->parent->concepts, "CELL.SATURATION"); // Need to change this later to grow the mask by the size of the convolution kernel // XXX does this function respect the mask / image cell region information? // XXX should probably take pmRegion *mask - pmMaskBadPixels(input, mask->image, MASK_MODE, saturation, PM_MASK_TRAP, 0); + + if (mask->image->type.type != PS_TYPE_MASK) { + image = psImageCopy (NULL, mask->image, PS_TYPE_MASK); + } else { + image = psMemIncrRefCounter (mask->image); + } + + pmMaskBadPixels(input, image, MASK_MODE, saturation, PM_MASK_TRAP, 0); + psFree (image); return true; } + + +// XXX the translation here should not be needed +// the mask should (but is not) be read in using the mask read functions +// by identifying the image type as a mask (in pmFPAfile). +// currently, pmFPAfileReadImage converts the image to F32 Index: /trunk/ppImage/src/ppImageDetrendNonLinear.c =================================================================== --- /trunk/ppImage/src/ppImageDetrendNonLinear.c (revision 6816) +++ /trunk/ppImage/src/ppImageDetrendNonLinear.c (revision 6817) @@ -1,6 +1,3 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImageDetrend.h" +#include "ppImage.h" bool ppImageDetrendNonLinearPolynomial(pmReadout *input, psMetadataItem *dataItem) { @@ -43,7 +40,8 @@ } -bool ppImageDetrendNonLinear(pmCell *cell, pmReadout *input, ppImageOptions *options) { +bool ppImageDetrendNonLinear(pmReadout *input, ppImageOptions *options) { psMetadataItem *concept; + pmCell *cell = input->parent; switch (options->nonLinearType) { Index: unk/ppImage/src/ppImageLoadPixels.c =================================================================== --- /trunk/ppImage/src/ppImageLoadPixels.c (revision 6816) +++ (revision ) @@ -1,69 +1,0 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImage.h" - -bool ppImageLoadPixels(ppFile *input, psDB *db, int chipNum, int cellNum) -{ - // an input chip is valid for processing if: - // (((chipNum == i) || (chipNum == -1)) && chip.process) - - // If we have not opened the file, skip it - if (input->fits == NULL) { - return false; - } - - psTrace(__func__, 1, "Loading %d,%d for %s\n", chipNum, cellNum, input->filename); - - pmFPA *fpa = input->fpa; // The entire focal plane array - psArray *chips = fpa->chips; // The array of chips - if (chipNum >= 0 && chipNum < chips->n) { - loadCells(chips->data[chipNum]); - if (! chip->exists && chip->process) { - loadCells(chip, cellNum); - } - } else { - for (int i = 0; i < chips->n; i++) { - pmChip *chip = chips->data[i]; - } - - // set input:valid flags according to process and chipNum/cellNum - for (int i = 0; i < input->fpa->chips->n; i++) { - pmChip *chip = input->fpa->chips->data[i]; // Chip in input image - chip->process = (! chip->exists && ((chipNum == i) || (chipNum == -1))); - - for (int j = 0; j < chip->cells->n; j++) { - pmCell *cell = chip->cells->data[j]; // Cell in input image - cell->process &= chip->process; - cell->process = (! cell->exists && ((cellNum == i) || (cellNum == -1))); - } - } - - // Read in the input pixels - if (! pmFPARead(input->fpa, input->fits, input->phu, db)) { - psErrorStackPrint(stderr, "Unable to populate camera from input FITS file\n"); - exit(EXIT_FAILURE); - } - -#if 0 - // XXX PAP: Not sure this is required any more. - - // reset input:valid flags to true - for (int i = 0; i < input->fpa->chips->n; i++) { - - pmChip *chip = input->fpa->chips->data[i]; - chip->process = true; - for (int j = 0; j < chip->cells->n; j++) { - - pmCell *cell = chip->cells->data[j]; - cell->process = true; - } - } -#endif - - return true; -} - -// XXX this is not very efficient with fseeks : each pmFPARead is randomly accessing the file -// XXX does this handle multi-file data? -// XXX this does NOT preserve the state of the input valid flags Index: /trunk/ppImage/src/ppImageLoop.c =================================================================== --- /trunk/ppImage/src/ppImageLoop.c (revision 6816) +++ /trunk/ppImage/src/ppImageLoop.c (revision 6817) @@ -1,141 +1,68 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImageDetrend.h" -#include "ppImage.h" -#include "ppMem.h" +# include "ppImage.h" -#define MEM_LEAKS 1 +bool ppImageLoop (ppImageOptions *options, pmConfig *config) { -// Read the entire FPA -void readFPA(ppFile *file, // File to read - pmConfig *config // Configuration, containing the DB handle - ) -{ - if (file->fpa && file->fits) { - pmFPARead(file->fpa, file->fits, config->database); + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PPIMAGE.INPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find input data!\n"); + exit(EXIT_FAILURE); } + + pmFPAview *view = pmFPAviewAlloc (0); + + // files associated with the science image + pmFPAfileIOChecks (config->files, view, PM_FPA_BEFORE); + + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { continue; } + pmFPAfileIOChecks (config->files, view, PM_FPA_BEFORE); + + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { continue; } + pmFPAfileIOChecks (config->files, view, PM_FPA_BEFORE); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + pmFPAfileIOChecks (config->files, view, PM_FPA_BEFORE); + if (! readout->data_exists) { continue; } + + // perform the detrend analysis + ppImageDetrendReadout (options, config, view); + + pmFPAfileIOChecks (config->files, view, PM_FPA_AFTER); + } + pmFPAfileIOChecks (config->files, view, PM_FPA_AFTER); + } + + // ppImageChipMosaic (config, view); + // ppImagePSPhot (config, view); + // ppImagePSAstro (config, view); + + pmFPAfileIOChecks (config->files, view, PM_FPA_AFTER); + + // XXX paul added these frees: make sure the same is happening in the pmFileIOChecks + // Now I can blow away the mosaic so I can then read more. + // psFree(inputChip); + // psFree(biasChip); + // psFree(darkChip); + // psFree(maskChip); + // psFree(flatChip); + // ppMemUsed(); + } + + // ppImageFPAMosaic (config, view); + // ppImageJpegFPA (config, view); + + pmFPAfileIOChecks (config->files, view, PM_FPA_AFTER); + + psFree (view); + return true; } -// Read a specified chip only -void readChip(ppFile *file, // File to read - pmConfig *config, // Configuration - int chipNum // Chip number to read - ) -{ - pmFPA *fpa = file->fpa; // The FPA - if (fpa && file->fits) { - pmChip *chip = fpa->chips->data[chipNum]; // The chip - pmChipRead(chip, file->fits, config->database); - } -} - - -void readCell(ppFile *file, pmConfig *config, int chipNum, int cellNum) -{ - pmFPA *fpa = file->fpa; // The FPA - if (fpa && file->fits) { - pmChip *chip = fpa->chips->data[chipNum]; // The chip - if (chip) { - pmCell *cell = chip->cells->data[cellNum]; // The cell - pmCellRead(cell, file->fits, config->database); - } - } -} - - -bool ppImageLoop(ppImageData *data, ppImageOptions *options, pmConfig *config) -{ - ppImageDetrend detrend; - - int processChip = psMetadataLookupS32(NULL, config->arguments, "-chip"); // Chip number to process or -1 - - // Load at FPA level if requested - if (processChip < 0 && options->imageLoadDepth == PP_LOAD_FPA) { - psTrace(__func__, 1, "Loading pixels for FPA...\n"); - readFPA(data->input, config); - readFPA(data->bias, config); - readFPA(data->dark, config); - readFPA(data->mask, config); - readFPA(data->flat, config); - } - - psArray *chips = data->input->fpa->chips; // Component chips - for (int i = 0; i < chips->n; i++) { - - if (processChip >= 0 && i != processChip) { - continue; - } - if (options->imageLoadDepth == PP_LOAD_CHIP) { - psTrace(__func__, 1, "Loading pixels for chip %d...\n", i); - readChip(data->input, config, i); - readChip(data->bias, config, i); - readChip(data->dark, config, i); - readChip(data->mask, config, i); - readChip(data->flat, config, i); - } - - pmChip *inputChip = chips->data[i]; - pmChip *biasChip = data->bias->fpa->chips->data[i]; - pmChip *darkChip = data->dark->fpa->chips->data[i]; - pmChip *maskChip = data->mask->fpa->chips->data[i]; - pmChip *flatChip = data->flat->fpa->chips->data[i]; - - psArray *cells = inputChip->cells; - for (int j = 0; j < cells->n; j++) { - detrend.input = inputChip->cells->data[j]; // Cell of interest in input image - detrend.bias = biasChip->cells->data[j]; // Cell of interest in bias image - detrend.dark = darkChip->cells->data[j]; // Cell of interest in dark imag - detrend.mask = maskChip->cells->data[j]; // Cell of interest in mask image - detrend.flat = flatChip->cells->data[j]; // Cell of interest in flat image - - if (! detrend.input->process) { continue; } - - if (options->imageLoadDepth == PP_LOAD_CELL) { - psTrace(__func__, 1, "Loading pixels for chip %d, cell %d...\n", i, j); - readCell(data->input, config, i, j); - readCell(data->bias, config, i, j); - readCell(data->dark, config, i, j); - readCell(data->mask, config, i, j); - readCell(data->flat, config, i, j); - } - - ppImageDetrendCell(&detrend, options, config); - - // Need to free detrend cells here so we have enough memory to do other stuff - psFree(detrend.bias); - psFree(detrend.dark); - psFree(detrend.mask); - psFree(detrend.flat); - } - -#if 0 - int numMosaicked = pmChipMosaic(inputChip, 1, 1); // Number of cells mosaicked together - psLogMsg(__func__, PS_LOG_INFO, "%d cells mosaicked.\n", numMosaicked); - - // XXX A kludge to get the write to behave w.r.t. the concepts --- we've changed the camera format, so - // the concepts don't know what on earth to do. - const psMetadata *camera = data->input->fpa->camera; - data->input->fpa->camera = NULL; -#endif - - // XXX Photometry goes here! - - pmChipWrite(inputChip, data->output, config->database); - -#if 0 - data->input->fpa->camera = camera; -#endif - - // Now I can blow away the mosaic so I can then read more. - psFree(inputChip); - psFree(biasChip); - psFree(darkChip); - psFree(maskChip); - psFree(flatChip); - - ppMemUsed(); - } - - return true; -} Index: /trunk/ppImage/src/ppImageOptions.c =================================================================== --- /trunk/ppImage/src/ppImageOptions.c (revision 6816) +++ /trunk/ppImage/src/ppImageOptions.c (revision 6817) @@ -1,7 +1,3 @@ -#include -#include "pslib.h" #include "ppImage.h" -#include "ppImageData.h" -#include "ppImageOptions.h" static void imageOptionsFree(ppImageOptions *options) @@ -25,34 +21,33 @@ } - -// XXX EAM : this needs signficant work to choose the detrend images based on the detrend database - -bool ppImageOptionsParse(ppImageData *data, ppImageOptions *options, pmConfig *config) +ppImageOptions *ppImageOptionsParse(pmConfig *config) { - // Flags for various activities - options->doMask = false; // Mask bad pixels - options->doNonLin = false; // Non-linearity correction - options->doBias = false; // Bias subtraction - options->doDark = false; // Dark subtraction + ppImageOptions *options = ppImageOptionsAlloc (); + + // default flags for various activities + options->doMask = false; // Mask bad pixels + options->doNonLin = false; // Non-linearity correction + options->doBias = false; // Bias subtraction + options->doDark = false; // Dark subtraction options->doOverscan = false; // Overscan subtraction - options->doFlat = false; // Flat-field normalisation - options->doFringe = false; // Fringe subtraction - options->doSource = false; // Source identification and photometry - options->doAstrom = false; // Astrometry - // Overscan options - options->overscan = NULL; // Overscan options - bool overscanSingle = false; // A single value for entire overscan? - pmFit overscanFit = PM_FIT_NONE; // Fit type for overscan - int overscanOrder = 0; // Order for overscan fit + options->doFlat = false; // Flat-field normalisation + options->doFringe = false; // Fringe subtraction + options->doSource = false; // Source identification and photometry + options->doAstrom = false; // Astrometry + + // Overscan defaults + options->overscan = NULL; // Overscan options + bool overscanSingle = false; // A single value for entire overscan? + pmFit overscanFit = PM_FIT_NONE; // Fit type for overscan + int overscanOrder = 0; // Order for overscan fit psStats *overscanStats = NULL; // Statistics for overscan - // Non-linearity options - options->nonLinearType = 0; // Type of non-linearity data (vector, string or metadata) - options->nonLinearData = NULL; // The non-linearity data + + // Non-linearity default options + options->nonLinearType = 0; // Type of non-linearity data (vector, string or metadata) + options->nonLinearData = NULL; // The non-linearity data options->nonLinearSource = NULL; // If the non-linearity data is a menu, this provides the key - // Various others - options->imageLoadDepth = PP_LOAD_NONE; // No load depth specified yet - + // select the recipe for this analysis bool mdStatus = false; // Result of MD lookup psMetadata *recipe = psMetadataLookupMD(&mdStatus, config->recipes, RECIPE_NAME); @@ -60,31 +55,4 @@ psLogMsg("ppImage", PS_LOG_ERROR, "Can't find recipe %s in the RECIPES.\n", RECIPE_NAME); exit(EXIT_FAILURE); - } - - const char *depth = psMetadataLookupStr(&mdStatus, recipe, "LOAD.DEPTH"); - if (! mdStatus || ! depth || strlen(depth) == 0) { - psLogMsg("ppImage", PS_LOG_ERROR, "LOAD.DEPTH not specified in recipe."); - exit(EXIT_FAILURE); - } - if (!strcasecmp(depth, "FPA")) { - options->imageLoadDepth = PP_LOAD_FPA; - } else if (!strcasecmp(depth, "CHIP")) { - options->imageLoadDepth = PP_LOAD_CHIP; - } else if (!strcasecmp(depth, "CELL")) { - options->imageLoadDepth = PP_LOAD_CELL; - } else { - psLogMsg(__func__, PS_LOG_ERROR, "LOAD.DEPTH in recipe %s is not FPA, CHIP or CELL.", RECIPE_NAME); - exit(EXIT_FAILURE); - } - - // Mask recipe options - if (psMetadataLookupBool(NULL, recipe, "MASK")) { - data->mask->filename = psMetadataLookupStr(NULL, config->arguments, "-mask"); - if (data->mask->filename && strlen(data->mask->filename) > 0) { - options->doMask = true; - } else { - psLogMsg(__func__, PS_LOG_WARN, "Masking is desired, but no mask was supplied" - " --- no masking will be performed.\n"); - } } @@ -124,26 +92,4 @@ "NONLIN.DATA is of invalid type in recipe %s.", RECIPE_NAME); exit(EXIT_FAILURE); - } - } - - // Bias recipe options - if (psMetadataLookupBool(NULL, recipe, "BIAS")) { - data->bias->filename = psMetadataLookupStr(NULL, config->arguments, "-bias"); - if (data->bias->filename && strlen(data->bias->filename) > 0) { - options->doBias = true; - } else { - psLogMsg(__func__, PS_LOG_WARN, "Bias subtraction is desired in recipe %s, but no bias was " - "supplied --- no bias subtraction will be performed.\n", RECIPE_NAME); - } - } - - // Dark recipe options - if (psMetadataLookupBool(NULL, recipe, "DARK")) { - data->dark->filename = psMetadataLookupStr(NULL, config->arguments, "-dark"); - if (data->dark->filename && strlen(data->dark->filename) > 0) { - options->doDark = true; - } else { - psLogMsg(__func__, PS_LOG_WARN, "Dark subtraction is desired in recipe %s, but no dark was " - "supplied --- no dark subtraction will be performed.\n", RECIPE_NAME); } } @@ -190,23 +136,24 @@ } - // flat-field - recipe options - if (psMetadataLookupBool(NULL, recipe, "FLAT")) { - data->flat->filename = psMetadataLookupStr(NULL, config->arguments, "-flat"); - if (strlen(data->flat->filename) > 0) { - options->doFlat = true; - } else { - psLogMsg(__func__, PS_LOG_WARN, "Flat-fielding is desired in recipe %s, but no flat was " - "supplied --- no flat-fielding will be performed.\n", RECIPE_NAME); - } + // Mask recipe options + if (psMetadataLookupBool(NULL, recipe, "MASK")) { + options->doMask = true; } - // XXX need to add the following: + // Bias recipe options + if (psMetadataLookupBool(NULL, recipe, "BIAS")) { + options->doBias = true; + } - // fringe - recipe options + // Dark recipe options + if (psMetadataLookupBool(NULL, recipe, "DARK")) { + options->doDark = true; + } - // photom - recipe options + // Flat recipe options + if (psMetadataLookupBool(NULL, recipe, "FLAT")) { + options->doFlat = true; + } - // astrom - recipe options - - return true; + return options; } Index: /trunk/ppImage/src/ppImageOptions.h =================================================================== --- /trunk/ppImage/src/ppImageOptions.h (revision 6816) +++ /trunk/ppImage/src/ppImageOptions.h (revision 6817) @@ -1,17 +1,4 @@ #ifndef PP_IMAGE_OPTIONS_H #define PP_IMAGE_OPTIONS_H - -#include "pslib.h" -#include "psmodules.h" -#include "ppImageData.h" -#include "ppConfig.h" - -// How much of the FPA to load at a time -typedef enum { - PP_LOAD_NONE, // Don't load anything - PP_LOAD_FPA, // Load the entire FPA at once - PP_LOAD_CHIP, // Load by chip - PP_LOAD_CELL, // Load by cell -} ppImageLoadDepth; // Options for ppImage processing @@ -30,5 +17,4 @@ psMetadataItem *nonLinearData; void *nonLinearSource; - ppImageLoadDepth imageLoadDepth; // How much of the FPA to load at once } ppImageOptions; @@ -36,8 +22,5 @@ // Determine the processing options -bool ppImageOptionsParse(ppImageData *data, // The data to be processed - ppImageOptions *options, // Processing options - pmConfig *config // Configuration - ); +ppImageOptions *ppImageOptionsParse(pmConfig *config); #endif Index: unk/ppImage/src/ppImageOutput.c =================================================================== --- /trunk/ppImage/src/ppImageOutput.c (revision 6816) +++ (revision ) @@ -1,22 +1,0 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImage.h" - -bool ppImageOutput(ppData *data, pmConfig *config) -{ - const char *outname = psMetadataLookupStr(NULL, config->arguments, "-output"); // Name of output file - psFits *outFile = psFitsOpen(outname, "a"); // File for writing - - // Write the output - pmFPAWrite(outFile, data->input->fpa, config->database); -#if 0 - pmFPAWriteMask(input, outputFile); - pmFPAWriteWeight(input, outputFile); -#endif - - psFitsClose(outFile); - psLogMsg("ppImage", PS_LOG_INFO, "Output completed after %f sec.\n", psTimerMark("ppImage")); - - return true; -} Index: unk/ppImage/src/ppImageParseCamera.c =================================================================== --- /trunk/ppImage/src/ppImageParseCamera.c (revision 6816) +++ (revision ) @@ -1,66 +1,0 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImage.h" - -bool ppImageParseCamera(ppImageData *data, pmConfig *config) -{ - // Initialise the containers where the files will go - data->input = ppFileAlloc(); - data->mask = ppFileAlloc(); - data->bias = ppFileAlloc(); - data->dark = ppFileAlloc(); - data->flat = ppFileAlloc(); - data->fringe = ppFileAlloc(); - - data->input->filename = psMemIncrRefCounter(psMetadataLookupStr(NULL, config->arguments, "-input")); - - // Open the input image - psLogMsg("ppImage", PS_LOG_INFO, "Opening input image: %s\n", data->input->filename); - data->input->fits = psFitsOpen(data->input->filename, "r"); // File handle for FITS file - if (! data->input->fits) { - // We can't open the input - psErrorStackPrint(stderr, "Can't open input image: %s\n", data->input->filename); - exit(EXIT_FAILURE); - } - data->input->phu = psFitsReadHeader(NULL, data->input->fits); // FITS header - - // Get camera configuration from header if not already defined - psMetadata *cameraFormat = NULL; // Camera format description - if (! config->camera) { - cameraFormat = pmConfigCameraFormatFromHeader(config, data->input->phu); - if (! config->camera) { - // There's no point in continuing if we can't recognise what we've got - psErrorStackPrint(stderr, "Can't find camera configuration!\n"); - exit(EXIT_FAILURE); - } - } else if (! pmConfigValidateCameraFormat(config->camera, data->input->phu)) { - // What we've got doesn't match what we've been told - psError(PS_ERR_IO, true, "%s does not seem to be from the specified camera.\n", - data->input->filename); - exit(EXIT_FAILURE); - } - - // Construct cameras in preparation for reading - data->input->fpa = pmFPAConstruct(config->camera); - data->input->view = pmFPAAddSource(data->input->fpa, data->input->phu, cameraFormat); - - ppFileOpen(data->mask, "-mask", config, config->camera, cameraFormat); - ppFileOpen(data->bias, "-bias", config, config->camera, cameraFormat); - ppFileOpen(data->dark, "-dark", config, config->camera, cameraFormat); - ppFileOpen(data->flat, "-flat", config, config->camera, cameraFormat); - ppFileOpen(data->fringe, "-fringe", config, config->camera, cameraFormat); - - psFree(cameraFormat); - - // Open output file - const char *outname = psMetadataLookupStr(NULL, config->arguments, "-output"); // Name of output file - data->output = psFitsOpen(outname, "w"); // File for writing - if (! data->output) { - psError(PS_ERR_IO, false, "Unable to open output file %s.\n", data->output); - exit(EXIT_FAILURE); - } - - return true; -} - Index: /trunk/ppImage/src/ppImageParseDetrend.c =================================================================== --- /trunk/ppImage/src/ppImageParseDetrend.c (revision 6816) +++ /trunk/ppImage/src/ppImageParseDetrend.c (revision 6817) @@ -1,23 +1,62 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppFile.h" -#include "ppImageData.h" -#include "ppImageOptions.h" #include "ppImage.h" -// open all needed detrend files -// read in primary headers -// validate camera for each detrend image +// identify all needed detrend and other I/O files +// create the pmFPAfiles for the active I/O files -// XXX : keep the primary headers in data? -// XXX : add fringe frame +bool ppImageParseDetrend(ppImageOptions *options, pmConfig *config) +{ -bool ppImageParseDetrend(ppImageData *data, ppImageOptions *options, pmConfig *config) -{ - ppFileOpen(data->mask, "mask", options->doMask); - ppFileOpen(data->bias, "bias", options->doBias); - ppFileOpen(data->dark, "dark", options->doDark); - ppFileOpen(data->flat, "flat", options->doFlat); + bool status; + + // some of the output files are associated with the input fpa + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PPIMAGE.INPUT"); + + // the following are defined from the argument list, if given, + // otherwise they revert to the config information + + if (options->doBias) { + bool status = false; + pmFPAfileFromArgs (&status, config, "PPIMAGE.BIAS", "BIAS"); + pmFPAfileFromConf (&status, config, "PPIMAGE.BIAS", input->fpa); + if (!status) psAbort ("ppImageParseDetrend", "can't find a bias image source"); + } + + if (options->doDark) { + bool status = false; + pmFPAfileFromArgs (&status, config, "PPIMAGE.DARK", "DARK"); + pmFPAfileFromConf (&status, config, "PPIMAGE.DARK", input->fpa); + if (!status) psAbort ("ppImageParseDetrend", "can't find a dark image source"); + } + + if (options->doMask) { + bool status = false; + pmFPAfileFromArgs (&status, config, "PPIMAGE.MASK", "MASK"); + pmFPAfileFromConf (&status, config, "PPIMAGE.MASK", input->fpa); + if (!status) psAbort ("ppImageParseDetrend", "can't find a mask image source"); + } + + if (options->doFlat) { + bool status = false; + pmFPAfileFromArgs (&status, config, "PPIMAGE.FLAT", "FLAT"); + pmFPAfileFromConf (&status, config, "PPIMAGE.FLAT", input->fpa); + if (!status) psAbort ("ppImageParseDetrend", "can't find a flat image source"); + } + + // the following files are output targets + // XXX which of these are required? + pmFPAfileDefine (config->files, config->camera, input->fpa, "PPIMAGE.OUTPUT"); + + // XXX do these need to construct a new fpa? + // pmFPAfileDefine (config->files, config->camera, input->input, "PPIMAGE.BIN1"); + // pmFPAfileDefine (config->files, config->camera, input->input, "PPIMAGE.BIN2"); + // pmFPAfileDefine (config->files, config->camera, input->input, "PPIMAGE.JPG1"); + // pmFPAfileDefine (config->files, config->camera, input->input, "PPIMAGE.JPG2"); + + // ppImage is supplied with an output name, which may be used by multiple + // output files to construct the output names. supply this value to the + // files which are write types. + char *output = psMetadataLookupPtr(&status, config->arguments, "OUTPUT"); + pmFPAfileAddFileNames (config->files, "OUTPUT", output, PM_FPA_MODE_WRITE); + return true; } Index: unk/ppImage/src/ppImageWeights.c =================================================================== --- /trunk/ppImage/src/ppImageWeights.c (revision 6816) +++ (revision ) @@ -1,32 +1,0 @@ -#include -#include "pslib.h" -#include "psmodules.h" -#include "ppImage.h" - -bool ppReadoutWeights (pmReadout *readout) { - - psImage *image = readout->image; - readout->mask = psImageAlloc(image->numCols, image->numRows, PS_TYPE_U8); - readout->weight = psImageAlloc(image->numCols, image->numRows, PS_TYPE_F32); - - // XXX : we need to determine rdnoise and gain from the appropriate location - float rdnoise = 5.0; // in electrons - float gain = 2.0; // in e/ADU - - float rgain = 1.0 / gain; - float rnoise = PS_SQR (rdnoise / gain); - - psF32 **vI = readout->image->data.F32; - psU8 **vM = readout->mask->data.U8; - psF32 **vW = readout->weight->data.F32; - - for (int j = 0; j < image->numRows; j++) { - for (int i = 0; i < image->numCols; i++) { - vM[j][i] = 0; - vW[j][i] = PS_MAX (rgain * vI[j][i] + rnoise, 0.0); - } - } - return true; -} - -// XXX : I am defining the 'weight' to be the variance, not the stdev