#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <strings.h>            /* for strn?casecmp */
#include <pslib.h>

#include "pmErrorCodes.h"
#include "pmConfig.h"
#include "pmDetrendDB.h"

#include "pmHDU.h"
#include "pmFPA.h"
#include "pmFPALevel.h"
#include "pmFPAview.h"
#include "pmFPAfile.h"
#include "pmFPAfile.h"
#include "pmFPAConstruct.h"


// Get the file rule of interest
// Look up the name of the set of file rules to use, get that set from the site configuration, and return the
// appropriate rule from the set.
static psMetadata *getFileRule(const pmConfig *config, // Configuration
                               const psMetadata *camera, // Camera configuration of interest
                               const char *name // Name of rule to read
    )
{
    assert(config);
    assert(config->site);

    psMetadataItem *item = psMetadataLookup(camera, "FILERULES"); // Item with the file rule of interest
    if (!item) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find FILERULES in the camera configuration.");
        return NULL;
    }

    psMetadata *filerules = NULL;       // File rules from the site configuration
    switch (item->type) {
      case PS_DATA_METADATA:
        // It's what we're after
        filerules = item->data.md;
        break;
      case PS_DATA_STRING: {
          // It's the name of a file --- read the file, and store it for future use
          if (!pmConfigFileRead(&filerules, item->data.str, "filerules")) {
              psError(PM_ERR_CONFIG, false, "Trouble reading reading file rules from %s  --- "
                      "ignored.\n", item->data.str);
              psFree(filerules);
              return NULL;
          }

          // Muck around under the hood to replace the filename with the metadata; don't try this at home,
          // kids
          item->type = PS_DATA_METADATA;
          psFree(item->data.str);
          item->data.md = filerules;
          break;
      }
      default:
        psError(PS_ERR_BAD_PARAMETER_TYPE, true,
                "Unexpected type for %s (%x) in FILERULES in SITE configuration.",
                name, item->type);
        return NULL;
    }

    // select the name from the FILERULES
    // check for alias name (type == STR, name is aliased name)
    bool mdok;                          // Status of MD lookup
    const char *realname = psMetadataLookupStr(&mdok, filerules, name); // Name of file rule to look up
    if (!realname || strlen(realname) == 0) {
        realname = name;
    }

    return psMetadataLookupMetadata(NULL, filerules, realname);
}

// Parse an option from a metadata, returning the appropriate integer value
static int parseOptionInt(const psMetadata *md, // Metadata containing the option
                          const char *name, // Option name
                          const char *source, // Description of source, for warning messages
                          int defaultValue // Default value
                          )
{
    psMetadataItem *item = psMetadataLookup(md, name); // Item with the value of interest
    if (!item) {
        psWarning("Unable to find value for %s in %s --- set to %d.", name, source, defaultValue);
        return defaultValue;
    }
    int value = psMetadataItemParseS32(item); // Value of interst
    return value;
}


// define an input-type pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineInput(const pmConfig *config, pmFPA *fpa, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    bool status;
    char *type;

    const psMetadata *camera = (fpa ? fpa->camera : config->camera); // Camera configuration for this file
    psMetadata *data = getFileRule(config, camera, name); // File rule
    if (!data) {
        psError(PS_ERR_IO, true, "Can't find file rule %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc ();

    // save the name of this pmFPAfile
    file->name = psStringCopy (name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.RULE"));

    type = psMetadataLookupStr (&status, data, "FILE.TYPE");
    file->type = pmFPAfileTypeFromString(type);
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        return NULL;
    }

    file->mode = PM_FPA_MODE_READ;
    file->fileLevel = PM_FPA_LEVEL_NONE; // the fileLevel depends on the input data

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr (&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;

    if (fpa) {
        file->fpa = psMemIncrRefCounter(fpa);
        file->camera = psMemIncrRefCounter((psMetadata *)fpa->camera);
        file->cameraName = psMemIncrRefCounter(config->cameraName); // XXX Is this the correct thing to do?
    } else {
        file->camera = psMemIncrRefCounter(config->camera);
        file->cameraName = psMemIncrRefCounter(config->cameraName);
    }

    // XXX ppImage and similar require the added file to be unique
    // XXX ppFocus wants to override the selection with the new selection
    // XXX require programs like ppFocus to remove existing files by hand
    if (!psMetadataAddPtr (config->files, PS_LIST_TAIL, name,
                           PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "", file)) {
        psError (PS_ERR_IO, false, "could not add %s to config files", name);
        return NULL;
    }
    psFree (file);
    return (file);
}

// Define an output pmFPAfile
pmFPAfile *pmFPAfileDefineOutputForFormat(const pmConfig *config, // Configuration
                                          pmFPA *fpa, // Optional FPA to bind
                                          const char *name, // Name of file rule
                                          psString cameraName, // Name of camera configuration to use
                                          psString formatName // Name of camera format to use
    )
{
    bool status;

    // Use the camera we were told to, the camera of the provided FPA, or default to the default camera
    psMetadata *camera;                 // Camera configuration
    if (!cameraName || strlen(cameraName) == 0) {
        if (fpa && fpa->camera) {
            camera = (psMetadata*)fpa->camera; // Casting away const, so I can put it in the file
        } else {
            camera = config->camera;
            cameraName = config->cameraName;
        }
    } else {
        bool mdok;                      // Status of MD lookup
        psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS"); // Known cameras
        if (!mdok || !cameras) {
            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find CAMERAS in the site configuration.\n");
            return NULL;
        }
        camera = psMetadataLookupMetadata(&mdok, cameras, cameraName); // Camera configuration of interest
        if (!mdok || !camera) {
            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find automatically generated "
                    "camera configuration %s in site configuration.\n", cameraName);
            return NULL;
        }

        if (fpa && fpa->camera && fpa->camera != camera) {
            psAbort("Camera of bound FPA is not the requested camera --- there is an inconsistency!");
        }
    }

    psMetadata *data = getFileRule(config, camera, name); // File rule
    if (!data) {
        psError(PS_ERR_IO, true, "Can't find file rule %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc();

    // save the name of this pmFPAfile
    file->name = psStringCopy(name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr(&status, data, "FILENAME.RULE"));

    const char *type = psMetadataLookupStr(&status, data, "FILE.TYPE");
    file->type = pmFPAfileTypeFromString(type);
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        psFree(file);
        return NULL;
    }

    file->mode = PM_FPA_MODE_WRITE;
    file->save = false;

// XXX we are deprecating the use of FILE.SAVE in the camera config
# if (0)
    char *save = psMetadataLookupStr (&status, data, "FILE.SAVE");
    if (save != NULL) {
        if (!strcasecmp (save, "TRUE"))     {
            file->save = true;
        } else if (!strcasecmp (save, "FALSE"))     {
            file->save = false;
        } else {
            psError(PS_ERR_IO, true, "Illegal value \"%s\" for FILE.SAVE for %s", save, name);
            psFree(file);
            return NULL;
        }
    }
# endif

    file->camera = psMemIncrRefCounter(camera);
    file->cameraName = psMemIncrRefCounter(cameraName);

    // Use the format we were told to, the format specified in the file rule, or default to the default format
    if (!formatName || strlen(formatName) == 0) {
        // select the format list from the selected camera
        formatName = psMetadataLookupStr(&status, data, "FILE.FORMAT");
        if (!formatName || strcmp(formatName, "NONE") == 0) {
            // Try to get by with the default
            formatName = config->formatName;
        }
    }
    psMetadata *formats = psMetadataLookupMetadata(&status, file->camera, "FORMATS"); // List of formats
    psMetadata *format = psMetadataLookupMetadata(&status, formats, formatName); // Camera format to use
    if (!format) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find format %s for file %s.\n",
                formatName, file->name);
        psFree(file);
        return NULL;
    }
    file->format = psMemIncrRefCounter(format);
    file->formatName = psStringCopy(formatName);

    if (fpa) {
        file->fpa = psMemIncrRefCounter(fpa);
    } else {
        file->fpa = pmFPAConstruct(file->camera);
    }

    // Get FITS output scheme
    const char *fitsType = psMetadataLookupStr(&status, data, "FITS.TYPE"); // Name of FITS scheme to use
    if (fitsType && strcasecmp(fitsType, "NONE") != 0) {
        psMetadata *fitsTypes = psMetadataLookupMetadata(&status, camera, "FITS"); // The FITS schemes
        if (!fitsTypes) {
            psWarning("Unable to find FITS in camera configuration --- compression disabled.");
            goto COMPRESSION_DONE;
        }
        psMetadata *scheme = psMetadataLookupMetadata(NULL, fitsTypes, fitsType); // FITS scheme
        if (!scheme) {
            psWarning("Unable to find %s in FITS in camera configuration --- compression disabled.",
                      fitsType);
            goto COMPRESSION_DONE;
        }
        const char *typeString = psMetadataLookupStr(NULL, scheme, "COMPRESSION"); // Compression type
        if (!typeString || strlen(typeString) == 0) {
            psWarning("Can't find COMPRESSION in FITS scheme %s --- compression disabled.", fitsType);
            goto COMPRESSION_DONE;
        }
        psFitsCompressionType type = psFitsCompressionTypeFromString(typeString); // Compression type enum

        psString source = NULL;         // Source of options
        psStringAppend(&source, "%s in FITS in camera %s", fitsType, cameraName);
        psVector *tile = psVectorAlloc(3, PS_TYPE_S32); // Tile sizes
        file->bitpix = parseOptionInt(scheme, "BITPIX", source, 0); // Bits per pixel
        tile->data.S32[0] = parseOptionInt(scheme, "TILE.X", source, 0); // Tiling in x
        tile->data.S32[1] = parseOptionInt(scheme, "TILE.Y", source, 1); // Tiling in y
        tile->data.S32[2] = parseOptionInt(scheme, "TILE.Z", source, 1); // Tiling in z
        int noise = parseOptionInt(scheme, "NOISE", source, 16); // Noise bits
        int hscale = parseOptionInt(scheme, "HSCALE", source, 0); // Scaling for HCOMPRESS
        int hsmooth = parseOptionInt(scheme, "HSMOOTH", source, 0); // Smoothing for HCOMPRESS
        psFree(source);

        file->compression = psFitsCompressionAlloc(type, tile, noise, hscale, hsmooth);
        psFree(tile);
    }
COMPRESSION_DONE:

    file->fileLevel = pmFPAPHULevel(format);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "Unable to determine file level for %s\n", name);
        psFree(file);
        return NULL;
    }

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr(&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        psFree(file);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;
    file->fileLevel = PS_MIN (file->fileLevel, file->dataLevel);

    // XXX the file/data/free level must be consistent with the reference fpa (but since we
    // don't have access to its pmFPAfile, we cannot enforce this here...

    pmFPALevel extLevel = pmFPAExtensionsLevel(format); // Level for extensions
    if (extLevel != PM_FPA_LEVEL_NONE) {
        if (extLevel < file->dataLevel) {
            psWarning("Level for extensions is higher than desired data level --- adjusting.\n");
            file->dataLevel = extLevel;
        }
        if (extLevel < file->freeLevel) {
            psWarning("Level for extensions is higher than desired free level --- adjusting.\n");
            file->freeLevel = extLevel;
        }
    } else {
        // if we do not have extensions in the file, we are forced to write out at the file level
        file->dataLevel = file->fileLevel;
        file->freeLevel = file->fileLevel;
    }

    psTrace ("psModules.camera", 5, "file: %s, format: %s, fileLevel: %s, extLevel: %s, dataLevel: %s, freeLevel: %s\n",
             file->name, file->formatName, pmFPALevelToName (file->fileLevel), pmFPALevelToName(extLevel), pmFPALevelToName (file->dataLevel), pmFPALevelToName (file->freeLevel));

    // add argument-supplied OUTPUT name to this file
    char *outname = psMetadataLookupStr(&status, config->arguments, "OUTPUT");
    psMetadataAddStr(file->names, PS_LIST_TAIL, "OUTPUT", PS_META_NO_REPLACE, "", outname);

    // place the resulting file in the config system
    psMetadataAddPtr (config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree (file); // we free this copy of file, but 'files' still has a copy
    return (file); // the returned value is a view into the version on 'files'
}

// define a pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineOutput(const pmConfig *config, pmFPA *fpa, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    return pmFPAfileDefineOutputForFormat(config, fpa, name, NULL, NULL);
}

// define a pmFPAfile, bind to the optional file if supplied
pmFPAfile *pmFPAfileDefineOutputFromFile(const pmConfig *config, pmFPAfile *file, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    char *cameraName = NULL, *formatName = NULL; // Name of camera and format
    pmFPA *fpa = NULL;                  // FPA for file
    if (file) {
        cameraName = file->cameraName;
        formatName = file->formatName;
        fpa = file->fpa;
    }

    return pmFPAfileDefineOutputForFormat(config, fpa, name, cameraName, formatName);
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (must represent a single FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromArgs(bool *success, pmConfig *config, const char *filename, const char *argname)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    // use success to identify valid exit conditions (as opposed to 'argument not supplied')
    if (success) *success = false;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        if (success) *success = true;
        return NULL;
    }
    if (infiles->n < 1) {
        psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments\n", infiles->n, argname);
        return NULL;
    }

    // this function is implicitly an INPUT operation: do not create the file
    psString realName = pmConfigConvertFilename (infiles->data[0], config, false);
    if (!realName) {
        psError(PS_ERR_IO, false, "Failed to convert file name %s\n", (char *) infiles->data[0]);
        return NULL;
    }

    // load the header of the first image
    // EXTWORD (fits->extword) is not relevant to the PHU
    fits = psFitsOpen (realName, "r");
    if (!fits) {
        psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
        psFree (realName);
        return NULL;
    }
    phu = psFitsReadHeader (NULL, fits);
    if (!phu) {
        psError(PS_ERR_IO, false, "Failed to read file header %s\n", realName);
        psFree (realName);
        return NULL;
    }
    psFitsClose(fits);

    // determine the current format from the header
    // determine camera if not specified already
    format = pmConfigCameraFormatFromHeader (config, phu, true);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
        psFree(phu);
        psFree (realName);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", realName);
        psFree (realName);
        return NULL;
    }
    psFree (realName);

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    psFree (format);
    file->format = psMemIncrRefCounter(format);
    file->formatName = psStringCopy(config->formatName);

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    file->filerule = psStringCopy ("@FILES");
    file->filesrc = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    file->fileLevel = pmFPAPHULevel(format);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "Unable to determine file level for %s\n", file->name);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        psFree(file);
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    for (int i = 0; i < infiles->n; i++) {
        if (i > 0) {
            // this function is implicitly an INPUT operation: do not create the file
            psString realName = pmConfigConvertFilename (infiles->data[i], config, false);
            if (!realName) {
                psError(PS_ERR_IO, false, "Failed to convert file name %s", (char *) infiles->data[i]);
                return NULL;
            }
            fits = psFitsOpen (realName, "r");
            if (!fits) {
                psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
                psFree (realName);
                return NULL;
            }
            phu = psFitsReadHeader (NULL, fits);
            if (!phu) {
                psError(PS_ERR_IO, false, "Failed to read file header %s", realName);
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            bool valid = false;
            if (!pmConfigValidateCameraFormat (&valid, format, phu)) {
                psError (PS_ERR_UNKNOWN, false, "Error in config scripts\n");
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            if (!valid) {
                psError(PS_ERR_IO, false, "file %s is not from the required camera", realName);
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            psFree(realName);
            psFitsClose (fits);
        }

        // set the view to the corresponding entry for this phu
        pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
        if (!view) {
            psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
            psFree(phu);
            psFree (fpa);
            psFree (format);
            return NULL;
        }

        // associate the filename with the FPA element
        char *name = pmFPAfileNameFromRule (file->filesrc, file, view);

        // save the name association in the pmFPAfile structure
        psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[i]);

        psFree (view);
        psFree (name);
        psFree (phu);
    }
    psFree (fpa);
    if (success) *success = true;

    return file;
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (must represent a single FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileBindFromArgs (bool *success, pmFPAfile *input, pmConfig *config, const char *filename, const char *argname)
{
    PS_ASSERT_PTR_NON_NULL(input, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;

    // use success to identify valid exit conditions (as opposed to 'argument not supplied')
    if (success) *success = false;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        // this is not an error: this just means no matching argument was supplied
        if (success) *success = true;
        return NULL;
    }
    if (infiles->n < 1) {
        psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments\n", infiles->n, argname);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    file = pmFPAfileDefineInput (config, input->fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        return NULL;
    }

    // set derived values
    file->fileLevel = input->fileLevel;

    // define the rule to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filesrc);
    file->filerule = psStringCopy ("@FILES");
    file->filesrc = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    psMetadata *format = NULL;
    for (int i = 0; i < infiles->n; i++) {
        // this function is implicitly an INPUT operation: do not create the file
        psString realName = pmConfigConvertFilename (infiles->data[i], config, false);
        if (!realName) {
            psError(PS_ERR_IO, false, "Failed to convert file name %s", (char *) infiles->data[i]);
            return NULL;
        }
        fits = psFitsOpen (realName, "r");
        if (!fits) {
            psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
            psFree (realName);
            return NULL;
        }
        phu = psFitsReadHeader (NULL, fits);
        if (!phu) {
            psError(PS_ERR_IO, false, "Failed to read file header %s", realName);
            psFree (realName);
            psFitsClose (fits);
            return NULL;
        }

        if (!format) {
            format = pmConfigCameraFormatFromHeader(config, phu, true);
            if (!format) {
                psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
                psFree(phu);
                psFree(realName);
                psFitsClose(fits);
                psFree(file);
                return NULL;
            }
        } else {
            bool valid = false;
            if (!pmConfigValidateCameraFormat(&valid, format, phu)) {
                psError(PS_ERR_UNKNOWN, false, "Error in config scripts\n");
                psFree(realName);
                psFree(file);
                psFitsClose(fits);
                return NULL;
            }
            if (!valid) {
                psError(PS_ERR_IO, false, "specified data file %s does not match format of supplied INPUT\n",
                        realName);
                psFree(realName);
                psFree(file);
                psFitsClose(fits);
                return NULL;
            }
        }

        psFree(realName);
        psFitsClose(fits);

        // set the view to the corresponding entry for this phu
        pmFPAview *view = pmFPAIdentifySourceFromHeader (input->fpa, phu, format);
        if (!view) {
            psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
            psFree(phu);
            return NULL;
        }

        // associate the filename with the FPA element
        char *name = pmFPAfileNameFromRule(file->filesrc, file, view);

        // save the name association in the pmFPAfile structure
        psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[i]);

        psFree (view);
        psFree (name);
        psFree (phu);
    }
    psFree(file->format);
    file->format = format;
    file->formatName = psStringCopy(config->formatName);

    if (success) *success = true;
    return file;
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (each represents the same FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filenames and bind them to the FPAs
// save the pmFPAfiles on config->files
// return the pmFPAfiles (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineSingleFromArgs (bool *success, pmConfig *config, const char *filename,
        const char *argname, int entry)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    if (success) *success = false;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        psTrace("psModules.camera", 5, "Failed to find %s in argument list", argname);
        if (success) *success = true;
        return NULL;
    }
    if (infiles->n <= entry) {
        psError(PS_ERR_IO, false, "only %ld files in %s in argument, entry %d requested\n", infiles->n, argname, entry);
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    fits = psFitsOpen (infiles->data[entry], "r");
    phu = psFitsReadHeader (NULL, fits);
    psFitsClose (fits);

    // on first call to this function, config->camera is not set.
    // later calls will give an error if the cameras do not match
    format = pmConfigCameraFormatFromHeader (config, phu, true);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", (char *)infiles->data[0]);
        psFree(phu);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", (char *)infiles->data[0]);
        psFree(phu);
        psFree(format);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    // we need a variable name here... (but in filerule)
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    psFree (file->format);
    file->format = format;
    file->formatName = psStringCopy(config->formatName);

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filesrc);
    file->filerule = psStringCopy ("@FILES");
    file->filesrc = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    file->fileLevel = pmFPAPHULevel(format);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "Unable to determine file level for %s\n", file->name);
        psFree(phu);
        psFree(fpa);
        psFree(file);
        psFree(format);
        return NULL;
    }

    // set the view to the corresponding entry for this phu
    pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
    if (!view) {
        psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
        psFree(phu);
        psFree(fpa);
        psFree(file);
        psFree(format);
        return NULL;
    }

    // associate the filename with the FPA element
    char *name = pmFPAfileNameFromRule (file->filesrc, file, view);

    // save the name association in the pmFPAfile structure
    psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[entry]);

    psFree(phu);
    psFree(fpa);
    psFree(view);
    psFree(name);
    psFree(format);

    if (success) *success = true;
    return file;
}

// define the named pmFPAfile from the camera->config
// only valid for pmFPAfile->mode = READ
pmFPAfile *pmFPAfileDefineFromConf (bool *success, const pmConfig *config, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    if (success) *success = false;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }

    // build the template fpa, set up the basic view
    pmFPA *fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    pmFPAfile *file = pmFPAfileDefineInput (config, fpa, filename);
    psFree (fpa);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        return NULL;
    }

    // image names may not come from file->names
    if (!strcasecmp (file->filerule, "@FILES")) {
        psError(PS_ERR_IO, true, "supplied filerule uses illegal value @FILES");
        // XXX remove the file from config->files
        psFree(file);
        return NULL;
    }

    // image names may come from the detrend database
    if (!strcasecmp (file->filerule, "@DETDB")) {
        psTrace ("pmFPAfile", 5, "requiring use of detrend database source\n");
        // don't free the file here: it is left on config->files
        // to be used optionally by pmFPAfileDefineFromDetDB (or others)
        if (success) *success = true;
        return NULL;
    }

    // Prepend the global path to the file rule
    // this function is implicitly an INPUT operation: do not create the file
    psString tmpName = pmConfigConvertFilename (file->filerule, config, false);
    psFree (file->filerule);
    file->filerule = tmpName;

    if (success) *success = true;

    return file;
}

// construct an FPA based on the supplied config->camera
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromDetDB (bool *success, const pmConfig *config, const char *filename,
                                     pmFPA *input, pmDetrendType type)
{
    PS_ASSERT_PTR_NON_NULL(input, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    bool status;
    pmFPA *fpa = NULL;
    pmFPAfile *file = NULL;

    if (success) *success = false;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }
    // a camera config is needed (as source of file rule)
    if (config->cameraName == NULL) {
        psAbort("camera defined but not cameraName!");
    }

    // find or define a pmFPAfile with this name
    file = psMetadataLookupPtr (NULL, config->files, filename);
    if (!file) {
        // build the template fpa, set up the basic view
        fpa = pmFPAConstruct (config->camera);
        if (!fpa) {
            psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
            return NULL;
        }
        // load the given filerule (from config->camera) and bind it to the fpa
        // the returned file is just a view to the entry on config->files
        file = pmFPAfileDefineInput (config, fpa, filename);
        if (!file) {
            psError(PS_ERR_IO, false, "file %s not defined\n", filename);
            psFree (fpa);
            return NULL;
        }
    }

    // we are constructing a detselect command of the form:
    //   detselect -search -inst (camera) -type (type) -time (time) [others]
    // camera, type, and time are derived from pmFPA *input, other options are
    // added if specified for the particular detrend type by the DETREND.CONSTRAINTS
    // note that the filter-dependent choices are set for ppImage in ppImageParseCamera
    // XXX make all of the detrend constraints explicit in DETREND.CONSTRAINTS?

    // Get the time from FPA.TIME
    psTime *time = psMetadataLookupPtr(NULL, input->concepts, "FPA.TIME");
    if (time->sec == 0 && time->nsec == 0) {
        psLogMsg ("psModules.camera", PS_LOG_WARN, "FPA.TIME has not been set.\n");
    }
    pmDetrendSelectOptions *options = pmDetrendSelectOptionsAlloc(config->cameraName, *time, type);

    // add additional constraints based on the type defined in the PPIMAGE recipe
    // XXX use PPIMAGE or DETREND for the recipe name?
    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, "PPIMAGE");
    if (!status) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE recipe not found.");
        psFree(options);
        psFree(fpa);
        return false;
    }
    psMetadata *detConstraints = psMetadataLookupPtr (&status, recipe, "DETREND.CONSTRAINTS");
    if (!status) {
        psWarning("DETREND.CONSTRAINTS not found --- no constraints will be applied.");
        goto DETREND_SELECT;
    }

    psString typeName = pmDetrendTypeToString (type);
    psMetadata *constraints = psMetadataLookupPtr (&status, detConstraints, typeName);
    if (!status) {
        psWarning("DETREND.CONSTRAINTS for type %s not found --- no contraints will be applied.", typeName);
        psFree(typeName);
        goto DETREND_SELECT;
    }
    psFree(typeName);

    // loop over the constraints and include in the detselect options
    psMetadataIterator *iter = psMetadataIteratorAlloc (constraints, PS_LIST_HEAD, NULL);
    psMetadataItem *item = NULL;
    while ((item = psMetadataGetAndIncrement (iter)) != NULL) {
        if (item->type != PS_DATA_STRING) {
            psWarning("Invalid type for DETREND.CONSTRAINT element %s --- ignoring constraint", item->name);
            continue;
        }
        char *option  = item->name;     // item->name must correspond to a valid detselect option
        char *concept = item->data.V;

        // these items refer to the corresponding values for the input image
        // (ie, -filter input:filter or -exptime input:exptime)
        if (!strcasecmp (option, "filter")) {
            options->filter = psMetadataLookupPtr (&status, input->concepts, concept);
            psMemIncrRefCounter (options->filter);
            if (!status)
                psAbort("failed to find filter (concept %s)", concept);
        } else if (!strcasecmp (option, "exptime")) {
            options->exptime = psMetadataLookupF32 (&status, input->concepts, concept);
            options->exptimeSet = true;
            if (!status)
                psAbort("exptime not found (concept %s)", concept);
        } else if (!strcasecmp (option, "airmass")) {
            options->airmass = psMetadataLookupF32 (&status, input->concepts, concept);
            options->airmassSet = true;
            if (!status)
                psAbort("airmass not found (concept %s)", concept);
        } else if (!strcasecmp (option, "dettemp")) {
            options->dettemp = psMetadataLookupF32 (&status, input->concepts, concept);
            options->dettempSet = true;
            if (!status)
                psAbort("dettemp not found (concept %s)", concept);
        } else if (!strcasecmp (option, "twilight")) {
            options->twilight = psMetadataLookupF32 (&status, input->concepts, concept);
            options->twilightSet = true;
            if (!status)
                psAbort("twilight not found (concept %s)", concept);
        }

        // the version is applied literally
        if (!strcasecmp (option, "version")) {
            options->version = psMemIncrRefCounter (concept);
        }
        // we can override the detrend database dettype if desired
        // ie, use DOMEFLAT for type FLAT
        // the dettype string is applied literally
        if (!strcasecmp (option, "dettype")) {
            options->dettype = psMemIncrRefCounter (concept);
        }
    }
    psFree(iter);

DETREND_SELECT:
    {
        // search for existing detrend data (detID)
        pmDetrendSelectResults *results = pmDetrendSelect (options, config);
        if (!results) {
            psError (PS_ERR_IO, false, "no matching detrend data");
            return NULL;
        }
        file->detrend = results;
    }

    psFree (options);

    if (success) *success = true;
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineFromFPA (const pmConfig *config, pmFPA *src, int xBin, int yBin, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(src, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    pmFPA *fpa = pmFPAConstruct(src->camera);
    pmFPAfile *file = pmFPAfileDefineOutput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s not defined\n", filename);
        return NULL;
    }
    file->src = psMemIncrRefCounter(src); // inherit output elements from this source pmFPA
    file->xBin = xBin;
    file->yBin = yBin;
    psFree (fpa);
    return file;
}

pmFPAfile *pmFPAfileDefineFromFile(const pmConfig *config, pmFPAfile *src, int xBin, int yBin,
                                   const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(src, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    pmFPAfile *file = pmFPAfileDefineOutputForFormat(config, NULL, filename, src->cameraName,
                                                     src->formatName);
    file->src = psMemIncrRefCounter(src->fpa); // inherit output elements from this source pmFPA
    file->xBin = xBin;
    file->yBin = yBin;
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineNewCamera (const pmConfig *config, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    pmFPAfile *file = pmFPAfileDefineOutput (config, NULL, filename);
    if (!file) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s not defined\n", filename);
        return NULL;
    }
    if (!file->camera) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s does not define a new camera\n", filename);
        return NULL;
    }
    file->fpa = pmFPAConstruct(file->camera);

    return file;
}

pmFPAfile *pmFPAfileDefineSkycell(const pmConfig *config, pmFPA *fpa, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->cameraName, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->formatName, NULL);

    pmFPAfile *file;                    // The new file

    if (config->cameraName[0] == '_' &&
        strcmp(config->cameraName + strlen(config->cameraName) - 8, "-SKYCELL") == 0) {
        // The input camera is already a skycell
        file = pmFPAfileDefineOutputForFormat(config, fpa, filename, config->cameraName, "SKYCELL");
    } else {
        psString cameraName = NULL;         // Name of the old camera configuration
        if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 5, "-CHIP") == 0) {
            cameraName = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 6);
        } else if (config->cameraName[0] == '_' &&
                   strcmp(config->cameraName + strlen(config->cameraName) - 4 , "-FPA") == 0) {
            cameraName = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 5);
        } else {
            cameraName = psMemIncrRefCounter(config->cameraName);
        }
        psString newCameraName = NULL;  // Name of the new (automatically-generated) camera configuration
        psStringAppend(&newCameraName, "_%s-SKYCELL", cameraName);
        file = pmFPAfileDefineOutputForFormat(config, fpa, filename, newCameraName, "SKYCELL");
        psFree(cameraName);
        psFree(newCameraName);
    }
    if (!file) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s not defined\n", filename);
        return NULL;
    }

    // Ensure everything is written out at the appropriate level
    file->fileLevel = PM_FPA_LEVEL_FPA;
    file->dataLevel = PM_FPA_LEVEL_FPA;
    file->freeLevel = PM_FPA_LEVEL_FPA;

    return file;
}

pmFPAfile *pmFPAfileDefineChipMosaic(const pmConfig *config, pmFPA *src, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(src, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->cameraName, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->formatName, NULL);

    pmFPAfile *file;                    // The new file
    if (config->cameraName[0] == '_' &&
        (strcmp(config->cameraName + strlen(config->cameraName) - 5, "-CHIP") == 0 ||
         strcmp(config->cameraName + strlen(config->cameraName) - 8, "-SKYCELL") == 0)) {
        // The input camera has already been mosaicked to this level
        file = pmFPAfileDefineOutputForFormat(config, NULL, filename, config->cameraName, config->formatName);
    } else {
        psString cameraName = NULL; // Name of the new (automatically-generated) camera configuration
        if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 4 , "-FPA") == 0) {
            cameraName = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 5);
        } else {
            cameraName = psMemIncrRefCounter(config->cameraName);
        }
        psString newCameraName = NULL;  // Name of the new (automatically-generated) camera configuration
        psStringAppend(&newCameraName, "_%s-CHIP", cameraName);

        // Find the correct camera configuration
        file = pmFPAfileDefineOutputForFormat(config, NULL, filename, newCameraName, config->formatName);
        psFree(newCameraName);
        psFree(cameraName);
    }
    if (!file) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s not defined\n", filename);
        return NULL;
    }

    file->src = psMemIncrRefCounter(src); // inherit output elements from this source pmFPA

    file->mosaicLevel = PM_FPA_LEVEL_CHIP; // don't do any I/O on this at a lower level

    return file;
}

pmFPAfile *pmFPAfileDefineFPAMosaic(const pmConfig *config, pmFPA *src, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(src, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->cameraName, NULL);

    pmFPAfile *file;                    // The new file
    if (config->cameraName[0] == '_' &&
        (strcmp(config->cameraName + strlen(config->cameraName) - 4 , "-FPA") == 0 ||
         strcmp(config->cameraName + strlen(config->cameraName) - 8, "-SKYCELL") == 0)) {
        // The input camera has already been mosaicked to this level
        file = pmFPAfileDefineOutputForFormat(config, NULL, filename, config->cameraName, config->formatName);
    } else {

        psString original = NULL;       // Name of the original camera configuration
        if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 5 , "-CHIP") == 0) {
            // It's a chip mosaic; we need to get the original name
            original = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 6);
        } else if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 8, "-SKYCELL") == 0) {
            original = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 9);
        } else {
            original = psMemIncrRefCounter(config->cameraName);
        }
        psString cameraName = NULL;
        psStringAppend(&cameraName, "_%s-FPA", original);
        psFree(original);

        file = pmFPAfileDefineOutputForFormat(config, NULL, filename, cameraName, config->formatName);
        psFree(cameraName);
    }
    if (!file) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "file %s not defined\n", filename);
        return NULL;
    }

    file->src = psMemIncrRefCounter(src); // inherit output elements from this source pmFPA

    file->mosaicLevel = PM_FPA_LEVEL_FPA; // don't do any I/O on this at a lower level

    return file;
}

// create a file with the given name, assign it type "INTERNAL", and supply it with an image
// of the requested dimensions. (image only, mask and weight are ignored)
pmReadout *pmFPAfileDefineInternal (psMetadata *files, const char *name, int Nx, int Ny, int type)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    pmReadout *readout = pmReadoutAlloc(NULL);
    readout->image = psImageAlloc(Nx, Ny, type);

    // I want an image from the
    pmFPAfile *file = pmFPAfileAlloc();
    file->mode = PM_FPA_MODE_INTERNAL;
    file->name = psStringCopy (name);

    file->readout = readout;
    psMetadataAddPtr(files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree(file);
    // we free this copy of file, but 'files' still has a copy

    return readout;
}

bool pmFPAfileDropInternal(psMetadata *files, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    bool status = false;

    pmFPAfile *file = psMetadataLookupPtr (&status, files, name);
    if (!status) {
        psTrace("psModules.camera", 6, "Internal File %s not in file list", name);
        return true;
    }
    if (file == NULL) {
        psError(PS_ERR_IO, true, "file %s is NULL", name);
        return false;
    }
    if (file->mode != PM_FPA_MODE_INTERNAL) {
        psTrace("psModules.camera", 6, "FPA File %s not Internal, not dropping", name);
        return true;
    }

    psTrace("psModules.camera", 6, "dropping Internal FPA File %s", name);
    psMetadataRemoveKey (files, name);
    return true;
}

