#include <stdio.h>
#include <pslib.h>

#include "pmConfig.h"
#include "pmHDU.h"
#include "pmFPA.h"
#include "pmFPAMaskWeight.h"
#include "pmFPAConstruct.h"
#include "pmFPAview.h"
#include "pmFPAfile.h"
#include "pmFPAfile.h"
#include "pmFPACopy.h"
#include "pmFPARead.h"
#include "pmFPAWrite.h"
#include "pmPeaks.h"
#include "pmMoments.h"
#include "pmModel.h"
#include "pmSource.h"
#include "pmSourceIO.h"
#include "pmGrowthCurve.h"
#include "pmPSF.h"
#include "pmPSF_IO.h"
#include "pmFPA_JPEG.h"
#include "pmDetrendDB.h"

// define an input-type pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineInput(pmConfig *config, pmFPA *fpa, char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_PTR_NON_NULL(name, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(name), NULL);

    bool status;
    char *type;

    // select the FILERULES from config->camera
    psMetadata *filerules = psMetadataLookupPtr (&status, config->camera, "FILERULES");
    if (filerules == NULL) {
        psError(PS_ERR_IO, true, "Can't find FILERULES in the CAMERA configuration!");
        return NULL;
    }

    // select the name from the FILERULES
    // check for alias name (type == STR, name is aliased name)
    char *realname = psMetadataLookupStr (&status, filerules, name);
    if (!realname || strlen(realname) == 0) {
        realname = name;
    }

    psMetadata *data = psMetadataLookupPtr (&status, filerules, realname);
    if (data == NULL) {
        psError(PS_ERR_IO, true, "Can't find file concept %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc ();

    // save the name of this pmFPAfile
    file->name = psStringCopy (name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.RULE"));
    file->filextra = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.XTRA"));
    file->extrule  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.RULE"));
    file->extxtra  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.XTRA"));

    file->type = PM_FPA_FILE_NONE;
    type = psMetadataLookupStr (&status, data, "FILE.TYPE");
    if (type != NULL) {
        if (!strcasecmp (type, "SX"))     {
            file->type = PM_FPA_FILE_SX;
        }
        if (!strcasecmp (type, "OBJ"))     {
            file->type = PM_FPA_FILE_OBJ;
        }
        if (!strcasecmp (type, "CMP"))     {
            file->type = PM_FPA_FILE_CMP;
        }
        if (!strcasecmp (type, "CMF"))     {
            file->type = PM_FPA_FILE_CMF;
        }
        if (!strcasecmp (type, "RAW"))     {
            file->type = PM_FPA_FILE_RAW;
        }
        if (!strcasecmp (type, "IMAGE"))     {
            file->type = PM_FPA_FILE_IMAGE;
        }
        if (!strcasecmp (type, "PSF"))     {
            file->type = PM_FPA_FILE_PSF;
        }
        if (!strcasecmp (type, "JPEG"))     {
            file->type = PM_FPA_FILE_JPEG;
        }
    }
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        return NULL;
    }

    file->mode = PM_FPA_MODE_READ;
    file->fileLevel = PM_FPA_LEVEL_NONE; // the fileLevel depends on the input data

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr (&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;

    if (fpa != NULL) {
        file->fpa = psMemIncrRefCounter(fpa);
        file->camera = psMemIncrRefCounter((psMetadata *)fpa->camera);
    }

    // XXX ppImage and similar require the added file to be unique
    // XXX ppFocus wants to override the selection with the new selection
    // XXX require programs like ppFocus to remove existing files by hand
    if (!psMetadataAddPtr (config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file)) {
        psError (PS_ERR_IO, false, "could not add %s to config files", name);
        return NULL;
    }
    psFree (file);
    return (file);
}

// define a pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineOutput(pmConfig *config, pmFPA *fpa, char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_PTR_NON_NULL(name, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(name), NULL);

    bool status;
    char *type;

    // select the FILERULES from the camera config
    psMetadata *filerules = psMetadataLookupPtr (&status, config->camera, "FILERULES");
    if (filerules == NULL) {
        psError(PS_ERR_IO, true, "Can't find FILERULES in the CAMERA configuration!");
        return NULL;
    }

    // select the name from the FILERULES
    // check for alias name (type == STR, name is aliased name)
    char *realname = psMetadataLookupStr (&status, filerules, name);
    if (!realname || strlen(realname) == 0) {
        realname = name;
    }

    psMetadata *data = psMetadataLookupPtr (&status, filerules, realname);
    if (data == NULL) {
        psError(PS_ERR_IO, true, "Can't find file concept %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc ();

    // save the name of this pmFPAfile
    file->name = psStringCopy (name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.RULE"));
    file->filextra = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.XTRA"));
    file->extrule  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.RULE"));
    file->extxtra  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.XTRA"));

    file->type = PM_FPA_FILE_NONE;
    type = psMetadataLookupStr (&status, data, "FILE.TYPE");
    if (type != NULL) {
        if (!strcasecmp (type, "SX"))     {
            file->type = PM_FPA_FILE_SX;
        }
        if (!strcasecmp (type, "OBJ"))     {
            file->type = PM_FPA_FILE_OBJ;
        }
        if (!strcasecmp (type, "CMP"))     {
            file->type = PM_FPA_FILE_CMP;
        }
        if (!strcasecmp (type, "CMF"))     {
            file->type = PM_FPA_FILE_CMF;
        }
        if (!strcasecmp (type, "RAW"))     {
            file->type = PM_FPA_FILE_RAW;
        }
        if (!strcasecmp (type, "IMAGE"))     {
            file->type = PM_FPA_FILE_IMAGE;
        }
        if (!strcasecmp (type, "PSF"))     {
            file->type = PM_FPA_FILE_PSF;
        }
        if (!strcasecmp (type, "JPEG"))     {
            file->type = PM_FPA_FILE_JPEG;
        }
    }
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        return NULL;
    }

    file->mode = PM_FPA_MODE_WRITE;
    file->save = false;
    char *save = psMetadataLookupStr (&status, data, "FILE.SAVE");
    if (save != NULL) {
        if (!strcasecmp (save, "TRUE"))     {
            file->save = true;
        }
        if (!strcasecmp (save, "FALSE"))     {
            file->save = false;
        }
    }

    file->fileLevel = pmFPALevelFromName(psMetadataLookupStr(&status, data, "FILE.LEVEL"));
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "FILE.LEVEL is not set for %s\n", name);
        return NULL;
    }

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr (&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;

    if (fpa != NULL) {
        file->fpa = psMemIncrRefCounter(fpa);
    }

    // for WRITE type of data, the output format needs to be determined
    // this is only needed if the output file is not identical in structure to the input?
    // for
    char *formatLine = psMetadataLookupStr (&status, data, "FILE.FORMAT");
    if (formatLine && strcasecmp (formatLine, "NONE")) {

        char *formatName = NULL;

        psArray *words = psStringSplitArray (formatLine, ":", false);
        if (!words || !words->n) {
            psAbort("pmFPAfileDefine", "programming error: cannot be NULL or empty!");
        }
        if (words->n == 1) {
            // format selected in this camera
            formatName = formatLine;
            file->camera = psMemIncrRefCounter (config->camera);
        }
        if (words->n == 2) {
            // select the requested camera
            formatName = words->data[1];
            file->camera = pmConfigCameraByName(config, words->data[0]);
            if (!file->camera) {
                psError(PS_ERR_IO, false, "camera %s not found\n", words->data[0]);
                psFree(words);
                return NULL;
            }
        }

        // select the format list from the selected camera
        psMetadata *formats = psMetadataLookupMD(&status, file->camera, "FORMATS"); // List of formats

        // select the desired format
        char *formatFile = psMetadataLookupStr (&status, formats, formatName);
        if (!formatFile) {
            psError(PS_ERR_IO, false, "format %s for %s not defined", formatName, name);
            psFree (file);
            psFree (words);
            return NULL;
        }
        readConfig (&file->format, formatFile, formatName);
        psFree (words);
    }

    // add argument-supplied OUTPUT name to this file
    char *outname = psMetadataLookupStr(&status, config->arguments, "OUTPUT");
    psMetadataAddStr (file->names, PS_LIST_TAIL, "OUTPUT", PS_META_NO_REPLACE, "", outname);

    // place the resulting file in the config system
    psMetadataAddPtr (config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree (file); // we free this copy of file, but 'files' still has a copy
    return (file); // the returned value is a view into the version on 'files'
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (must represent a single FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromArgs (bool *found, pmConfig *config, char *filename, char *argname)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(filename, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(filename), NULL);
    PS_ASSERT_PTR_NON_NULL(argname, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(argname), NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    if (*found)
        return NULL;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        psTrace("pmFPAfile", 5, "Failed to find %s in argument list", argname);
        return NULL;
    }
    if (infiles->n < 1) {
        psError(PS_ERR_IO, false, "Found n == %d files in %s in arguments\n", infiles->n, argname);
        return NULL;
    }

    // load the header of the first image
    fits = psFitsOpen (infiles->data[0], "r");
    phu = psFitsReadHeader (NULL, fits);
    psFitsClose (fits);

    // determine the current format from the header
    // determine camera if not specified already
    format = pmConfigCameraFormatFromHeader (config, phu);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", infiles->data[0]);
        psFree(phu);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", infiles->data[0]);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // find the matching fileLevel
    psMetadata *filemenu = psMetadataLookupPtr (&status, format, "FILE");
    if (!filemenu) {
        psError (PS_ERR_IO, true, "missing FILE in FORMAT");
        return NULL;
    }
    char *levelName = psMetadataLookupStr (&status, filemenu, "PHU");
    if (!levelName) {
        psError (PS_ERR_IO, true, "missing PHU in FILE in FORMAT");
        return NULL;
    }
    file->fileLevel = pmFPALevelFromName (levelName);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError (PS_ERR_IO, true, "unknown level");
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    for (int i = 0; i < infiles->n; i++) {
        if (i > 0) {
            fits = psFitsOpen (infiles->data[i], "r");
            phu = psFitsReadHeader (NULL, fits);
            pmConfigValidateCameraFormat (format, phu);
            psFitsClose (fits);
        }

        // set the view to the corresponding entry for this phu
        pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
        if (!view) {
            psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
            psFree(phu);
            psFree (fpa);
            psFree (format);
            return NULL;
        }

        // associate the filename with the FPA element
        char *name = pmFPAfileNameFromRule (file->filextra, file, view);

        // save the name association in the pmFPAfile structure
        psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[i]);

        psFree (view);
        psFree (name);
        psFree (phu);
    }
    psFree (fpa);

    file->format = format;
    *found = true;
    return file;
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (each represents the same FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filenames and bind them to the FPAs
// save the pmFPAfiles on config->files
// return the pmFPAfiles (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineSingleFromArgs (bool *found, pmConfig *config, char *filename, char *argname, int entry)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(filename, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(filename), NULL);
    PS_ASSERT_PTR_NON_NULL(argname, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(argname), NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    if (*found)
        return NULL;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        psTrace("pmFPAfile", 5, "Failed to find %s in argument list", argname);
        return NULL;
    }
    if (infiles->n <= entry) {
        psError(PS_ERR_IO, false, "only %d files in %s in argument, entry %d requested\n", infiles->n, argname, entry);
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    fits = psFitsOpen (infiles->data[entry], "r");
    phu = psFitsReadHeader (NULL, fits);
    psFitsClose (fits);

    // on first call to this function, config->camera is not set.
    // later calls will give an error if the cameras do not match
    format = pmConfigCameraFormatFromHeader (config, phu);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", infiles->data[0]);
        psFree(phu);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", infiles->data[0]);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    // we need a variable name here... (but in filerule)
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    file->format = format;

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // find the matching fileLevel
    psMetadata *filemenu = psMetadataLookupPtr (&status, format, "FILE");
    if (!filemenu) {
        psError (PS_ERR_IO, true, "missing FILE in FORMAT");
        return NULL;
    }
    char *levelName = psMetadataLookupStr (&status, filemenu, "PHU");
    if (!levelName) {
        psError (PS_ERR_IO, true, "missing PHU in FILE in FORMAT");
        return NULL;
    }
    file->fileLevel = pmFPALevelFromName (levelName);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError (PS_ERR_IO, true, "unknown level");
        return NULL;
    }

    // set the view to the corresponding entry for this phu
    pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
    if (!view) {
        psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
        psFree(phu);
        psFree (fpa);
        psFree (format);
        return NULL;
    }

    // associate the filename with the FPA element
    char *name = pmFPAfileNameFromRule (file->filextra, file, view);

    // save the name association in the pmFPAfile structure
    psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[entry]);

    psFree (view);
    psFree (name);
    psFree (phu);
    psFree (fpa);

    file->format = format;
    *found = true;
    return file;
}

// define the named pmFPAfile from the camera->config
// only valid for pmFPAfile->mode = READ
pmFPAfile *pmFPAfileDefineFromConf (bool *found, pmConfig *config, char *filename)
{
    bool status;

    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(filename, false);
    PS_ASSERT_INT_POSITIVE(strlen(filename), false);

    if (*found)
        return NULL;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }

    // build the template fpa, set up the basic view
    pmFPA *fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    pmFPAfile *file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(fpa);
        psFree(file);
        return NULL;
    }

    // image names may not come from file->names
    if (!strcasecmp (file->filerule, "@FILES")) {
        psError(PS_ERR_IO, true, "supplied filerule uses illegal value @FILES");
        // XXX remove the file from config->files
        psFree(fpa);
        psFree(file);
        return NULL;
    }

    // image names may come from the detrend database
    if (!strcasecmp (file->filerule, "@DETDB")) {
        psTrace ("pmFPAfile", 5, "requiring use of detrend database source\n");
        // don't free the file here: it is left on config->files
        // to be used optionally by pmFPAfileDefineFromDetDB (or others)
        psFree(fpa);
        return NULL;
    }

    // use the supplied filename
    char *infile = psStringCopy (file->filerule);

    // load the header of the first image
    psFits *fits = psFitsOpen (infile, "r");
    psMetadata *phu = psFitsReadHeader (NULL, fits);
    psFitsClose (fits);

    // determine the current format from the header
    // determine camera if not specified already
    file->format = pmConfigCameraFormatFromHeader (config, phu);
    if (!file->format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", infile);
        psFree(phu);
        psFree(fpa);
        psFree(file);
        psFree(infile);
        return NULL;
    }

    // set the view to the corresponding entry for this phu
    pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, file->format);
    if (!view) {
        psError(PS_ERR_IO, false, "Unable to determine source for %s (%s)", file->name, infile);
        psFree(phu);
        psFree(fpa);
        psFree(file);
        psFree(infile);
        return NULL;
    }

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // find the matching fileLevel
    psMetadata *filemenu = psMetadataLookupPtr (&status, file->format, "FILE");
    if (!filemenu) {
        psError (PS_ERR_IO, true, "missing FILE in FORMAT");
        psFree(phu);
        psFree(fpa);
        psFree(file);
        psFree(view);
        psFree(infile);
        return NULL;
    }
    char *levelName = psMetadataLookupStr (&status, filemenu, "PHU");
    if (!levelName) {
        psError (PS_ERR_IO, true, "missing PHU in FILE in FORMAT");
        return NULL;
    }
    file->fileLevel = pmFPALevelFromName (levelName);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError (PS_ERR_IO, true, "unknown level");
        return NULL;
    }

    // associate the filename with the FPA element
    char *name = pmFPAfileNameFromRule (file->filextra, file, view);

    // save the name association in the pmFPAfile structure
    psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infile);

    psFree (phu);
    psFree (fpa);
    psFree (view);
    psFree (name);
    psFree (infile);

    *found = true;
    return file;
}

// construct an FPA based on the supplied config->camera
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromDetDB (bool *found, pmConfig *config, char *filename, pmFPA *input, pmDetrendType type)
{
    PS_ASSERT_PTR_NON_NULL(input, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(filename, NULL);

    pmFPA *fpa = NULL;
    pmFPAfile *file = NULL;

    if (*found)
        return NULL;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }

    // find or define a pmFPAfile with this name
    file = psMetadataLookupPtr (NULL, config->files, filename);
    if (!file) {
        // build the template fpa, set up the basic view
        fpa = pmFPAConstruct (config->camera);
        if (!fpa) {
            psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
            return NULL;
        }
        // load the given filerule (from config->camera) and bind it to the fpa
        // the returned file is just a view to the entry on config->files
        file = pmFPAfileDefineInput (config, fpa, filename);
        if (!file) {
            psError(PS_ERR_IO, false, "file %s not defined\n", filename);
            psFree (fpa);
            return NULL;
        }
    }

    // detselect -camera (camera) -time (time) -type (type) [others]
    // camera and time are functions of (pmFPA *input)

    // XXX does this do something NASTY to the stack??
    psTime *time = psTimeGetNow (PS_TIME_TAI);

    psString camera = psStringCopy ("megacam");
    pmDetrendSelectOptions *options = pmDetrendSelectOptionsAlloc(camera, *time, type);
    psFree (camera);
    psFree (time);

    // add other options here f(pmFPA *input, type)

    // search for existing detrend data (detID)
    pmDetrendSelectResults *results = pmDetrendSelect (options);
    if (!results) {
        psLogMsg ("pmFPAfile", 2, "no matching detrend data");
        psError (PS_ERR_IO, true, "no matching detrend data");
        return NULL;
    }

    // replace the existing value of filextra with the selected detID
    psFree (file->filextra);
    file->filextra  = psStringCopy (results->detID);

    // set the fileLevel based on the detDB results
    file->fileLevel = results->level;

    psFree (results);
    psFree (options);

    *found = true;
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineFromFPA (pmConfig *config, pmFPA *src, int xBin, int yBin, char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(src, false);
    PS_ASSERT_PTR_NON_NULL(filename, false);
    PS_ASSERT_INT_POSITIVE(strlen(filename), false);

    pmFPA *fpa = pmFPAConstruct (src->camera);
    pmFPAfile *file = pmFPAfileDefineOutput (config, fpa, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    file->src = src; // inherit output elements from this source pmFPA
    file->xBin = xBin;
    file->yBin = yBin;

    psFree (fpa);
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineNewCamera (pmConfig *config, char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(filename, false);
    PS_ASSERT_INT_POSITIVE(strlen(filename), false);

    pmFPAfile *file = pmFPAfileDefineOutput (config, NULL, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    if (!file->camera) {
        psErrorStackPrint(stderr, "file %s does not define a new camera\n", filename);
        return NULL;
    }
    file->fpa = pmFPAConstruct(file->camera);

    return file;
}

// create a file with the given name, assign it type "INTERNAL", and supply it with an image
// of the requested dimensions. (image only, mask and weight are ignored)
pmReadout *pmFPAfileDefineInternal (psMetadata *files, char *name, int Nx, int Ny, int type)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_PTR_NON_NULL(name, false);
    PS_ASSERT_INT_POSITIVE(strlen(name), false);

    pmReadout *readout = pmReadoutAlloc(NULL);
    readout->image = psImageAlloc(Nx, Ny, type);

    // I want an image from the
    pmFPAfile *file = pmFPAfileAlloc();
    file->mode = PM_FPA_MODE_INTERNAL;
    file->name = psStringCopy (name);

    file->readout = readout;
    psMetadataAddPtr(files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree(file);
    // we free this copy of file, but 'files' still has a copy

    return readout;
}

bool pmFPAfileDropInternal(psMetadata *files, char *name)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_PTR_NON_NULL(name, false);
    PS_ASSERT_INT_POSITIVE(strlen(name), false);

    bool status = false;

    pmFPAfile *file = psMetadataLookupPtr (&status, files, name);
    if (!status) {
        psTrace("pmFPAfile", 6, "Internal File %s not in file list", name);
        return true;
    }
    if (file == NULL) {
        psError(PS_ERR_IO, true, "file %s is NULL", name);
        return false;
    }
    if (file->mode != PM_FPA_MODE_INTERNAL) {
        psTrace("pmFPAfile", 6, "FPA File %s not Internal, not dropping", name);
        return true;
    }

    psMetadataRemoveKey (files, name);
    return true;
}

