/** @file  pmConfig.h
 *
 *  @author PAP (IfA)
 *  @author EAM (IfA)
 *
 *  @version $Revision: 1.104.2.1 $ $Name: not supported by cvs2svn $
 *  @date $Date: 2007-09-22 01:05:48 $
 *
 *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
 *
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <strings.h>            /* for strn?casecmp */
#include <unistd.h>
#include <libgen.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <glob.h>
#include <pslib.h>

#include "pmConfig.h"
#include "pmErrorCodes.h"
#include "pmFPALevel.h"
#include "pmConfigRecipes.h"
#include "pmConfigCamera.h"

#ifdef HAVE_NEBCLIENT
#include <nebclient.h>
#endif // ifdef HAVE_NEBCLIENT

#define PS_SITE "PS_SITE"         // Name of the environment variable containing the site config file
#define PS_DEFAULT_SITE ".ipprc"  // Default site config file

static bool readCameraConfig = true;    // Read the camera config on startup (with pmConfigRead)?
static psArray *configPath = NULL;      // Search path for configuration files

bool pmConfigReadParamsSet(bool newReadCameraConfig)
{
    bool oldReadCameraConfig = readCameraConfig;
    readCameraConfig = newReadCameraConfig;
    return oldReadCameraConfig;
}

static void configFree(pmConfig *config)
{
    psFree(config->site);
    psFree(config->files);
    psFree(config->camera);
    psFree(config->cameraName);
    psFree(config->format);
    psFree(config->formatName);
    psFree(config->recipes);
    psFree(config->recipeSymbols);
    psFree(config->arguments);
    psFree(config->database);
}

pmConfig *pmConfigAlloc()
{
    pmConfig *config = psAlloc(sizeof(pmConfig));
    (void)psMemSetDeallocator(config, (psFreeFunc)configFree);

    // Initialise
    config->site = NULL;
    config->camera = NULL;
    config->cameraName = NULL;
    config->format = NULL;
    config->formatName = NULL;
    config->recipes = psMetadataAlloc();
    config->recipesRead = PM_RECIPE_SOURCE_NONE;
    config->recipeSymbols = psMetadataAlloc();
    config->arguments = psMetadataAlloc();
    config->database = NULL;
    config->defaultRecipe = NULL;

    // the file structure is used to carry pmFPAfiles
    config->files = psMetadataAlloc ();
    return config;
}

// Resolve environment variables within a directory name; returns the resolved directory string.
// The returned string is likely a new pointer; the old pointer should be freed by psStringSubstitute.
static psString resolveEnvVar(psString dir // Directory to check for environment variables
                             )
{
    char *envStart;                     // Start of any environment variable
    while ((envStart = strchr(dir, '$'))) {
        char *envName = envStart + 1;   // Start of the environment variable name
        if (envName[0] == '\0') {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Path %s contains a bad environment variable.\n", dir);
            return NULL;
        }
        if (envName[0] == '{') {
            envName++;
            if (envName[0] == '\0') {
                psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                        "Path %s contains a bad environment variable.\n", dir);
                return NULL;
            }
        }
        char *envStop = strpbrk(envStart, "}/"); // End of the environment variable
        ssize_t nameLength = envStop ? envStop - envName : strlen(envName); // Length of the name
        psString name = psStringNCopy(envName, nameLength); // The environment variable name
        char *value = getenv(name);     // Value of the environment variable
        psFree(name);
        psString valueSlash = NULL;    // Value with appended slash
        psStringAppend(&valueSlash, "%s/", value);

        ssize_t envvarLength = envStop ? envStop - envStart : strlen(envStart); // Length, w/o '}'
        psString envvar = psStringNCopy(envStart, envvarLength + 1);  // Environment variable, with $, {, }

        psTrace("psModules.config", 7, "Replacing %s with %s in directory %s\n", envvar, valueSlash, dir);
        psStringSubstitute(&dir, valueSlash, envvar);
        psFree(envvar);
        psFree(valueSlash);
    }

    return dir;
}

void pmConfigSet(const char *path)
{
    PS_ASSERT_STRING_NON_EMPTY(path,);

    pmConfigDone();

    psList *list = psStringSplit(path, ":", false);
    configPath = psListToArray(list);
    // Resolve environment variables
    for (long i = 0; i < configPath->n; i++) {
        configPath->data[i] = resolveEnvVar(configPath->data[i]);
        psTrace("psModules.config", 4, "Path %ld: %s\n", i, (char*)configPath->data[i]);
    }
    psFree(list);
}

void pmConfigDone(void)
{
    psFree(configPath);
    configPath = NULL;
}

bool pmConfigFileRead(psMetadata **config, const char *name, const char *description)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_STRING_NON_EMPTY(name, false);
    PS_ASSERT_STRING_NON_EMPTY(description, false);

    char *realName = NULL;
    unsigned int numBadLines = 0;
    struct stat filestat;

    psTrace("psModules.config", 3, "Loading %s configuration from file %s\n",
            description, name);

    uid_t uid = getuid();
    gid_t gid = getgid();

    // we try: name, path[0]/name, path[1]/name, ...
    // find the first existing entry in the path (starting with the bare name)
    realName = psStringCopy (name);
    psTrace ("psModules.config", 8, "trying %s\n", realName);

    int status = stat (realName, &filestat);
    if (status == 0) {
        if ((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR)) {
            goto found;
        }
        if ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP)) {
            goto found;
        }
        if (filestat.st_mode & S_IROTH) {
            goto found;
        }
    }
    psFree (realName);

    if (configPath == NULL) {
        psError(PS_ERR_IO, true, "Cannot find %s configuration file (%s) in path\n", description, name);
        return false;
    }

    for (int i = 0; i < configPath->n; i++) {
        realName = psStringCopy (configPath->data[i]);
        psStringAppend (&realName, "/%s", name);
        psTrace ("psModules.config", 8, "trying %s\n", realName);

        status = stat (realName, &filestat);
        if (status == 0) {
            if ((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR)) {
                goto found;
            }
            if ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP)) {
                goto found;
            }
            if (filestat.st_mode & S_IROTH) {
                goto found;
            }
        }
        psFree (realName);
    }

    psError(PS_ERR_IO, true, "Cannot find %s configuration file %s in path\n", description, name);
    return false;

found:
    *config = psMetadataConfigRead(NULL, &numBadLines, realName, true);
    if (numBadLines > 0) {
        psError(PS_ERR_IO, false, "%d bad lines in %s configuration file (%s)",
                numBadLines, description, realName);
        psFree (realName);

        return false;
    }
    if (!*config) {
        psError(PS_ERR_IO, true, "Unable to read %s configuration from %s",
                description, realName);
        psFree (realName);
        return false;
    }

    psFree (realName);
    return true;
}

// Read metadata config files in a metadata
// The metadata contains file names, which will be replaced with the metadata that are in the files.
static bool metadataReadFiles(psMetadata *source, // Source metadata
                              const char *description // Description, for error messages
                             )
{
    assert(source);
    psMetadataIterator *iter = psMetadataIteratorAlloc(source, PS_LIST_HEAD, NULL); // Iterator
    psMetadataItem *item;               // Item from iteration
    while ((item = psMetadataGetAndIncrement(iter))) {
        if (item->type == PS_DATA_METADATA) {
            continue;       // We've already read it
        }
        if (item->type != PS_DATA_STRING) {
           psTrace("config", 2, "Element %s in %s metadata is not of type STR.\n",
                   item->name, description);
           continue;
        }

        psTrace("config", 2, "Reading %s %s: %s\n", description, item->name, item->data.str);
        psMetadata *new = NULL;         // New metadata
        if (!pmConfigFileRead(&new, item->data.str, item->name)) {
            psError(PM_ERR_CONFIG, false, "Trouble reading reading %s %s --- "
                    "ignored.\n", description, item->name);
            psFree(new);
            psFree(iter);
            return false;
        }

        // Muck around under the hood to replace the filename with the metadata; don't try this at home, kids
        item->type = PS_DATA_METADATA;
        psFree(item->data.str);
        item->data.md = new;
    }
    psFree(iter);

    return true;
}

// Read the formats for a camera
static bool cameraReadFormats(psMetadata *camera, // Camera for which to read the formats
                              const char *name // Name of the camera, for error messages
                             )
{
    assert(camera);
    assert(name);

    bool mdok;                          // Status of MD lookup
    psMetadata *formats = psMetadataLookupMetadata(&mdok, camera, "FORMATS"); // Formats
    if (!mdok || !formats) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find FORMATS in camera configuration %s.\n", name);
        return false;
    }
    if (!metadataReadFiles(formats, "camera format")) {
        psError(PS_ERR_UNKNOWN, false, "Unable to read formats within camera configuration %s.\n", name);
        return false;
    }

    return true;
}

// Read the calibrations for a camera
static bool cameraReadCalibrations(psMetadata *camera, // Camera for which to read the formats
                                   const char *cameraName // Name of the camera, for error messages
    )
{
    assert(camera);
    assert(cameraName);

    psMetadataItem *darkNorm = psMetadataLookup(camera, "DARK.NORM"); // The dark normalisation calibration
    if (darkNorm) {
        if (darkNorm->type == PS_DATA_STRING) {
            const char *darkNormName = darkNorm->data.str; // The file name
            psTrace("config", 2, "Reading %s dark normalisation: %s\n", cameraName, darkNormName);
            psMetadata *new = NULL;         // New metadata
            if (!pmConfigFileRead(&new, darkNormName, "Dark normalisation")) {
                psError(PM_ERR_CONFIG, false, "Trouble reading reading %s dark normalisation %s --- "
                        "ignored.\n", cameraName, darkNormName);
                psFree(new);
                return false;
            }

            // Muck around under the hood to replace the filename with the metadata;
            // don't try this at home, kids
            darkNorm->type = PS_DATA_METADATA;
            psFree(darkNorm->data.str);
            darkNorm->data.md = new;
        } else if (darkNorm->type != PS_DATA_METADATA) {
            psWarning("DARK.NORM in camera %s is not of type STR or METADATA (%x)", cameraName, darkNorm->type);
        }
    } else {
        // Add a dummy entry
        psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); // Dummy polynomial
        poly->coeff[0] = 0.0;
        poly->coeff[1] = 1.0;
        psMetadata *polyMD = psMetadataAlloc(); // Container for the polynomial
        (void)psPolynomial1DtoMetadata(polyMD, poly, "_DEFAULT"); // Metadata to insert
        psFree(poly);
        psMetadataAddMetadata(camera, PS_LIST_TAIL, "DARK.NORM", 0, "Dark normalisation polynomial",
                              polyMD);
        psMetadataAddStr(camera, PS_LIST_TAIL, "DARK.NORM.KEY", 0, "Key for dark normalisation", "_DEFAULT");
        psFree(polyMD);
    }

    return true;
}

pmConfig *pmConfigRead(int *argc, char **argv, const char *defaultRecipe)
{
    PS_ASSERT_PTR_NON_NULL(argc, NULL);
    PS_ASSERT_INT_POSITIVE(*argc, NULL);
    PS_ASSERT_PTR_NON_NULL(argv, NULL);

    pmConfig *config = pmConfigAlloc(); // The configuration, containing site, camera and recipes
    config->defaultRecipe = defaultRecipe;

    //
    // The following section of code attempts to determine which file is
    // the configuration file.  At the end of this code block, the siteName
    // variable will contain the name of the configuration file.
    //
    char *siteName = NULL;
    //
    // First, try command line
    //
    psS32 argNum = psArgumentGet(*argc, argv, "-site");
    if (argNum != 0) {
        //
        // We remove the "-site" argument from argv.  Then
        // we look for the next argument, which should be the filename, and
        // remove it as well.
        //
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-site command-line switch provided without the required filename --- ignored.\n");
        } else {
            siteName = psStringCopy(argv[argNum]);
            psArgumentRemove(argNum, argc, argv);
        }
    }
    //
    // Next, try environment variable
    //
    if (!siteName) {
        siteName = getenv(PS_SITE);
        if (siteName) {
            siteName = psStringCopy (siteName);
        }
    }

    //
    // Last chance is ~/.ipprc
    //
    if (!siteName) {
        char *home = getenv("HOME");
        siteName = psStringCopy(home);
        psStringAppend(&siteName, "/%s", PS_DEFAULT_SITE);
    }


    // We have the configuration filename; now we read and parse the config
    // file and store in psMetadata struct site.
    //

    if (!pmConfigFileRead(&config->site, siteName, "site")) {
        psFree(config);
        return NULL;
    }

    // Set options based on the site configuration.
    {
        bool mdok = true;   // Status of MD lookup result

        // Initialise the psLib time handling
        psString timeName = psMetadataLookupStr(&mdok, config->site, "TIME");
        if (mdok && timeName)
        {
            psTrace("psModules.config", 7, "Initialising psTime with file %s\n", timeName);
            psTimeInit(timeName);
        }


        // Set logging level
        int logLevel = psMetadataLookupS32(&mdok, config->site, "LOGLEVEL");
        if (mdok && logLevel >= 0)
        {
            psTrace("psModules.config", 7, "Setting log level to %d\n", logLevel);
            psLogSetLevel(logLevel);
        }


        // Set logging format
        psString logFormat = psMetadataLookupStr(&mdok, config->site, "LOGFORMAT");
        if (mdok && logFormat)
        {
            psTrace("psModules.config", 7, "Setting log format to %s\n", logFormat);
            psLogSetFormat(logFormat);
        }

        argNum = psArgumentGet(*argc, argv, "-log");
        if (argNum > 0)
        {
            psArgumentRemove(argNum, argc, argv);
            if (argNum >= *argc) {
                psLogMsg("psModules.config", PS_LOG_WARN, "-log command-line switch provided without the "
                         "required log destination --- ignored.\n");
            } else {
                if (!psLogSetDestination(psMessageDestination(argv[argNum]))) {
                    psLogMsg("psModules.config", PS_LOG_WARN, "Unable to set log destination to %s\n",
                             argv[argNum]);
                }
                psArgumentRemove(argNum, argc, argv);
            }
        } else
        {
            // Set logging destination
            psString logDest = psMetadataLookupStr(&mdok, config->site, "LOGDEST");
            if (mdok && logDest && strlen(logDest) > 0) {
                // XXX: Only stdout and stderr are provided for now; this section should be
                // expanded in the future to do files, and perhaps even sockets.
                if (!psLogSetDestination(psMessageDestination(logDest))) {
                    psLogMsg("psModules.config", PS_LOG_WARN, "Unable to set log destination to %s\n",
                             argv[argNum]);
                }
            }
        }

        // Set trace levels
        psMetadata *trace = psMetadataLookupMetadata(&mdok, config->site, "TRACE");
        if (mdok && trace)
        {
            psMetadataIterator *traceIter = psMetadataIteratorAlloc(trace, PS_LIST_HEAD, NULL); // Iterator
            psMetadataItem *traceItem = NULL; // Item from MD iteration
            while ((traceItem = psMetadataGetAndIncrement(traceIter))) {
                if (traceItem->type != PS_DATA_S32) {
                    psLogMsg("psModules.config", PS_LOG_WARN,
                             "The level for trace component %s is not of type S32 (%x)\n",
                             traceItem->name, traceItem->type);
                    continue;
                }
                psTrace("psModules.config", 7, "Setting trace level for %s to %d\n", traceItem->name, traceItem->data.S32);
                (void)psTraceSetLevel(traceItem->name, traceItem->data.S32);
            }
            psFree(traceIter);
        }

        // Set trace formats
        psString traceFormat = psMetadataLookupStr(&mdok, config->site, "TRACEFORMAT");
        if (mdok && traceFormat)
        {
            psTrace("psModules.config", 7, "Setting trace format to %s\n", traceFormat);
            (void)psTraceSetFormat(traceFormat);
        }

        // Set trace destinations
        #ifndef PS_NO_TRACE
        psString traceDest = psMetadataLookupStr(&mdok, config->site, "TRACEDEST");
        if (mdok && traceDest && strlen(traceDest) > 0)
        {
            psTrace("psModules.config", 7, "Setting trace destination to %s\n", traceDest);
            // XXX: Only stdout and stderr are provided for now; this section should be
            // expanded in the future to do files, and perhaps even sockets.
            if (!psTraceSetDestination(psMessageDestination(traceDest))) {
                psLogMsg("psModules.config", PS_LOG_WARN, "Unable to set trace destination to %s\n", traceDest);

            }
        }
        #endif

        // Allow command line options to override defaults for logging.
        // XXX: Is it appropriate to use the ArgVerbosity function for this?
        //   A: it removes the options from the command line.
        //   B: will the pmConfigRead function always be called on initialization.
        //
        psArgumentVerbosity(argc, argv);
        // XXX: substitute the string for the default log level for "2".
    }

    // define the config-file search path (configPath).  Ensure that
    // it contains the directory where we found the config file in
    // the first place
    if (configPath) {
        pmConfigDone();
    }

    psString siteNameDir = psStringCopy(dirname(siteName));
    psFree(siteName);

    psString path = psMetadataLookupStr(NULL, config->site, "PATH");
    if (path) {
        psStringAppend(&path, ":%s", siteNameDir);
    } else {
        path = psMemIncrRefCounter(siteNameDir);
    }
    pmConfigSet(path);
    psFree(siteNameDir);

    // Next, we do a similar thing for the camera configuration file.  The
    // file is read and parsed into psMetadata struct "camera".
    //
    argNum = psArgumentGet(*argc, argv, "-camera");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psWarning("-camera command-line switch provided without the required camera or filename --- "
                      "ignored.\n");
        } else {
            bool mdok = true;           // Status of MD lookup
            char *cameraName = argv[argNum]; // symbolic name of the camera

            // look for the CAMERAS list in config->site
            psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS");
            if (!cameras) {
                psError(PS_ERR_IO, false, "Unable to find CAMERAS in site configuration.\n");
                psFree(config);
                return NULL;
            }

            // look for the symbolic camera name in the CAMERAS metadata
            char *cameraFile = psMetadataLookupStr(&mdok, cameras, cameraName); // The filename
            if (!cameraFile) {
                psError(PS_ERR_IO, false, "%s is not listed in the site CAMERAS list\n", cameraName);
                psFree(config);
                return NULL;
            }

            // load this camera's configuration informatoin
            if (!pmConfigFileRead(&config->camera, cameraFile, "camera")) {
                psError(PM_ERR_CONFIG, false, "Problem reading %s", cameraName);
                psFree(config);
                return NULL;
            }
            // save the name for future uses
            config->cameraName = psStringCopy (cameraName);

            psArgumentRemove(argNum, argc, argv);

            // Read in the formats
            if (!cameraReadFormats(config->camera, cameraFile)) {
                psError(PS_ERR_UNKNOWN, false, "Unable to read formats within camera configuration %s.\n",
                        cameraFile);
                psFree(config);
                return NULL;
            }

            // Read in any camera-specific calibrations
            if (!cameraReadCalibrations(config->camera, cameraName)) {
                psError(PS_ERR_UNKNOWN, false,
                        "Unable to read calibrations within camera configuration %s.\n",
                        cameraName);
                psFree(config);
                return NULL;
            }

            psMetadataAddMetadata(cameras, PS_LIST_HEAD, cameraName, PS_META_REPLACE,
                                  "Camera specified on command line", config->camera);

            if (!pmConfigCameraSkycellVersion(config->site, cameraName)) {
                psError(PS_ERR_UNKNOWN, false,
                        "Unable to generate skycell versions of specified camera %s.\n",
                        cameraName);
                psFree(config);
                return NULL;
            }

            if (!pmConfigCameraMosaickedVersions(config->site, cameraName)) {
                psError(PS_ERR_UNKNOWN, false,
                        "Unable to generate mosaicked versions of specified camera %s.\n",
                        cameraName);
                psFree(config);
                return NULL;
            }
        }
    }

    // Read the camera configurations, if not already defined, and not turned off
    if (!config->camera && readCameraConfig) {
        bool mdok;                      // Status of MD lookup
        psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS"); // List of cameras
        if (!mdok || !cameras) {
            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find CAMERAS in the site configuration.\n");
            return false;
        }

        if (!metadataReadFiles(cameras, "camera configuration")) {
            psError(PS_ERR_UNKNOWN, false, "Unable to read cameras within site configuration.\n");
            psFree(config);
            return NULL;
        }

        // Now fill in the formats and calibrations
        psMetadataIterator *iter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, NULL); // Iterator
        psMetadataItem *item;           // Item from iteration
        while ((item = psMetadataGetAndIncrement(iter))) {
            assert(item->type == PS_DATA_METADATA);
            if (!cameraReadFormats(item->data.md, item->name)) {
                psWarning("Unable to read formats for camera %s: removed.\n", item->name);
                psMetadataRemoveKey(cameras, item->name);
            }
            if (!cameraReadCalibrations(item->data.md, item->name)) {
                psWarning("Unable to read calibrations for camera %s: removed.\n", item->name);
                psMetadataRemoveKey(cameras, item->name);
            }
        }
        psFree(iter);

        if (!pmConfigCameraSkycellVersionsAll(config->site)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to generate skycell versions of cameras.\n");
            psFree(config);
            return NULL;
        }
        if (!pmConfigCameraMosaickedVersionsAll(config->site)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to generate mosaicked versions of cameras.\n");
            psFree(config);
            return NULL;
        }
    }

    // Load the recipes from the camera file, if appropriate
    if(!pmConfigReadRecipes(config, PM_RECIPE_SOURCE_SITE | PM_RECIPE_SOURCE_CAMERA)) {
        psError(PS_ERR_IO, false, "Failed to read recipes from camera file");
        psFree(config);
        return NULL;
    }

    // load command-line options of the form -recipe NAME RECIPE
    pmConfigLoadRecipeArguments(argc, argv, config);

    // read in command-line options to specific recipe values
    pmConfigLoadRecipeOptions(argc, argv, config, "-D");
    pmConfigLoadRecipeOptions(argc, argv, config, "-Di");
    pmConfigLoadRecipeOptions(argc, argv, config, "-Df");
    pmConfigLoadRecipeOptions(argc, argv, config, "-Db");

    // Look for command-line options for files to replace
    argNum = psArgumentGet(*argc, argv, "-F");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum + 1 >= *argc) {
            psError(PS_ERR_BAD_PARAMETER_SIZE, true,
                    "Filerule switch (-F) provided without old and new filerule.");
            psFree(config);
            return NULL;
        }

        const char *old = argv[argNum]; // The old file, to be replaced
        psArgumentRemove(argNum, argc, argv);
        const char *new = argv[argNum]; // The new file, the replacement
        psArgumentRemove(argNum, argc, argv);

        psMetadata *cameras = psMetadataLookupMetadata(NULL, config->site, "CAMERAS"); // List of cameras
        if (!cameras) {
            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find CAMERAS in the site configuration.\n");
            return false;
        }

        psMetadataIterator *camerasIter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, NULL); // Iterator
        psMetadataItem *cameraItem;     // Item from iteration
        while ((cameraItem = psMetadataGetAndIncrement(camerasIter))) {
            if (cameraItem->type != PS_DATA_METADATA) {
                psWarning("Entry %s in CAMERAS is not of type METADATA --- ignored.", cameraItem->name);
                continue;
            }
            psMetadata *camera = cameraItem->data.md; // Camera configuration
            psMetadata *filerules = psMetadataLookupMetadata(NULL, camera, "FILERULES"); // File rules
            if (!filerules) {
                psWarning("Can't find FILERULES of type METADATA in camera %s --- ignored.",
                          cameraItem->name);
                continue;
            }
            psMetadataItem *newItem = psMetadataLookup(filerules, new); // The rule of interest, or a redir.
            if (!newItem) {
                psWarning("Can't find file rule %s in camera %s --- ignored.", new, cameraItem->name);
                continue;
            }
            psMetadata *newRule;        // New rule (replacement)
            switch (newItem->type) {
              case PS_DATA_STRING:
                // A redirection
                newRule = psMetadataLookupMetadata(NULL, filerules, newItem->data.str);
                break;
              case PS_DATA_METADATA:
                // The rule itself
                newRule = newItem->data.md;
                break;
              default:
                psWarning("Unable to find filerule %s of the correct type in camera %s --- ignored.",
                          new, cameraItem->name);
                continue;
            }
            psMetadataAddMetadata(filerules, PS_LIST_TAIL, old, PS_META_REPLACE,
                                  "Original replaced by -F option", newRule);
        }
        psFree(camerasIter);
    }

    // check for values that override DB* keywords
    argNum = psArgumentGet(*argc, argv, "-dbserver");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-dbserver command-line switch provided without the required server name --- ");
        } else {
            char *dbserver = argv[argNum]; // The camera configuration file to read
            if (!psMetadataAddStr(config->site, PS_LIST_TAIL, "DBSERVER", PS_META_REPLACE, NULL, dbserver)) {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "failed to overwrite .ipprc DBSERVER value --- ");
            }

            psArgumentRemove(argNum, argc, argv);
        }
    }

    argNum = psArgumentGet(*argc, argv, "-dbname");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-dbname command-line switch provided without the required database name --- ");
        } else {
            char *dbname = argv[argNum]; // The camera configuration file to read
            if (!psMetadataAddStr(config->site, PS_LIST_TAIL, "DBNAME", PS_META_REPLACE, NULL, dbname)) {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "failed to overwrite .ipprc DBNAME value --- ");
            }

            psArgumentRemove(argNum, argc, argv);
        }
    }

    argNum = psArgumentGet(*argc, argv, "-dbuser");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-dbuser command-line switch provided without the required database name --- ");
        } else {
            char *dbuser = argv[argNum]; // The camera configuration file to read
            if (!psMetadataAddStr(config->site, PS_LIST_TAIL, "DBUSER", PS_META_REPLACE, NULL, dbuser)) {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "failed to overwrite .ipprc DBUSER value --- ");
            }

            psArgumentRemove(argNum, argc, argv);
        }
    }

    argNum = psArgumentGet(*argc, argv, "-dbpassword");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-dbpassword command-line switch provided without the required password --- ");
        } else {
            char *dbpassword = argv[argNum]; // The camera configuration file to read
            if (!psMetadataAddStr(config->site, PS_LIST_TAIL, "DBPASSWORD", PS_META_REPLACE, NULL, dbpassword)) {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "failed to overwrite .ipprc DBPASSWORD value --- ");
            }

            psArgumentRemove(argNum, argc, argv);
        }
    }

    argNum = psArgumentGet(*argc, argv, "-dbport");
    if (argNum > 0) {
        psArgumentRemove(argNum, argc, argv);
        if (argNum >= *argc) {
            psLogMsg("psModules.config", PS_LOG_WARN,
                     "-dbpport command-line switch provided without the required port number --- ");
        } else {
            char *dbport = argv[argNum]; // The camera configuration file to read
            if (!psMetadataAddS32(config->site, PS_LIST_TAIL, "DBPORT", PS_META_REPLACE, NULL, (psS32)atoi(dbport))) {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "failed to overwrite .ipprc DBPORT value --- ");
            }

            psArgumentRemove(argNum, argc, argv);
        }
    }


    psErrorClear();   // we may have failed to find some items in the metadata

    return config;
}


// XXX aren't these (return false) below errors: call psError?
bool pmConfigValidateCameraFormat(bool *valid, const psMetadata *cameraFormat, const psMetadata *header)
{
    PS_ASSERT_PTR_NON_NULL(cameraFormat, false);
    PS_ASSERT_PTR_NON_NULL(header, false);

    // Read the rule for that camera format
    bool mdStatus = true;

    psMetadata *rule = psMetadataLookupMetadata(&mdStatus, cameraFormat, "RULE");
    if (! mdStatus || ! rule) {
        psError(PS_ERR_UNKNOWN, false, "Unable to read rule for camera.");
        *valid = false;
        return false;
    }

    // Apply the rules
    psMetadataIterator *ruleIter = psMetadataIteratorAlloc(rule, PS_LIST_HEAD, NULL); // Rule iterator
    psMetadataItem *ruleItem = NULL;    // Item from the metadata

    *valid = true;
    while (*valid && (ruleItem = psMetadataGetAndIncrement(ruleIter))) {
        // Check for the existence of the rule
        psMetadataItem *headerItem = psMetadataLookup(header, ruleItem->name);
        if (! headerItem) {
            // rule item not found in header
            psTrace("psModules.config", 5, "Can't find %s", ruleItem->name);
            *valid = false;
        }

        // Check to see if the rule works
        if (headerItem && !psMetadataItemCompare(headerItem, ruleItem)) {
            psTrace("psModules.config", 5, "%s doesn't match.", ruleItem->name);
            *valid = false;
        }
    } // Iterating through the RULEs

    psFree(ruleIter);
    return true;
}

// Given a camera and a header, see if any of the camera formats match the header
static bool formatFromHeader(psMetadata **format, // Format to return
                             psString *name, // Name to return
                             psMetadata *camera, // Camera configuration
                             const psMetadata *header, // FITS header
                             const char *cameraName // Name of camera
                            )
{
    assert(format);
    assert(camera);
    assert(header);
    assert(cameraName);
    assert(*cameraName);

    bool result = false;                // Did we find the first match?

    // Read the list of formats
    bool mdok = true;                   // Status of MD lookup
    psMetadata *formats = psMetadataLookupMetadata(&mdok, camera, "FORMATS"); // List of formats
    if (!mdok || !formats) {
        psError(PS_ERR_UNKNOWN, false, "Unable to find list of FORMATS in camera %s", cameraName);
        return false;
    }

    if (!metadataReadFiles(formats, "camera format")) {
        psError(PS_ERR_UNKNOWN, false, "Unable to read cameras formats within camera configuration.\n");
        return false;
    }

    // Iterate over the formats
    psMetadataIterator *formatsIter = psMetadataIteratorAlloc(formats, PS_LIST_HEAD, NULL);
    psMetadataItem *formatsItem = NULL; // Item from formats
    while ((formatsItem = psMetadataGetAndIncrement(formatsIter))) {
        assert(formatsItem->type == PS_DATA_METADATA); // Since we have just read it in or deleted it
        psMetadata *testFormat = formatsItem->data.md; // Format to test against

        psTrace("psModules.config", 5, "trying format %s", formatsItem->name);

        bool valid = false;
        if (!pmConfigValidateCameraFormat(&valid, testFormat, header)) {
            psError (PS_ERR_UNKNOWN, false, "Error in config scripts for camera %s, format %s\n",
                     cameraName, formatsItem->name);
            return false;
        }
        if (valid) {
            if (!*format) {
                psLogMsg("psModules.config", PS_LOG_INFO, "Camera %s, format %s matches header.\n",
                         cameraName, formatsItem->name);
                *format = psMemIncrRefCounter(testFormat);
                *name = psStringCopy(formatsItem->name);
                result = true;
            } else {
                psLogMsg("psModules.config", PS_LOG_WARN,
                         "Camera %s, format %s also matches header --- ignored.\n",
                         cameraName, formatsItem->name);
            }
        }
    }
    psFree(formatsIter);
    return result;
}

// we only need to read the recipe for the main camera images.  for additional images (eg, sky
// cells) do not need to re-read the recipe files!
psMetadata *pmConfigCameraFormatFromHeader(pmConfig *config, const psMetadata *header, bool readRecipes)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(header, NULL);

    psMetadata *format = NULL;          // The winning format
    psString name = NULL;               // Name of the winning format

    // If we don't know what sort of camera we have, we try all that we know
    if (! config->camera) {
        bool mdok;                      // Metadata lookup status
        psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS");
        if (! mdok || !cameras) {
            psError(PS_ERR_IO, true, "Unable to find CAMERAS in the configuration.");
            return NULL;
        }

        if (!metadataReadFiles(cameras, "camera configuration")) {
            psError(PS_ERR_UNKNOWN, false, "Unable to read cameras within site configuration.\n");
            return NULL;
        }

        // Iterate over the cameras
        psMetadataIterator *camerasIter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, NULL);
        psMetadataItem *camerasItem = NULL; // Item from the metadata
        while ((camerasItem = psMetadataGetAndIncrement(camerasIter))) {
            // Open the camera information
            psTrace("psModules.config", 3, "Inspecting camera %s (%s)\n", camerasItem->name,
                    camerasItem->comment);
            assert(camerasItem->type == PS_DATA_METADATA); // It should be because we've read it in or deleted
            psMetadata *testCamera = camerasItem->data.md; // Camera to test against what we've got:
            if (formatFromHeader(&format, &name, testCamera, header, camerasItem->name)) {
                // Set the defaults, if they are not already set
                if (!config->camera && !config->cameraName) {
                    config->camera = psMemIncrRefCounter(testCamera);
                    config->cameraName = psStringCopy(camerasItem->name);
                }
                if (!config->format && !config->formatName) {
                    config->formatName = name;
                    config->format = format;
                } else {
                    psFree(name);
                    psFree(format);
                }
            } else {
                psErr *error = psErrorLast();
                if (error->code != PS_ERR_NONE) {
                    psError(PS_ERR_IO, false, "Error reading camera config data for %s", camerasItem->name);
                    return NULL;
                }
                psFree(error);
            }
        } // Done looking at all cameras
        psFree(camerasIter);

        if (! config->camera) {
            psError(PS_ERR_IO, false, "Unable to find a camera that matches input FITS header!");
            return NULL;
        }

        // Now we have the camera, we can read the recipes
        if (readRecipes && !pmConfigReadRecipes(config, PM_RECIPE_SOURCE_CAMERA | PM_RECIPE_SOURCE_CL)) {
            psError(PS_ERR_IO, false, "Error reading recipes from camera config for %s", config->cameraName);
            return NULL;
        }
        return psMemIncrRefCounter(format);
    }

    // Otherwise, try the specific camera
    if (!formatFromHeader(&format, &name, config->camera, header, config->cameraName)) {
        psError(PS_ERR_IO, true, "Unable to find a format with the specified camera (%s) that matches the "
                "given header.\n", config->cameraName);
        return NULL;
    }

    if (!config->format && !config->formatName) {
        config->formatName = name;
        config->format = format;
    } else {
        psFree(format);
        psFree(name);
    }
    return psMemIncrRefCounter(format);
}

// Return the requested camera configuration
psMetadata *pmConfigCameraByName(pmConfig *config, const char *cameraName)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(cameraName, NULL);

    psMetadata *cameras = psMetadataLookupMetadata(NULL, config->site, "CAMERAS");
    if (!cameras) {
        psError(PS_ERR_IO, true, "Unable to find CAMERAS in the configuration.");
        return NULL;
    }

    char *cameraPath = psMetadataLookupStr(NULL, cameras, cameraName);
    if (!cameraPath) {
        psError(PS_ERR_IO, true, "Unable to find requested CAMERA in the configuration.");
        return NULL;
    }

    psMetadata *camera = NULL; // Camera to test against what we've got:

    if (!pmConfigFileRead(&camera, cameraPath, cameraName)) {
        psLogMsg("psModules.config", PS_LOG_WARN, "Trouble reading reading camera configuration %s", cameraName);
        psFree(camera);
        return NULL;
    }

    return camera;
}

psDB *pmConfigDB(pmConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->site, NULL);

    #ifndef HAVE_PSDB

    psError(PS_ERR_UNKNOWN, false,
            "Cannot configure database: psModules was compiled without database support.");
    return NULL;

    #else

    psBool mdStatus01 = false;
    psBool mdStatus02 = false;
    psBool mdStatus03 = false;
    psBool mdStatus04 = false;
    psBool mdStatus05 = false;

    // XXX leaky strings
    psString dbServer = psMetadataLookupStr(&mdStatus01, config->site, "DBSERVER");
    psString dbUsername = psMetadataLookupStr(&mdStatus02, config->site, "DBUSER");
    psString dbPassword = psMetadataLookupStr(&mdStatus03, config->site, "DBPASSWORD");
    psString dbName = psMetadataLookupStr(&mdStatus04, config->site, "DBNAME");
    psS32 dbPort = psMetadataLookupS32(&mdStatus05, config->site, "DBPORT");
    if (!mdStatus05) {
        dbPort = 0;
    }
    if (!(mdStatus01 && mdStatus02 && mdStatus03 && mdStatus04)) {
        psLogMsg("psModules.config", PS_LOG_WARN,
                 "Could not determine database server, name, user, and password from site metadata.\n");
        return NULL;
    }

    return psDBInit(dbServer, dbUsername, dbPassword, dbName, dbPort);
    #endif
}


bool pmConfigConformHeader(psMetadata *header, const psMetadata *format)
{
    PS_ASSERT_PTR_NON_NULL(header, false);
    PS_ASSERT_PTR_NON_NULL(format, false);

    bool mdok = true;                   // Status of MD lookup
    psMetadata *rules = psMetadataLookupMetadata(&mdok, format, "RULE"); // How to identify this format
    if (!mdok || !rules) {
        psError(PS_ERR_IO, true, "Unable to find RULE in camera format.\n");
        return false;
    }

    psMetadataIterator *rulesIter = psMetadataIteratorAlloc(rules, PS_LIST_HEAD, NULL); // Iterator for rules
    psMetadataItem *rulesItem = NULL;   // Item from iteration
    while ((rulesItem = psMetadataGetAndIncrement(rulesIter))) {
        psMetadataItem *newItem = psMetadataItemCopy(rulesItem); // Copy of item
        psMetadataAddItem(header, newItem, PS_LIST_TAIL, PS_META_REPLACE);
        psFree(newItem);                // Drop reference
    }
    psFree(rulesIter);

    // XXX do we need to update the header based on elements of the FILE?
    # if (0)
    psMetadata *file = psMetadataLookupMetadata(&mdok, format, "FILE"); // How to identify this format
    if (!mdok || !file) {
        psError(PS_ERR_IO, true, "Unable to find FILE in camera format.\n");
        return false;
    }
    psString fpaName = psMetadataLookupStr(&mdok, file, "FPA.NAME");
    # endif

    return true;
}

psArray *pmConfigFileSets(int *argc, char **argv, const char *file, const char *list)
{
    PS_ASSERT_PTR_NON_NULL(argc, NULL);
    PS_ASSERT_INT_NONNEGATIVE(*argc, NULL);
    PS_ASSERT_PTR_NON_NULL(argv, NULL);

    int Narg;

    // we load all input files onto a psArray, to be parsed later
    psArray *input = psArrayAllocEmpty(16);

    // load the list of filenames the supplied file
    // maybe a comma-separated list of words
    // each word may be a glob: "file*.fits"
    if (file && strlen(file) > 0 && (Narg = psArgumentGet (*argc, argv, file))) {

        // select the word after 'file' and split by comma
        psArgumentRemove (Narg, argc, argv);
        psArray *words = psStringSplitArray (argv[Narg], ",", true);
        psArgumentRemove (Narg, argc, argv);

        // parse the word as a glob
        glob_t globList;
        for (int i = 0; i < words->n; i++) {
            globList.gl_offs = 0;
            glob (words->data[i], 0, NULL, &globList);

            // if the glob does not match, save the literal word:
            // otherwise save all glob matches
            if (globList.gl_pathc == 0) {
                psArrayAdd (input, 16, words->data[i]);
            } else {
                for (int j = 0; j < globList.gl_pathc; j++) {
                    char *filename = psStringCopy (globList.gl_pathv[j]);
                    psArrayAdd (input, 16, filename);
                    psFree (filename);
                }
            }
        }
        psFree (words);
    }

    // load the list from the supplied text file
    if (list && strlen(list) > 0 && (Narg = psArgumentGet (*argc, argv, list))) {
        int nItems;
        char line[1024]; // XXX limits the list lines to 1024 chars
        char word[1024];
        char *filename;

        psArgumentRemove (Narg, argc, argv);
        FILE *f = fopen (argv[Narg], "r");
        if (f == NULL) {
            psAbort("unable to open specified list file");
        }
        while (fgets (line, 1024, f) != NULL) {
            nItems = sscanf (line, "%s", word);
            switch (nItems) {
            case 0:
                break;
            case 1:
                filename = psStringCopy (word);
                psArrayAdd (input, 16, filename);
                psFree (filename);
                break;
            default:
                // rigid format, no comments allowed?
                psAbort("error parsing input list file");
                break;
            }
        }
        psArgumentRemove (Narg, argc, argv);
    }

    return input;
}

bool pmConfigFileSetsMD(psMetadata *metadata, int *argc, char **argv, const char *name,
                        const char *file, const char *list)
{
    PS_ASSERT_PTR_NON_NULL(metadata, false);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    psErrorClear();   // pmConfigFileSets may or may not call psError, so
    // if files->n == 0 we'll want to call psError(..., false, ...)
    psArray *files = pmConfigFileSets(argc, argv, file, list);
    if (files->n == 0) {
        // psError(PS_ERR_IO, false, "pmConfigFileSets failed to find %s in the argument list", name);
        psFree (files);
        return false;
    }

    psMetadataAddPtr(metadata, PS_LIST_TAIL, name,  PS_DATA_ARRAY, "", files);
    psFree (files);
    return true;
}

// convert the supplied name, create a new output psString
psString pmConfigConvertFilename(const char *filename, const pmConfig *config, bool create)
{
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    // strip file:// from front of name
    if (!strncasecmp(filename, "file:", strlen("file:"))) {
        psString newName = psStringCopy(filename);

        char *point = newName + strlen("file:");
        while (*point == '/')
            point ++;
        char *tmpName = NULL;
        psStringAppend (&tmpName, "/%s", point);
        psFree (newName);
        newName = tmpName;

        if (!create && access(newName, R_OK) != 0) {
            psError(PS_ERR_IO, true, "Unable to access file %s", newName);
            psFree(newName);
            return NULL;
        }

        return newName;
    }

    // replace path://PATH with matched datapath
    if (!strncasecmp(filename, "path://", strlen("path://"))) {
        PS_ASSERT_METADATA_NON_NULL(config->site, NULL);

        psString newName = psStringCopy(filename);

        // filename should be of the form: path://PATH/rest/of/file
        // replace PATH with matching name from config->site:DATAPATH
        psMetadata *datapath = psMetadataLookupPtr (NULL, config->site, "DATAPATH");
        if (datapath == NULL) {
            psError(PS_ERR_UNKNOWN, true, "DATAPATH is not defined in config.site");
            psFree (newName);
            return NULL;
        }

        char *point = newName + strlen("path://");
        char *mark = strchr (point, '/');
        if (mark == NULL) {
            psError(PS_ERR_UNKNOWN, true, "syntax error in PATH-style name %s", newName);
            psFree (newName);
            return false;
        }

        psString path = psStringNCopy (point, mark - point);
        char *realpath = psMetadataLookupStr (NULL, datapath, path);
        if (realpath == NULL) {
            psError(PS_ERR_UNKNOWN, true,
                    "path (%s) not defined in config.site:DATAPATH for PATH-style name %s",
                    path, newName);
            psFree(newName);
            psFree(path);
            return false;
        }
        psFree(path);

        char *tmpName = NULL;
        psStringAppend(&tmpName, "%s/%s", realpath, mark + 1);
        psFree(newName);
        newName = tmpName;

        if (!create && access(newName, R_OK) != 0) {
            psError(PS_ERR_IO, true, "Unable to access file %s", newName);
            psFree(newName);
            return NULL;
        }
        return newName;
    }

    // substitute neb://name with matched nebulous name
    if (!strncasecmp(filename, "neb://", strlen("neb://"))) {
        #ifdef HAVE_NEBCLIENT

        bool status = false;
        psString neb_server = NULL;

        // check the env first
        neb_server = getenv("NEB_SERVER");

        // if env isn't set, check the config system
        if (!neb_server) {
            neb_server = psMetadataLookupStr(&status, config->site, "NEB_SERVER");
            if (!status) {
                psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER.");
                return NULL;
            }
        }

        if (!neb_server) {
            psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI.");
            return NULL;
        }

        nebServer *server = nebServerAlloc(neb_server);
        if (!server) {
            psError(PM_ERR_SYS, true, "failed to create a nebServer object: %s", nebErr(server));
            nebServerFree(server);
            return NULL;
        }

        // ignore the leading neb://
        char *cleanFilename = (char *)filename + strlen("neb://");

        char *nebfile = NULL;
        if (create) {
            nebfile = nebCreate(server, cleanFilename, NULL, NULL);
            if (!nebfile) {
                psError(PM_ERR_SYS, true, "failed to create a new nebulous key: %s", nebErr(server));
                nebServerFree(server);
                return NULL;
            }
        } else {
            nebfile = nebFind(server, cleanFilename);
            if (!nebfile) {
                psError(PM_ERR_SYS, true, "failed to find nebulous key: %s", nebErr(server));
                nebServerFree(server);
                return NULL;
            }
        }

        // convert nebfile into a psString
        psString path = psStringCopy(nebfile);
        nebFree(nebfile);
        nebServerFree(server);

        return path;

        #else // ifdef HAVE_NEBCLIENT

        psError(PM_ERR_PROG, true, "psModules was compiled without nebulous support.");
        return NULL;
        #endif // ifdef HAVE_NEBCLIENT

    }

    // if we go this far, do nothing
    return psStringCopy(filename);
}
