/** @file  psMetadata.c
 *
 *
 *  @brief Contains metadata structures, enumerations and functions prototypes.
 *
 *  This file defines metadata item, metadata type, metadata flags, metadata containers, and function
 *  prototypes necessary creating psLib metadata APIs
 *
 *  @ingroup Metadata
 *
 *  @author Robert DeSonia, MHPCC
 *  @author Ross Harman, MHPCC
 *
 *  @version $Revision: 1.87 $ $Name: not supported by cvs2svn $
 *  @date $Date: 2005-09-30 01:34:45 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
 */

/******************************************************************************/
/*  INCLUDE FILES                                                             */
/******************************************************************************/
#include<stdio.h>
#include<stdarg.h>
#include<string.h>

#include "fitsio.h"
#include "psType.h"
#include "psMemory.h"
#include "psError.h"
#include "psAbort.h"
#include "psList.h"
#include "psHash.h"
#include "psVector.h"
#include "psMetadata.h"
#include "psLookupTable.h"
#include "psString.h"
#include "psErrorText.h"
#include "psConstants.h"
#include "psLogMsg.h"
#include "psTrace.h"

/******************************************************************************/
/*  DEFINE STATEMENTS                                                         */
/******************************************************************************/

/** Maximum size of a string */
#define MAX_STRING_LENGTH 1024

/******************************************************************************/
/*  TYPE DEFINITIONS                                                          */
/******************************************************************************/

// None

/*****************************************************************************/
/*  GLOBAL VARIABLES                                                         */
/*****************************************************************************/

// None

/*****************************************************************************/
/*  FILE STATIC VARIABLES                                                    */
/*****************************************************************************/

static psS32 metadataId = 0;

/*****************************************************************************/
/*  FUNCTION IMPLEMENTATION - LOCAL                                          */
/*****************************************************************************/

static psMetadataItem* makeMetaMulti(psHash* table,
                                     const char* key,
                                     psMetadataItem* existing)
{

    if (existing != NULL && existing->type == PS_DATA_METADATA_MULTI) {
        return existing;
    }

    // move any existing entry into a psList
    psList* newList = psListAlloc(existing);

    psMetadataItem* item = psMetadataItemAlloc(key,
                           PS_DATA_METADATA_MULTI,
                           "",
                           newList);

    if (existing != NULL) {
        psHashRemove(table,key); // take out the old entry
    }

    psHashAdd(table, key, item); // put in the new entry
    psMemDecrRefCounter(item); // get rid of extra reference

    // free local references of newly allocated items.
    psFree(newList);

    return item;
}

static void metadataItemFree(psMetadataItem* metadataItem)
{
    psDataType type;

    type = metadataItem->type;

    if (metadataItem == NULL) {
        return;
    }

    psMemDecrRefCounter(metadataItem->name);
    psMemDecrRefCounter(metadataItem->comment);

    if(!PS_DATA_IS_PRIMITIVE(type)) {
        psFree(metadataItem->data.V);
    }
}

static void metadataIteratorFree(psMetadataIterator* iter)
{
    if (iter == NULL) {
        return;
    }
    psFree(iter->iter);

    if (iter->regex != NULL) {
        regfree(iter->regex);
    }
}

static void metadataFree(psMetadata* metadata)
{
    if (metadata == NULL) {
        return;
    }
    psFree(metadata->list);
    psFree(metadata->hash);

}

/*****************************************************************************/
/* FUNCTION IMPLEMENTATION - PUBLIC                                          */
/*****************************************************************************/

psMetadataItem* psMetadataItemAlloc(const char *name,
                                    psDataType type,
                                    const char *comment,
                                    ...)
{
    va_list argPtr;
    psMetadataItem* metadataItem = NULL;

    // Get the variable list parameters to pass to allocation function
    va_start(argPtr, comment);

    // Call metadata item allocation
    metadataItem = psMetadataItemAllocV(name, type, comment, argPtr);

    // Clean up stack after variable arguement has been used
    va_end(argPtr);

    return metadataItem;
}

bool psMemCheckMetadataItem(psPtr ptr)
{
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataItemFree );
}


#define METADATAITEM_ALLOC_TYPE(NAME,TYPE,METATYPE) \
psMetadataItem* psMetadataItemAlloc##NAME(const char* name, \
        const char* comment, \
        TYPE value) \
{ \
    return psMetadataItemAlloc(name, METATYPE, comment, value); \
}

METADATAITEM_ALLOC_TYPE(Str,const char*,PS_DATA_STRING)
METADATAITEM_ALLOC_TYPE(F32,psF32,PS_DATA_F32)
METADATAITEM_ALLOC_TYPE(F64,psF64,PS_DATA_F64)
METADATAITEM_ALLOC_TYPE(S32,psS32,PS_DATA_S32)
METADATAITEM_ALLOC_TYPE(Bool,psBool,PS_DATA_BOOL)

psMetadataItem* psMetadataItemAllocPtr(const char *name,
                                       psDataType type,
                                       const char *comment,
                                       psPtr value)
{
    return (psMetadataItemAlloc(name, type, comment, value));
}


psMetadataItem* psMetadataItemAllocV(const char *name,
                                     psDataType type,
                                     const char *comment,
                                     va_list argPtr)
{
    psMetadataItem* metadataItem = NULL;


    PS_ASSERT_PTR_NON_NULL(name,NULL);

    // Allocate metadata item
    metadataItem = (psMetadataItem*) psAlloc(sizeof(psMetadataItem));
    metadataItem->data.V = NULL;

    // Set deallocator
    psMemSetDeallocator(metadataItem, (psFreeFunc) metadataItemFree);

    // Allocate and set metadata item comment
    if (comment == NULL) {
        // Per SDRS, null isn't allowed, must use "" instead
        metadataItem->comment = psStringCopy("");
    } else {
        metadataItem->comment = psStringCopy(comment);
    }

    // Set metadata item unique id
    *(psS32 *)(&metadataItem->id) = ++metadataId;

    // Set metadata item type
    metadataItem->type = type & PS_METADATA_TYPE_MASK;

    // Allocate and set metadata item name
    metadataItem->name = (char *)psAlloc(sizeof(char) * MAX_STRING_LENGTH);
    vsnprintf(metadataItem->name, MAX_STRING_LENGTH, name, argPtr);

    // Set metadata item value
    switch(metadataItem->type) {
    case PS_DATA_BOOL:
        metadataItem->data.B = (psBool)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S32:
        metadataItem->data.S32 = (psS32)va_arg(argPtr, psS32);
        break;
    case PS_DATA_F32:
        metadataItem->data.F32 = (psF32)va_arg(argPtr, psF64);
        break;
    case PS_DATA_F64:
        metadataItem->data.F64 = (psF64)va_arg(argPtr, psF64);
        break;
    case PS_DATA_STRING:
        // Perform copy of input strings
        metadataItem->data.V = psStringNCopy(va_arg(argPtr, char *), MAX_STRING_LENGTH);
        break;
    case     PS_DATA_ARRAY:                     ///< psArray
    case     PS_DATA_BITSET:                    ///< psBitSet
    case     PS_DATA_CELL:                      ///< psCell
    case     PS_DATA_CHIP:                      ///< psChip
    case     PS_DATA_CUBE:                      ///< psCube
    case     PS_DATA_FITS:                      ///< psFits
    case     PS_DATA_HASH:                      ///< psHash
    case     PS_DATA_HISTOGRAM:                 ///< psHistogram
    case     PS_DATA_IMAGE:                     ///< psImage
    case     PS_DATA_KERNEL:                    ///< psKernel
    case     PS_DATA_LIST:                      ///< psList
    case     PS_DATA_LOOKUPTABLE:               ///< psLookupTable
    case     PS_DATA_METADATA:                  ///< psMetadata
    case     PS_DATA_METADATAITEM:              ///< psMetadataItem
    case     PS_DATA_MINIMIZATION:              ///< psMinimization
    case     PS_DATA_PIXELS:                    ///< psPixels
    case     PS_DATA_PLANE:                     ///< psPlane
    case     PS_DATA_PLANEDISTORT:              ///< psPlaneDistort
    case     PS_DATA_PLANETRANSFORM:            ///< psPlaneTransform
    case     PS_DATA_POLYNOMIAL1D:              ///< psPolynomial1D
    case     PS_DATA_POLYNOMIAL2D:              ///< psPolynomial2D
    case     PS_DATA_POLYNOMIAL3D:              ///< psPolynomial3D
    case     PS_DATA_POLYNOMIAL4D:              ///< psPolynomial4D
    case     PS_DATA_PROJECTION:                ///< psProjection
    case     PS_DATA_READOUT:                   ///< psReadout
    case     PS_DATA_REGION:                    ///< psRegion
    case     PS_DATA_SCALAR:                    ///< psScalar
    case     PS_DATA_SPHERE:                    ///< psSphere
    case     PS_DATA_SPHEREROT:                 ///< psSphereTransform
    case     PS_DATA_SPLINE1D:                  ///< psSpline1D
    case     PS_DATA_STATS:                     ///< psStats
    case     PS_DATA_TIME:                      ///< psTime
    case     PS_DATA_VECTOR:                    ///< psVector
    case     PS_DATA_METADATA_MULTI:
        // Copy of input data not performed due to variability of data types
        metadataItem->data.V = psMemIncrRefCounter(va_arg(argPtr, psPtr));
        break;
    default:
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_METATYPE_INVALID, type);
        psFree(metadataItem);
        metadataItem = NULL;
    }

    return metadataItem;
}

psMetadata* psMetadataAlloc(void)
{
    psList* list = NULL;
    psHash* hash = NULL;
    psMetadata* metadata = NULL;

    // Allocate metadata
    metadata = (psMetadata*) psAlloc(sizeof(psMetadata));
    // Set deallocator
    psMemSetDeallocator(metadata, (psFreeFunc) metadataFree);

    // Allocate metadata's internal containers
    list = (psList*) psListAlloc(NULL);
    hash = (psHash*) psHashAlloc(10);

    metadata->list = list;
    metadata->hash = hash;

    return metadata;
}


bool psMemCheckMetadata(psPtr ptr)
{
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataFree );
}



bool psMetadataAddItem(psMetadata *md,
                       const psMetadataItem *item,
                       int location,
                       psS32 flags)
{
    char * key = NULL;
    psHash *mdTable = NULL;
    psList *mdList = NULL;
    psMetadataItem *existingEntry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    PS_ASSERT_PTR_NON_NULL(item,NULL);
    PS_ASSERT_PTR_NON_NULL(item->name,NULL);

    mdTable = md->hash;
    mdList = md->list;
    key = item->name;

    // See if key is already in table
    existingEntry = (psMetadataItem*)psHashLookup(mdTable, key);

    if (item->type == PS_DATA_METADATA_MULTI) {
        // the incoming entry is PS_DATA_METADATA_MULTI

        // force the hash entry to be PS_DATA_METADATA_MULTI
        existingEntry = makeMetaMulti(mdTable,key,existingEntry);

        // add all the items in the incoming entry to metadata
        psList* list = item->data.list;
        if (list != NULL) {
            psListIterator* iter = psListIteratorAlloc(list,PS_LIST_HEAD,true);
            psMetadataItem* listItem;
            while ((listItem=(psMetadataItem*)psListGetAndIncrement(iter)) != NULL) {
                psMetadataAddItem(md,listItem,location,flags);
            }
            psFree(iter);
        }

        return true; // all done.
    }

    // how the item is added to the hash depends on prior existence, flags, etc.
    if(existingEntry != NULL) { // prior existence
        if (existingEntry->type == PS_DATA_METADATA_MULTI || (flags & PS_META_DUPLICATE_OK) != 0) {
            // duplicate entries allowed - add another entry.

            // make sure the existing entry is PS_DATA_METADATA_MULTI
            existingEntry = makeMetaMulti(mdTable,key,existingEntry);

            // add to the hash's list of duplicate entries
            if (! psListAdd(existingEntry->data.list, PS_LIST_TAIL, (psPtr)item) ) {
                psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_COLLECTION_FAILED,key);
                return false;
            }
        } else if ((flags & PS_META_REPLACE) != 0) {
            // replace entry instead of creating a duplicate entry.

            // remove the existing entry from metadata
            psMetadataRemove(md,0,key);

            // treat as if new (added to list below)
            if(!psHashAdd(mdTable, key, (psPtr)item)) {
                psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_TABLE_FAILED,key);
                return false;
            }

            // Generate warning that item has been replaced
            //            psLogMsg(__func__,PS_LOG_WARN,"Metadata item %s has been replaced",
            //                     existingEntry->name);
        } else {
            // default is to error on duplicate entry.
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    PS_ERRORTEXT_psMetadata_DUPLICATE_NOT_ALLOWED);
            return false;
        }
    } else {
        // OK, this is a new item.

        // Node doesn't exist - Add new metadata item to metadata collection's hash
        if(!psHashAdd(mdTable, key, (psPtr)item)) {
            psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_TABLE_FAILED,key);
            return false;
        }
    }

    if(!psListAdd(mdList, location, (psPtr)item)) {
        psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_COLLECTION_FAILED,key);
        return false;
    }

    return true;
}

bool psMetadataAdd(psMetadata *md,
                   long location,
                   const char *name,
                   int format,
                   const char *comment,
                   ...)
{
    va_list argPtr;

    va_start(argPtr, comment);
    psBool result = psMetadataAddV(md,location,name,format,comment,argPtr);
    va_end(argPtr);

    return result;
}

bool psMetadataAddV(psMetadata *md,
                    long location,
                    const char *name,
                    int format,
                    const char *comment,
                    va_list list)
{
    psMetadataItem* metadataItem = NULL;

    metadataItem = psMetadataItemAllocV(name, format & PS_METADATA_TYPE_MASK, comment, list);

    if (!psMetadataAddItem(md, metadataItem, location, format & PS_METADATA_FLAGS_MASK)) {
        psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_FAILED);
        psFree(metadataItem);
        return false;
    }
    // Decrement reference count, since the metadata item is now in metadata collection and no longer needed
    psFree(metadataItem);

    return true;
}

#define METADATA_ADD_TYPE(NAME,TYPE,METATYPE) \
psBool psMetadataAdd##NAME(psMetadata* md, long where, const char* name, \
                           int format, const char* comment, TYPE value) { \
    return psMetadataAdd(md,where,name, format | METATYPE,comment,value); \
}

METADATA_ADD_TYPE(Bool,psBool,PS_DATA_BOOL)
METADATA_ADD_TYPE(S32,psS32,PS_DATA_S32)
METADATA_ADD_TYPE(F32,psF32,PS_DATA_F32)
METADATA_ADD_TYPE(F64,psF64,PS_DATA_F64)
METADATA_ADD_TYPE(List,psList*,PS_DATA_LIST)
METADATA_ADD_TYPE(Str,const char*,PS_DATA_STRING)
METADATA_ADD_TYPE(Vector,psVector*,PS_DATA_VECTOR)
METADATA_ADD_TYPE(Image,psImage*,PS_DATA_IMAGE)
METADATA_ADD_TYPE(Hash,psHash*,PS_DATA_HASH)
METADATA_ADD_TYPE(LookupTable,psLookupTable*,PS_DATA_LOOKUPTABLE)
METADATA_ADD_TYPE(Unknown,void*,PS_DATA_UNKNOWN)
METADATA_ADD_TYPE(Metadata,psMetadata*,PS_DATA_METADATA)
METADATA_ADD_TYPE(Array,psArray*,PS_DATA_ARRAY)
METADATA_ADD_TYPE(Time,psTime*,PS_DATA_TIME)
//METADATA_ADD_TYPE(Ptr,psPtr,PS_DATA_UNKNOWN)

bool psMetadataAddPtr(psMetadata *md,
                      long location,
                      const char *name,
                      psDataType type,
                      const char *comment,
                      psPtr value)
{
    return (psMetadataAdd(md, location, name, type, comment, value) );

}

bool psMetadataRemove(psMetadata *md,
                      long where,
                      const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);

    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psList* mdList = md->list;

    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    psHash* mdTable = md->hash;

    // Select removal by key or index
    if (key != NULL) {
        // Remove by key name
        psMetadataItem* entry = psHashLookup(mdTable,key);
        if (entry == NULL) {
            psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
            return false;
        }
        if (entry->type == PS_DATA_METADATA_MULTI) {
            psMetadataItem* listItem;
            psListIterator* iter = psListIteratorAlloc(
                                       entry->data.list,
                                       PS_LIST_HEAD,true);
            while ((listItem=psListGetAndIncrement(iter)) != NULL) {
                psListRemoveData(mdList, listItem);
            }
            psFree(iter);
            psHashRemove(mdTable,key);

        } else {
            psListRemoveData(mdList, entry);
            psHashRemove(mdTable, key);
        }
    } else {
        // Remove by index
        psMetadataItem* entry = psListGet(mdList, where);
        if (entry == NULL) {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_FIND_INDEX_FAILED, where);
            return false;
        }
        key = entry->name;

        if (key == NULL) {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_REMOVE_LIST_INDEX_FAILED, where);
            return false;
        }

        psMetadataItem* tableItem = psHashLookup(mdTable, key);
        if (tableItem == NULL) {
            psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
            return false;
        }

        if (tableItem->type == PS_DATA_METADATA_MULTI) {
            // multiple entries with same key, remove just the specified one
            psListRemoveData(tableItem->data.list, entry);
        } else {
            if (!psHashRemove(mdTable, key)) {
                psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
                return false;
            }
        }
        psListRemove(mdList, where);
    }

    return true;
}

bool psMetadataRemoveKey(psMetadata *md,
                         const char *key)
{
    return ( psMetadataRemove(md, PS_LIST_HEAD, key) );
}

bool psMetadataRemoveIndex(psMetadata *md,
                           int location)
{
    return ( psMetadataRemove(md, location, NULL) );
}

psMetadataItem* psMetadataLookup(const psMetadata *md,
                                 const char *key)
{
    psHash* mdTable = NULL;
    psMetadataItem* entry = NULL;


    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    PS_ASSERT_PTR_NON_NULL(key,NULL);

    mdTable = md->hash;
    entry = (psMetadataItem*)psHashLookup(mdTable, key);

    return entry;
}

void* psMetadataLookupPtr(bool *status,
                          const psMetadata *md,
                          const char *key)
{
    psMetadataItem *metadataItem = NULL;

    if (status) {
        *status = true;
    }

    metadataItem = psMetadataLookup(md, key);
    if(metadataItem == NULL) {
        if (status) {
            *status = false;
        }
        return NULL;
    }
    if (metadataItem->type == PS_DATA_METADATA_MULTI) {
        // if multiple keys found, use the first.
        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head);
    }

    if(PS_DATA_IS_PRIMITIVE(metadataItem->type)) {
        if (status) {
            *status = false;
        }
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                PS_ERRORTEXT_psMetadata_METATYPE_INVALID,
                metadataItem->type);
        return NULL;
    } else {
        return metadataItem->data.V;
    }
}

#define psMetadataLookupNumTYPE(TYPE) \
ps##TYPE psMetadataLookup##TYPE(bool *status, const psMetadata *md, const char *key) \
{ \
    psMetadataItem *metadataItem = NULL; \
    ps##TYPE value = 0; \
    \
    if (status) { \
        *status = true; \
    } \
    \
    metadataItem = psMetadataLookup(md, key); \
    if(metadataItem == NULL) { \
        if (status) { \
            *status = false; \
        } \
        return 0; \
    } \
    if (metadataItem->type == PS_DATA_METADATA_MULTI) { \
        /* if multiple keys found, use the first. */ \
        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head); \
    } \
    \
    switch (metadataItem->type) { \
    case PS_DATA_S32: \
        value = (ps##TYPE)metadataItem->data.S32; \
        break; \
    case PS_DATA_F32: \
        value = (ps##TYPE)metadataItem->data.F32; \
        break; \
    case PS_DATA_F64: \
        value = (ps##TYPE)metadataItem->data.F64; \
        break; \
    case PS_DATA_BOOL: \
        if (metadataItem->data.B) { \
            value = 1; \
        } \
        break; \
    default: \
        /* if you get to this point, the value is not a number. */ \
        if (status) { \
            *status = false; \
        } \
        break; \
    } \
    \
    /* psFree(metadataItem); currently, the lookup doesn't increment the ref count */ \
    return value; \
}

psMetadataLookupNumTYPE(F32)
psMetadataLookupNumTYPE(F64)
psMetadataLookupNumTYPE(S32)
psMetadataLookupNumTYPE(Bool)

psMetadataItem* psMetadataGet(const psMetadata *md,
                              int location)
{
    psMetadataItem* entry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    entry = (psMetadataItem*) psListGet(md->list, location);
    if (entry == NULL) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_FIND_INDEX_FAILED, location);
        return NULL;
    }

    return entry;
}

psMetadataIterator* psMetadataIteratorAlloc(psMetadata* md,
        long location,
        const char* regex)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    psMetadataIterator* newIter = psAlloc(sizeof(psMetadataIterator));
    newIter->regex = NULL;
    newIter->iter = NULL;

    // Set deallocator
    psMemSetDeallocator(newIter, (psFreeFunc) metadataIteratorFree);

    if (regex == NULL) {
        newIter->iter = psListIteratorAlloc(md->list, location, false);
        return newIter;
    } else {
        int regRtn = regcomp(newIter->regex,regex,0);
        if (regRtn != 0) {
            char errMsg[256];
            regerror(regRtn, newIter->regex, errMsg, 256);
            regfree(newIter->regex);
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    PS_ERRORTEXT_psMetadata_REGEX_INVALID,
                    errMsg);
            psFree(newIter);
            return NULL;
        }
    }

    psMetadataIteratorSet(newIter, location); // XXX: do we error if no match is found?

    return newIter;
}

bool psMetadataIteratorSet(psMetadataIterator* iterator,
                           long location)
{
    int match;
    psMetadataItem* cursor;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return psListIteratorSet(iter,location);
    }

    if (location < 0) {
        // match from the tail
        match = 0;
        psListIteratorSet(iter,PS_LIST_TAIL);
        while ( (cursor=(psMetadataItem*)iter->cursor) != NULL) {
            if (regexec(regex, cursor->name, 0, NULL, 0) == 0) {
                // this key is a match
                match--;
                if (match == location) {
                    break;
                }
            }
            (void)psListGetAndDecrement(iter);
        }
        return (match == location);
    }

    // find the n-th match from the head
    match = -1;
    psListIteratorSet(iter,PS_LIST_HEAD);
    while ( (cursor=(psMetadataItem*)iter->cursor) != NULL) {
        if (regexec(regex, cursor->name, 0, NULL, 0) == 0) {
            // this key is a match
            match++;
            if (match == location) {
                break;
            }
        }
        (void)psListGetAndIncrement(iter);
    }
    return (match == location);
}

psMetadataItem* psMetadataGetAndIncrement(psMetadataIterator* iterator)
{
    psMetadataItem* oldValue;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndIncrement(iter);
    }

    oldValue = (psMetadataItem*)iter->cursor;

    while (psListGetAndIncrement(iter) != NULL) {
        if (iter->cursor != NULL &&
                regexec(regex, ((psMetadataItem*)iter->cursor)->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return oldValue;
}

psMetadataItem* psMetadataGetAndDecrement(psMetadataIterator* iterator)
{
    psMetadataItem* oldValue;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndDecrement(iter);
    }

    oldValue = (psMetadataItem*)iter->cursor;

    while (psListGetAndDecrement(iter) != NULL) {
        if (iter->cursor != NULL &&
                regexec(regex, ((psMetadataItem*)iter->cursor)->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return oldValue;
}

psMetadata *psMetadataLookupMD(bool *status, const
                               psMetadata *md,
                               const char *key)
{
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    psMetadata *value = NULL;  // The value to return
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n");
        }
    } else if (item->type != PS_DATA_METADATA) {
        // The value at the key isn't metadata
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_META, as expected.\n");
        }
        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.md; // The requested metadata
    }
    return value;
}


char *psMetadataLookupStr(bool *status,
                          const psMetadata *md,
                          const char *key)
{
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    char *value = NULL;   // The value to return
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n");
        }
    } else if (item->type != PS_DATA_STRING) {
        // The value at the key isn't of the desired type
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_STRING, as expected.\n");
        }
        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.V; // The requested metadata
    }
    return value;
}


void psMetadataPrint(psMetadata *md,
                     int level)
{
    psMetadataIterator *iter = psMetadataIteratorAlloc(md, PS_LIST_HEAD, NULL); // Iterator
    psMetadataItem *item = NULL; // Item from metadata
    while ( (item = psMetadataGetAndIncrement(iter)) ) {
        // Indent...
        for (int i = 0; i < level; i++) {
            printf(" ");
        }
        printf("%s", item->name);
        if (item->comment && strlen(item->comment) > 0) {
            printf(" (%s)", item->comment);
        }
        printf(": ");
        switch (item->type) {
        case PS_DATA_STRING:
            printf("%s", (char*)(item->data.V));
            break;
        case PS_DATA_BOOL:
            if (item->data.B) {
                printf("True");
            } else {
                printf("False");
            }
            break;
        case PS_DATA_S32:
            printf("%d", item->data.S32);
            break;
        case PS_DATA_F32:
            printf("%f", item->data.F32);
            break;
        case PS_DATA_F64:
            printf("%f", item->data.F64);
            break;
        case PS_DATA_METADATA:
            printf("\n");
            psMetadataPrint(item->data.V, level + 1);
            break;
        default:
            psError(PS_ERR_IO, false, "Non-printable metadata type: %x\n", item->type);
        }
        printf("\n");
    }
    psFree(iter);

    return;
}


// Set verbosity level
int psArgumentVerbosity(int *argc, char **argv)
{
    int logLevel = 2;   // Default log level
    int argnum = 0;   // Argument number

    // set in order, so that -vvv overrides -vv overrides -v
    if ( (argnum = psArgumentGet(*argc, argv, "-v")) ) {
        psArgumentRemove(argnum, argc, argv);
        logLevel = 3;
    }
    if ( (argnum = psArgumentGet(*argc, argv, "-vv")) ) {
        psArgumentRemove(argnum, argc, argv);
        logLevel = 4;
    }
    if ( (argnum = psArgumentGet(*argc, argv, "-vvv")) ) {
        psArgumentRemove(argnum, argc, argv);
        logLevel = 5;
    }
    psLogSetLevel (logLevel);  // XXX: This function should return an error if the log level is invalid

    if ( (argnum = psArgumentGet(*argc, argv, "-logfmt")) ) {
        if (*argc < argnum + 2) {
            psError(PS_ERR_IO, true, "-logfmt switch specified without a format.");
        } else {
            psArgumentRemove(argnum, argc, argv);
            psLogSetFormat(argv[argnum]); // XXX EAM : this function should return an error if the log format is invalid
            psArgumentRemove(argnum, argc, argv);
        }
    }

    // Now the trace stuff
    // argument format is: -trace (facil) (level)
    while ( (argnum = psArgumentGet(*argc, argv, "-trace")) ) {
        if ( (*argc < argnum + 3) ) {
            psError(PS_ERR_IO, true, "-trace switch specified without facility and level.");
        }
        psArgumentRemove(argnum, argc, argv);
        psTraceSetLevel(argv[argnum], atoi(argv[argnum+1])); // XXX: This function should return an error if the trace level is invalid
        psArgumentRemove(argnum, argc, argv);
        psArgumentRemove(argnum, argc, argv);
    }
    if ((argnum = psArgumentGet(*argc, argv, "-trace-levels"))) {
        psTracePrintLevels();
        exit(2);
    }

    return logLevel;
}

// Find the location of the specified argument
int psArgumentGet(int argc, char **argv, const char *arg)
{
    for (int i = 1; i < argc; i++) {
        if (!strcmp(argv[i], arg))
            return i;
    }

    return 0;
}

// Remove the specified argument (by location)
bool psArgumentRemove(int argnum, int *argc, char **argv)
{
    if (argnum > 0) {
        (*argc)--;
        for (int i = argnum; i < *argc; i++) {
            argv[i] = argv[i+1];
        }
    } else {
        return false;
    }

    return true;
}


static psMetadataItem *argumentRead(psMetadataItem *item, // Item to read into
                                    int argnum, // Argument number
                                    int *argc, // Number of arguments in total
                                    char **argv) // The arguments

{
    psMetadataItem *newItem = NULL;
    switch(item->type)
    {
        // Only doing a representative set of types
    case PS_DATA_S32:
        newItem = psMetadataItemAlloc(item->name, item->type, item->comment, atoi(argv[argnum]));
        psArgumentRemove(argnum, argc, argv);
        break;
    case PS_DATA_F32:
        newItem = psMetadataItemAlloc(item->name, item->type, item->comment, atof(argv[argnum]));
        psArgumentRemove(argnum, argc, argv);
        break;
    case PS_DATA_BOOL:
        // Turn option on; no optional argument to remove
        newItem = psMetadataItemAlloc(item->name, item->type, item->comment, true);
        break;
        // XXX: Include the other numerical types
    case PS_DATA_STRING: {
            //psString string = psStringCopy(argv[argnum]); // Get the argument into PS memory management
            //psFree(string);
            newItem = psMetadataItemAlloc(item->name, item->type, item->comment, argv[argnum]);
            psArgumentRemove(argnum, argc, argv);
        }
        break;
    default:
        psError(PS_ERR_IO, true, "Argument type (%x) is not supported --- argument %s ignored\n",
                item->type, item->name);
        psFree(newItem);
        return NULL;
    }

    return newItem;
}


// XXX: There is a memory leak in the MULTI section.  I think it might have something to do with reference
// counting between lists and MD, in the second section of the code (copy newArgs into arguments), but I'm not
// entirely sure.
bool psArgumentParse(psMetadata *arguments,
                     int *argc,
                     char **argv)
{
    // We need to do a bit of mucking around in order to preserve the arguments metadata until the last
    // minute --- if there is a bad argument, we need to return the old "arguments", since they contain
    // the default values, which we probably want to output in a "help" message (we don't want to print
    // the changed values and have the user think that they are default values).

    psMetadata *newArgs = psMetadataAlloc(); // Place to read arguments into
    psList *changed = psListAlloc(NULL);// List of keys that have changed

    for (int i = 1; i < *argc; i++) {
        psTrace(__func__, 7, "Looking at %s\n", argv[i]);
        psMetadataItem *argItem = psMetadataLookup(arguments, argv[i]);
        if (argItem) {
            psArgumentRemove(i, argc, argv); // Remove the switch
            if (argItem->type != PS_DATA_METADATA_MULTI) {
                if (argItem->type != PS_DATA_BOOL && *argc < i + 1) {
                    psError(PS_ERR_IO, true, "Required argument for %s is missing.\n", argItem->name);
                    // XXX: Cleanup before returning
                    psFree(newArgs);
                    return false;
                }
                psMetadataItem *newItem = argumentRead(argItem, i, argc, argv);
                psMetadataAddItem(newArgs, newItem, PS_LIST_TAIL, PS_META_REPLACE);
                psFree(newItem);
            } else {
                // Go through the MULTI
                psList *multi = argItem->data.V; // The list of MULTI psMetadataItems
                if (*argc < i + multi->n) {
                    psError(PS_ERR_IO, true, "Not enough arguments for %s.\n", argItem->name);
                    // Remove the arguments --- they will be ignored
                    for (int j = i; j < *argc; j++) {
                        psArgumentRemove(i, argc, argv);
                    }
                    // XXX: Cleanup before returning
                    psFree(newArgs);
                    return false;
                }

                // Remove any prior existence in the newArgs --- this is important because we specify
                // adding the new items as DUPLICATE_OK, so if some idiot specifies it twice, we'd end
                // up with two copies of everything.
                psMetadataItem *checkItem = psMetadataLookup(newArgs, argItem->name);
                if (checkItem) {
                    (void)psMetadataRemove(newArgs, 0, argItem->name);
                    (void)psListRemoveData(changed, argItem->name);
                }

                psListIterator *multiIter = psListIteratorAlloc(multi, PS_LIST_HEAD, true);
                psMetadataItem *nextItem = NULL; // Item from list
                while ( (nextItem = psListGetAndIncrement(multiIter)) ) {
                    psMetadataItem *newItem = argumentRead(nextItem, i, argc, argv);
                    psMetadataAddItem(newArgs, newItem, PS_LIST_TAIL, PS_META_DUPLICATE_OK);
                    //psFree(newItem);
                }
                psFree(multiIter);
            }

            // Some book-keeping
            //     psString name = psStringCopy(argItem->name);
            psListAdd(changed, PS_LIST_TAIL, argItem->name);
            i--;

        } else if ( (strncmp(argv[i], "-", 1) == 0 ) || (strncmp(argv[i], "+", 1) == 0) ) {
            // Someone's specified a bad option
            psError(PS_ERR_IO, true, "Unknown option: %s\n", argv[i]);
            psFree(newArgs);
            return false;
        }
    }

    // All the arguments are good, so now we can copy the newArgs over
    psListIterator *changedIter = psListIteratorAlloc(changed, PS_LIST_HEAD, false); // Iterator
    psString name = NULL;  // Item from iteration
    while ( (name = psListGetAndIncrement(changedIter)) ) {
        printf("Updating %s\n", name);
        psMetadataItem *oldItem = psMetadataLookup(arguments, name);
        psMetadataItem *newItem = psMetadataLookup(newArgs, name);
        if ( (oldItem->type != newItem->type) ) {
            psAbort(__func__, "Shouldn't reach here!\n");
        }
        switch (oldItem->type) {
            // Only doing a representative set of types
        case PS_DATA_S32:
            oldItem->data.S32 = newItem->data.S32;
            break;
        case PS_DATA_F32:
            oldItem->data.F32 = newItem->data.F32;
            break;
        case PS_DATA_BOOL:
            oldItem->data.B = newItem->data.B;
            break;
            // XXX: Include the other numerical types
        case PS_DATA_STRING:
            psFree(oldItem->data.V);
            oldItem->data.V = psMemIncrRefCounter(newItem->data.V);
            break;
        case PS_DATA_METADATA_MULTI: {
                psList *newMulti = psMemIncrRefCounter(newItem->data.V); // The new list of MULTI
                psList *oldMulti = oldItem->data.V; // The old list of MULTI
                psListIterator *newMultiIter = psListIteratorAlloc(newMulti, PS_LIST_HEAD, false);
                psListIterator *oldMultiIter = psListIteratorAlloc(oldMulti, PS_LIST_HEAD, true);
                psMetadataItem *newMultiItem = NULL; // Item from iterator
                while ( (newMultiItem = psListGetAndIncrement(newMultiIter)) ) {
                    psMetadataItem *oldMultiItem = psListGetAndIncrement(oldMultiIter);
                    if (!oldMultiItem) {
                        psAbort(__func__,
                                "Something went very wrong here!  The lists SHOULD be of the same length!\n");
                    }
                    switch (oldMultiItem->type) {
                        // Only doing a representative set of types
                    case PS_DATA_S32:
                        oldItem->data.S32 = newItem->data.S32;
                        break;
                    case PS_DATA_F32:
                        oldItem->data.F32 = newItem->data.F32;
                        break;
                        // XXX: Include the other numerical types
                    case PS_DATA_STRING:
                        psFree(oldItem->data.V);
                        oldItem->data.V = psMemIncrRefCounter(newItem->data.V);
                        break;
                    default:
                        psAbort(__func__, "Should never ever get here, ever.\n");
                    }
                    psFree(oldMultiItem);
                    psFree(newMultiItem);
                }
                psFree(newMultiIter);
                psFree(oldMultiIter);
            }
            break;
        default:
            psAbort(__func__, "Should never ever ever get here.\n");
        }
    }
    psFree(changedIter);
    psFree(changed);

    // Now, blow away the newArgs and we're done.
    psFree(newArgs);
    return true;
}


static int argLength(psMetadataItem *arg)
{
    switch (arg->type) {
        // Only doing a representative set of types
    case PS_DATA_S32:
        return arg->data.S32 >= 0 ? (int)log10f((float)arg->data.S32) + 1 :
               (int)log10f(-(float)arg->data.S32) + 2;
        // XXX: Other numerical types
    case PS_DATA_F32:
        return arg->data.F32 >= 0 ? 12 : 13; // -d.dddddde?dd
    case PS_DATA_F64:
        return arg->data.F64 >= 0 ? 12 : 13; // -d.dddddde?dd
    case PS_DATA_BOOL:
        return arg->data.B ? 4 : 5;
    case PS_DATA_STRING:
        return strlen(arg->data.V);
    default:
        psAbort(__func__, "Argument type (%x) is not supported.\n", arg->type);
    }

    return 0;
}

#define NUM_SPACES 4   // Number of spaces between

void psArgumentHelp(psMetadata *arguments)
{
    printf("Optional arguments, with default values:\n");
    psMetadataIterator *argIter = psMetadataIteratorAlloc(arguments, PS_LIST_HEAD, NULL);
    psMetadataItem *argItem = NULL; // Item from iterator
    int maxName = 4;   // Maximum length of a name
    int maxValue = 4;   // Maximum length of a value

    // First pass to get the sizes
    while ( (argItem = psMetadataGetAndIncrement(argIter)) ) {
        if (strlen(argItem->name) > maxName) {
            maxName = strlen(argItem->name);
        }
        int valLength = argLength(argItem);
        if (valLength > maxValue) {
            maxValue = valLength;
        }
    }

    // Second pass to print
    psMetadataIteratorSet(argIter, PS_LIST_HEAD);
    psString lastName = NULL;  // Last name we printed
    while ( (argItem = psMetadataGetAndIncrement(argIter)) ) {
        // Initial indent
        for (int i = 0; i < NUM_SPACES; i++) {
            printf(" ");
        }

        // Print the name if required
        int position = 0; // Number of spaces in
        if (! lastName || strcmp(lastName, argItem->name) != 0) {
            // A new name
            printf("%s", argItem->name);
            position += strlen(argItem->name);
            lastName = argItem->name;
        }
        for (int i = position; i < maxName + NUM_SPACES; i++) {
            printf(" ");
        }

        // Print the value
        printf("(");
        switch (argItem->type) {
            // Only doing a representative set of types
        case PS_DATA_S32:
            printf("%d", argItem->data.S32);
            break;
            // XXX: Other numerical types
        case PS_DATA_F32:
            printf("%.6e", argItem->data.F32);
            break;
        case PS_DATA_F64:
            printf("%.6e", argItem->data.F64);
            break;
        case PS_DATA_BOOL:
            if (argItem->data.B) {
                printf("TRUE");
            } else {
                printf("FALSE");
            }
            break;
        case PS_DATA_STRING:
            printf("%s", (char*)(argItem->data.V));
            break;
        default:
            psAbort(__func__, "Argument type (%x) is not supported.\n", argItem->type);
        }
        printf(")");
        for (int i = argLength(argItem); i < maxValue + NUM_SPACES; i++) {
            printf(" ");
        }

        // Print the comment
        if (argItem->comment) {
            printf("%s", argItem->comment);
        }
        printf("\n");
    }

    psFree(argIter);
}

// we have log levels 1 (Error), 2 (Warning), 3 (Info), 4 (Details), 5 (Minutiae)
// 2 = default, -v = 3, -vv = 4, -vvv = 5
psS32 psLogArguments (int *argc, char **argv)
{

    int N, level;

    // default log level is 2
    level = 2;

    // set in order, so that -vvv overrides -vv overrides -v
    if ((N = psArgumentGet (*argc, argv, "-v"))) {
        psArgumentRemove (N, argc, argv);
        level = 3;
    }
    if ((N = psArgumentGet (*argc, argv, "-vv"))) {
        psArgumentRemove (N, argc, argv);
        level = 4;
    }
    if ((N = psArgumentGet (*argc, argv, "-vvv"))) {
        psArgumentRemove (N, argc, argv);
        level = 5;
    }

    if ((N = psArgumentGet (*argc, argv, "-logfmt"))) {
        if (*argc < N + 2) {
            psAbort ("psLogArguments", "USAGE: -logfmt (format)");
        }
        psArgumentRemove (N, argc, argv);
        psLogSetFormat (argv[N]); // XXX EAM : this function should return an error if the log format is invalid
        psArgumentRemove (N, argc, argv);
    }

    // set the level, return the level
    psLogSetLevel (level);
    return (level);
}

// set trace levels by facility
psS32 psTraceArguments (int *argc, char **argv)
{

    int N;

    // argument format is: -trace (facil) (level)
    while ((N = psArgumentGet (*argc, argv, "-trace"))) {
        if (*argc < N + 3) {
            psAbort ("psTraceArguments", "USAGE: -trace (facility) (level)");
        }
        psArgumentRemove (N, argc, argv);
        psTraceSetLevel (argv[N], atoi(argv[N+1]));
        psArgumentRemove (N, argc, argv);
        psArgumentRemove (N, argc, argv);
    }
    if ((N = psArgumentGet (*argc, argv, "-trace-levels"))) {
        psTracePrintLevels ();
        exit (2);
    }
    return (TRUE);
}

