/*
 * magictool.c
 *
 * Copyright (C) 2006-2007  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "magictool.h"

static psS64 definerunMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool addinputskyfileMode(pxConfig *config);
static bool inputtreeMode(pxConfig *config);
static bool toprocessMode(pxConfig *config);
static bool addresultMode(pxConfig *config);
static bool tomaskMode(pxConfig *config);
static bool addmaskMode(pxConfig *config);
static bool toskyfilemaskMode(pxConfig *config);
static bool addskyfilemaskMode(pxConfig *config);

static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = magictoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(MAGICTOOL_MODE_DEFINERUN,      definerunMode);
        MODECASE(MAGICTOOL_MODE_UPDATERUN,      updaterunMode);
        MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode);
        MODECASE(MAGICTOOL_MODE_INPUTTREE,      inputtreeMode);
        MODECASE(MAGICTOOL_MODE_TOPROCESS,      toprocessMode);
        MODECASE(MAGICTOOL_MODE_ADDRESULT,      addresultMode);
        MODECASE(MAGICTOOL_MODE_TOMASK,         tomaskMode);
        MODECASE(MAGICTOOL_MODE_ADDMASK,        addmaskMode);
        MODECASE(MAGICTOOL_MODE_TOSKYFILEMASK,  toskyfilemaskMode);
        MODECASE(MAGICTOOL_MODE_ADDSKYFILEMASK, addskyfilemaskMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}

static psS64 definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    bool status = false;
    psString workdir = psMetadataLookupStr(&status, config->args, "-workdir");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir");
        return false;
    }
    if (!workdir) {
        psError(PS_ERR_UNKNOWN, true, "-workdir is required");
        return false;
    }

    psString label = psMetadataLookupStr(&status, config->args, "-label");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
        return false;
    }

    psString dvodb = psMetadataLookupStr(&status, config->args, "-dvodb");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dvodb");
        return false;
    }

    psTime *registered = NULL;
    {
        psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered");
            return false;
        }
        // pass through NULL as this is an optional field
        if (registeredStr) {
            registered = psTimeFromISO(registeredStr, PS_TIME_UTC);
        } else {
            registered = NULL;
        }
    }

   magicRunRow *run = magicRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            "dirty",    // workdir_state
            label,
            dvodb,
            registered
    );
    psFree(registered);
    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object");
        return true;
    }
    if (!magicRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return true;
    }

    // get the assigned warp_id
    psS64 magic_id = psDBLastInsertID(config->dbh);
    run->magic_id = magic_id;

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            psFree(run);
            return false;
        }
    }

    if (!magicRunPrintObject(stdout, run, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(run);
            return false;
    }

    psFree(run);

    return magic_id;
}


static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString magic_id = psMetadataLookupStr(&status, config->args, "-magic_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magic_id");
        return false;
    }
    if (!magic_id) {
        psError(PS_ERR_UNKNOWN, true, "-magic_id is required");
        return false;
    }

    psString state = psMetadataLookupStr(&status, config->args, "-state");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state");
        return false;
    }
    if (!state) {
        psError(PS_ERR_UNKNOWN, true, "-state is required");
        return false;
    }

    if (state) {
        // set detRun.state to state
        return setmagicRunState(config, (psS64)atoll(magic_id), state);
    }

    return true;
}


static bool addinputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString magic_id = psMetadataLookupStr(&status, config->args, "-magic_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magic_id");
        return false;
    }
    if (!magic_id) {
        psError(PS_ERR_UNKNOWN, true, "-magic_id is required");
        return false;
    }

    psString diff_id = psMetadataLookupStr(&status, config->args, "-diff_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -diff_id");
        return false;
    }
    if (!diff_id) {
        psError(PS_ERR_UNKNOWN, true, "-diff_id is required");
        return false;
    }

    psString node = psMetadataLookupStr(&status, config->args, "-node");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -node");
        return false;
    }
    if (!node) {
        psError(PS_ERR_UNKNOWN, true, "-node is required");
        return false;
    }

    magicInputSkyfileInsert(
            config->dbh,
            (psS64)atoll(magic_id),
            (psS64)atoll(diff_id),
            node
    );

    return true;
}


static bool inputtreeMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString magic_id = psMetadataLookupStr(&status, config->args, "-magic_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magic_id");
        return false;
    }
    if (!magic_id) {
        psError(PS_ERR_UNKNOWN, true, "-magic_id is required");
        return false;
    }

    psString dep_file = psMetadataLookupStr(&status, config->args, "-dep_file");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dep_file");
        return false;
    }
    if (!dep_file) {
        psError(PS_ERR_UNKNOWN, true, "-dep_file is required");
        return false;
    }

    if (!parseAndInsertNodeDeps(config, (psS64)atoll(magic_id), dep_file)) {
    }

    return true;
}


pxNode *pxTreeFromMagicTreeRows(psArray *rows)
{
    psHash *nodeNames = psHashAlloc(10);

    for (long i = 0; i < psArrayLength(rows); i++) {
        magicTreeRow *row = rows->data[i];
        pxNode *parent = psHashLookup(nodeNames, row->dep);
        pxNode *node = pxNodeAlloc(row->node, parent);
        psHashAdd(nodeNames, row->node, node);
        psFree(node);
    }

    pxNode *root = psHashLookup(nodeNames, "root");
    psFree(nodeNames);

    return root;
}

static bool toprocessMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString magic_id = psMetadataLookupStr(&status, config->args, "-magic_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magic_id");
        return false;
    }

    psMetadata *where = NULL;
    if (magic_id) {
        psMetadata *where = psMetadataAlloc();
        psMetadataAddS64(where, PS_LIST_TAIL, "magic_id", 0, "==", (psS64)atoll(magic_id));
    }

    psArray *rows = magicTreeSelectRowObjects(
        config->dbh,
        where,
        0 // limit
    );
    psFree(where);

    pxNode *root = pxTreeFromMagicTreeRows(rows);
    psFree(rows);
    // silence unused warnings...
    if (root) {}

    return true;
}


static bool addresultMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    return true;
}


static bool tomaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    return true;
}


static bool addmaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    return true;
}


static bool toskyfilemaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    return true;
}


static bool addskyfilemaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    return true;
}

#if 0

static psS64 definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    bool status = false;
    psString mode = psMetadataLookupStr(&status, config->args, "-mode");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -survey_mode");
        return false;
    }
    if (!mode) {
        psError(PS_ERR_UNKNOWN, true, "-mode is required");
        return false;
    }
    // check mode
    if (mode && !isValidMode(config, mode)) {
        psError(PS_ERR_UNKNOWN, false, "invalud mode");
        return false;
    }

    psString workdir = psMetadataLookupStr(&status, config->args, "-workdir");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir");
        return false;
    }
    if (!workdir) {
        psError(PS_ERR_UNKNOWN, true, "-workdir is required");
        return false;
    }

    psTime *registered = NULL;
    {
        psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered");
            return false;
        }
        // pass through NULL as this is an optional field
        if (registeredStr) {
            registered = psTimeFromISO(registeredStr, PS_TIME_UTC);
        } else {
            registered = NULL;
        }
    }

    warpRunRow *warpRun = warpRunRowAlloc(
            0,          // ID
            mode,
            "reg",      // state
            workdir,
            NULL,       // dvodb
            registered
    );
    psFree(registered);
    if (!warpRun) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc warpRun object");
        return true;
    }
    if (!warpRunInsertObject(config->dbh, warpRun)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(warpRun);
        return true;
    }

    // get the assigned warp_id
    psS64 warp_id = psDBLastInsertID(config->dbh);
    warpRun->warp_id = warp_id;

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            psFree(warpRun);
            return false;
        }
    }

    if (!warpRunPrintObject(stdout, warpRun, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(warpRun);
            return false;
    }

    psFree(warpRun);

    return warp_id;
}

static bool runoneMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psS64 warp_id = definerunMode(config);
    if (!warp_id) {
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "failed to define warpRun");
        return false;
    }

    psString warp_id_str = psDBIntToString(warp_id);
    if (!psMetadataAddStr(config->args, PS_LIST_TAIL, "-warp_id", 0, NULL, warp_id_str)) {
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "failed to add item warp_id");
        psFree(warp_id_str);
        return false;
    }
    psFree(warp_id_str);

    if (!addinputexpMode(config)) {
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "failed to add cam_id to warpRun");
        return false;
    }
    if (!setwarpRunState(config, warp_id, "run")) {
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.state to run");
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warpt_id");
        return false;
    }
    if (!warp_id) {
        psError(PS_ERR_UNKNOWN, true, "-warp_id is required");
        return false;
    }

    psString state = psMetadataLookupStr(&status, config->args, "-state");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state");
        return false;
    }
    if (!state) {
        psError(PS_ERR_UNKNOWN, true, "-state is required");
        return false;
    }

    if (state) {
        // set detRun.state to state
        return setwarpRunState(config, (psS64)atoll(warp_id), state);
    }

    return true;
}


static bool addinputexpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id");
        return false;
    }
    if (!warp_id) {
        psError(PS_ERR_UNKNOWN, true, "-warp_id is required");
        return false;
    }

    psString cam_id = psMetadataLookupStr(&status, config->args, "-cam_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -cam_id");
        return false;
    }
    if (!cam_id) {
        psError(PS_ERR_UNKNOWN, true, "-cam_id is required");
        return false;
    }

    // defaults to false
    bool magiced = psMetadataLookupBool(&status, config->args, "-magiced");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magiced");
        return false;
    }

    // XXX need to validate the warp_id here
    // XXX instead of validiting it here we should just use forgein key
    // constrants
    if (!warpInputExpInsert(config->dbh,
            (psS64)atoll(warp_id),
            (psS64)atoll(cam_id),
            magiced
        )) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool expMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_exp.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpInputExp", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool imfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_imfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool tooverlapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_tooverlap.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpInputExp", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool addoverlapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString mapfile = psMetadataLookupStr(&status, config->args, "-mapfile");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mapfile");
        return false;
    }
    if (!mapfile) {
        psError(PS_ERR_UNKNOWN, true, "-mapfile is required");
        return false;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!parseAndInsertSkyCellMap(config, mapfile)) {
        psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile);
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile)
{
    unsigned int nFail = 0;
    psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false);
    if (!skycells) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile);
        return false;
    }
    if (nFail) {
        psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile);
        psFree(skycells);
        return false;
    }

    psMetadataItem *item = NULL;
    psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL);
    while ((item = psMetadataGetAndIncrement(iter))) {
        if (item->type != PS_DATA_METADATA) {
            psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        psMetadata *sc = item->data.md;
        // this conversion isn't strictly nessicary but it's an easy way of
        // validating the format
        warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc);
        if (!row) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        if (!warpSkyCellMapInsertObject(config->dbh, row)) {
            psErrorCode err = psErrorCodeLast();
            switch (err) {
                case PS_ERR_DB_CLIENT:
                    psError(PXTOOLS_ERR_SYS, false, "database error");
                case PS_ERR_DB_SERVER:
                    psError(PXTOOLS_ERR_PROG, false, "database error");
                default:
                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
            }
            psFree(row);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        psFree(row);
    }
    psFree(iter);
    psFree(skycells);

    return true;
}


static bool scmapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_scmap.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyCellMap");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool towarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_towarped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyCellMap");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool addwarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id");
        return false;
    }
    if (!warp_id) {
        psError(PS_ERR_UNKNOWN, true, "-warp_id is required");
        return false;
    }

    psString skycell_id = psMetadataLookupStr(&status, config->args, "-skycell_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -skycell_id");
        return false;
    }
    if (!skycell_id) {
        psError(PS_ERR_UNKNOWN, true, "-skycell_id is required");
        return false;
    }

    psString tess_id = psMetadataLookupStr(&status, config->args, "-tess_id");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -tess_id");
        return false;
    }
    if (!tess_id) {
        psError(PS_ERR_UNKNOWN, true, "-tess_id is required");
        return false;
    }

    psString uri = psMetadataLookupStr(&status, config->args, "-uri");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri");
        return false;
    }
    if (!uri) {
        psError(PS_ERR_UNKNOWN, true, "-uri is required");
        return false;
    }

    psString path_base = psMetadataLookupStr(&status, config->args, "-path_base");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base");
        return false;
    }
    if (!path_base) {
        psError(PS_ERR_UNKNOWN, true, "-path_base is required");
        return false;
    }

    // optional
    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
        return false;
    }

    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
        return false;
    }

    psF64 pixel_fill = psMetadataLookupF64(&status, config->args, "-pixel_fill");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pixel_fill");
        return false;
    }

    // default values
    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
        return false;
    }

    // we don't want to insert the last skyfile in a run but then not mark the
    // run as 'stop'
    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // XXX need to validate that this coresponds to an warpInputImfile
    if (!warpSkyfileInsert(config->dbh,
            (psS64)atoll(warp_id),
            skycell_id,
            tess_id,
            uri,
            path_base,
            bg,
            bg_stdev,
            pixel_fill,
            code
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!warpCompletedRuns(config)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

bool warpCompletedRuns(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // XXX this SQL has not been broken out to into seperate files as the MYSQL
    // < 5 & MYSQL 5 versions need to be kept in sync
#if MYSQL5
    // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this
    // statement as it thinks it is trying to select from the table being
    // updated. The 4.1 manual says that nested sub-queries are explicited
    // allowed to do this with update statements as a temporary table is
    // created so that you are not actually selecting from the table you are
    // modifying.
    char *query =
        "UPDATE warpRun\n"
        "   SET warpRun.state = 'stop'\n"
        " WHERE\n"
        "   warpRun.warp_id =\n"
        "   (SELECT DISTINCT\n"
        "       warp_id\n"
        "   FROM\n"
        "       (SELECT DISTINCT\n"
        "           warpRun.warp_id,\n"
        "           warpSkyCellMap.warp_id as foo,\n"
        "           warpSkyfile.warp_id as bar\n"
        "       FROM warpRun\n"
        "       JOIN warpSkyCellMap\n"
        "           USING(warp_id)\n"
        "       LEFT JOIN warpSkyfile\n"
        "           USING(warp_id, skycell_id, tess_id)\n"
        "       WHERE\n"
        "           warpRun.state = 'run'\n"
        "       GROUP BY\n"
        "           warpRun.warp_id\n"
        "       HAVING\n"
        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
        "       ) as Foo\n"
        "   )\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
#else // if MYSQL5
{
    char *query =
        "CREATE TEMPORARY TABLE finished\n"
        " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}

{
    char *query =
        "INSERT INTO finished\n"
        " SELECT\n"
        "   warp_id\n"
        " FROM\n"
        "   (SELECT DISTINCT\n"
        "       warpRun.warp_id,\n"
        "       warpSkyCellMap.warp_id as foo,\n"
        "       warpSkyfile.warp_id as bar\n"
        "   FROM warpRun\n"
        "   JOIN warpSkyCellMap\n"
        "       USING(warp_id)\n"
        "   LEFT JOIN warpSkyfile\n"
        "       USING(warp_id, skycell_id, tess_id)\n"
        "   WHERE\n"
        "       warpRun.state = 'run'\n"
        "   GROUP BY\n"
        "       warpRun.warp_id\n"
        "   HAVING\n"
        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
        " ) as Foo \n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}

{
    char *query =
        "UPDATE warpRun\n"
        "   SET warpRun.state = 'stop'\n"
        " WHERE\n"
        "   warpRun.warp_id =\n"
        "   (SELECT DISTINCT\n"
        "       warp_id\n"
        "   FROM finished\n"
        "   )\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}
#endif // if MYSQL5

    return true;
}

static bool warpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_warped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyfile");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        if (!convertIdToStr(output)) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
            psFree(output);
            return false;
        }

        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool revertwarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psString query = pxDataGet("warptool_revertwarped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyfile");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    if (psDBAffectedRows(config->dbh) < 1) {
        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
        return false;
    }

    return true;
}
#endif


static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
{
    PS_ASSERT_PTR_NON_NULL(state, false);

    // check that state is a valid string value
    if (!(
            (strncmp(state, "run", 4) == 0)
            || (strncmp(state, "stop", 5) == 0)
            || (strncmp(state, "reg", 4) == 0)
        )
    ) {
        psError(PS_ERR_UNKNOWN, false,
                "invalid magicRun state: %s", state);
        return false;
    }

    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
    if (!p_psDBRunQuery(config->dbh, query, state, magic_id)) {
        psError(PS_ERR_UNKNOWN, false,
                "failed to change state for magic_id %" PRId64, magic_id);
        return false;
    }

    return true;
}


#if 0
static bool isValidMode(pxConfig *config, const char *mode)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(mode, false);

    // check that state is a valid string value
    if (!(
            (strncmp(mode, "warp", 5) == 0)
            || (strncmp(mode, "diff", 5) == 0)
            || (strncmp(mode, "stack", 6) == 0)
            || (strncmp(mode, "magic", 6) == 0)
        )
    ) {
        psError(PS_ERR_UNKNOWN, false,
                "invalid warpRun mode: %s", mode);
        return false;
    }

    return true;
}
#endif

static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename)
{
    unsigned int nFail = 0;
    psMetadata *deps = psMetadataConfigRead(NULL, &nFail, filename, false);
    if (!deps) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse file: %s", filename);
        return false;
    }
    if (nFail) {
        psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing file: %s", nFail, filename);
        psFree(deps);
        return false;
    }

    psMetadataItem *item = NULL;
    psMetadataIterator *iter = psMetadataIteratorAlloc(deps, 0, NULL);
    while ((item = psMetadataGetAndIncrement(iter))) {
        if (item->type != PS_DATA_STRING) {
            psError(PS_ERR_UNKNOWN, false, "file: %s is in the wrong format", filename);
            psFree(iter);
            psFree(deps);
            return false;
        }

        char *name = item->name;
        char *dependsOn = item->data.str;

        if (!magicTreeInsert(
                config->dbh,
                magic_id,
                name,
                dependsOn
            )) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(iter);
            psFree(deps);
            return false;
        }
    }
    psFree(iter);
    psFree(deps);

    return true;
}
