/*
 * difftool.c
 *
 * Copyright (C) 2007  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "difftool.h"

static bool definerunMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool addinputskyfileMode(pxConfig *config);
static bool inputskyfileMode(pxConfig *config);
static bool todiffskyfileMode(pxConfig *config);
static bool adddiffskyfileMode(pxConfig *config);
static bool diffskyfileMode(pxConfig *config);
static bool revertdiffskyfileMode(pxConfig *config);
static bool definepoprunMode(pxConfig *config);
static bool definewarpstackMode(pxConfig *config);
static bool definewarpwarpMode(pxConfig *config);
static bool pendingcleanuprunMode(pxConfig *config);
static bool pendingcleanupskyfileMode(pxConfig *config);
static bool donecleanupMode(pxConfig *config);
static bool updatediffskyfileMode(pxConfig *config);
static bool exportrunMode(pxConfig *config);
static bool importrunMode(pxConfig *config);

static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked);
static bool diffRunComplete(pxConfig *config);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = difftoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(DIFFTOOL_MODE_DEFINERUN,             definerunMode);
        MODECASE(DIFFTOOL_MODE_UPDATERUN,             updaterunMode);
        MODECASE(DIFFTOOL_MODE_ADDINPUTSKYFILE,       addinputskyfileMode);
        MODECASE(DIFFTOOL_MODE_INPUTSKYFILE,          inputskyfileMode);
        MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE,         todiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE,        adddiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_DIFFSKYFILE,           diffskyfileMode);
        MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
        MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
        MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
        MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
        MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
        MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}


static bool definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);

    // default
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    diffRunRow *run = diffRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            label,
            reduction,
            NULL,       // dvodb
            registered,
            tess_id,
            bothways,
            exposure,
            false
    );
    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
        return false;
    }
    if (!diffRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return false;
    }

    // get the assigned diff_id
    run->diff_id = psDBLastInsertID(config->dbh);

    if (!diffRunPrintObject(stdout, run, !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print object");
        psFree(run);
        return false;
    }

    psFree(run);

    return true;
}


static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);

    if (state) {
        // set detRun.state to state
        return setdiffRunState(config, diff_id, state, false);
    }

    return true;
}


static bool addinputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);

    // optional
    PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false);
    PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false);
    PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false);
    PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false);
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);

    // must provide either stack1 or warp1 but not BOTH
    if (!(stack1 || warp1)) {
        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
        return false;
    }
    if (stack1 && warp1) {
        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
        return false;
    }
    // must provide either stack2 or warp2 but not BOTH
    if (!(stack2 || warp2)) {
        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
        return false;
    }
    if (stack2 && warp2) {
        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
        return false;
    }

    // if a warp1 was provided we need to lookup the and tess_id from the diffRun
    psString tess_id = NULL;
    if (warp1) {
        if (!p_psDBRunQueryF(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }

        psArray *output = p_psDBFetchResult(config->dbh);
        if (!output) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
        if (!psArrayLength(output)) {
            psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id);
            psFree(output);
            return false;
        }

        diffRunRow *run = diffRunObjectFromMetadata(output->data[0]);
        tess_id = run->tess_id;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffInputSkyfileInsert(config->dbh,
            diff_id,
            skycell_id,
            warp1 ? warp1 : PS_MAX_S64, // defined or NULL
            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
            warp2 ? warp2 : PS_MAX_S64, // defined or NULL
            stack2 ? stack2 : PS_MAX_S64, // defined or NULL
            tess_id,
            0                             // diff_skyfile_id
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!p_psDBRunQueryF(config->dbh, "SELECT count(diff_id) FROM diffRun JOIN diffInputSkyfile USING(diff_id) WHERE diff_id = %" PRId64, diff_id)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id);
        psFree(output);
        return false;
    }

    bool status;
    psS32 count = psMetadataLookupS32(&status, output->data[0], "count(diff_id)");

    if (count == 2) {
        if (!setdiffRunState(config, diff_id, "new", false)) {
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool inputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();

    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diff_id", "==");
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
    PXOPT_LOOKUP_BOOL(input, config->args, "-input", false);

    if (template && input) {
        // User apparently wants both, which is the default behaviour
        template = false;
        input = false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("difftool_inputskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    psString whereClause = NULL;
    if (psListLength(where->list)) {
        whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringPrepend(&whereClause, "\n AND ");
    }
    psFree(where);

    // Add condition to get only templates or only inputs
    {
        psString templateClause = NULL;
        if (template) {
            psStringAppend(&templateClause, " %s", " template != 0");
        } else if (input) {
            psStringAppend(&templateClause, " %s", " template = 0");
        }
        if (templateClause) {
            psStringAppend(&whereClause, "\n AND %s", templateClause);
        }
        psFree(templateClause);
    }


    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(whereClause);
        psFree(query);
        return false;
    }
    psFree(whereClause);
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffInputSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool todiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("difftool_todiffskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool adddiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);

    // optional
    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
    PXOPT_LOOKUP_F32(dtime_diff, config->args, "-dtime_diff", false, false);
    PXOPT_LOOKUP_F32(dtime_match, config->args, "-dtime_match", false, false);
    PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false);
    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
    PXOPT_LOOKUP_S32(stamps_num, config->args, "-stamps_num", false, false);
    PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false);
    PXOPT_LOOKUP_F32(stamps_rms, config->args, "-stamps_rms", false, false);
    PXOPT_LOOKUP_F32(norm, config->args, "-norm", false, false);
    PXOPT_LOOKUP_F32(bg_diff, config->args, "-bg_diff", false, false);
    PXOPT_LOOKUP_F32(kernel_x, config->args, "-kernel_x", false, false);
    PXOPT_LOOKUP_F32(kernel_y, config->args, "-kernel_y", false, false);
    PXOPT_LOOKUP_F32(kernel_xx, config->args, "-kernel_xx", false, false);
    PXOPT_LOOKUP_F32(kernel_xy, config->args, "-kernel_xy", false, false);
    PXOPT_LOOKUP_F32(kernel_yy, config->args, "-kernel_yy", false, false);
    PXOPT_LOOKUP_F32(deconv_max, config->args, "-deconv_max", false, false);
    PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false);
    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffSkyfileInsert(config->dbh,
                           diff_id,
                           skycell_id,
                           path_base,
                           "full",
                           bg,
                           bg_stdev,
                           stamps_num,
                           stamps_mean,
                           stamps_rms,
                           norm,
                           bg_diff,
                           kernel_x,
                           kernel_y,
                           kernel_xx,
                           kernel_xy,
                           kernel_yy,
                           deconv_max,
                           sources,
                           dtime_diff,
                           dtime_match,
                           dtime_phot,
                           dtime_script,
                           hostname,
                           good_frac,
                           fault,
                           quality,
                           magicked
          )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffRunComplete(config)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }


    return true;
}


static bool diffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
    PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
    PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("difftool_skyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " WHERE %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool revertdiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
    PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
    PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(where);
        return false;
    }

    // Update state to 'new'
    {
        psString query = pxDataGet("difftool_revertdiffskyfile_update.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(query);

        psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
    }

    // Delete product
    {
        psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(query);

        psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh));
    }

    psFree(where);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked)
{
    PS_ASSERT_PTR_NON_NULL(state, false);

    // check that state is a valid string value
    if (!pxIsValidState(state)) {
        psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state);
        return false;
    }

    char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64;

    if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) {
        psError(PS_ERR_UNKNOWN, false,
                "failed to change state for diff_id %"PRId64, diff_id);
        return false;
    }

    return true;
}

// Generate a single populated run
static bool populatedrun(psArray *list, // List of runs, to print
                         const char *workdir, // Working directory
                         const char *skycell_id, // Skycell identifier
                         const char *tess_id, // Tessellation identifier
                         const char *label, // label
                         const char *reduction, // reduction
                         psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none
                         psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none
                         psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                         psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
                         pxConfig *config // Configuration
                         )
{
    PS_ASSERT_STRING_NON_EMPTY(workdir, false);
    PS_ASSERT_STRING_NON_EMPTY(skycell_id, false);
    PS_ASSERT_STRING_NON_EMPTY(tess_id, false);
    if ((input_warp_id == PS_MAX_S64 && input_stack_id == PS_MAX_S64) ||
        (input_warp_id != PS_MAX_S64 && input_stack_id != PS_MAX_S64)) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, input must be defined.");
        return false;
    }
    if ((template_warp_id == PS_MAX_S64 && template_stack_id == PS_MAX_S64) ||
        (template_warp_id != PS_MAX_S64 && template_stack_id != PS_MAX_S64)) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, template must be defined.");
        return false;
    }

    // default
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);

    diffRunRow *run = diffRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            label,
            reduction,
            NULL,       // dvodb
            registered,
            tess_id,
            false,
            false,
            false       // magicked
    );

    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
        return true;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return true;
    }

    // get the assigned diff_id
    run->diff_id = psDBLastInsertID(config->dbh);

    // Template
    if (!diffInputSkyfileInsert(config->dbh,
            run->diff_id,
            skycell_id,
            input_warp_id,
            input_stack_id,
            template_warp_id,
            template_stack_id,
            tess_id,
            0
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    char *query = "UPDATE diffRun SET state = 'new' WHERE diff_id = '%" PRId64 "'";
    if (!p_psDBRunQueryF(config->dbh, query, run->diff_id)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false,
                "failed to set state to run for diff_id %" PRId64, run->diff_id);
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (list) {
        psArrayAdd(list, list->n, run);
    }

    psFree(run);

    return true;
}


static bool definepoprunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); // required options
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
    PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false);
    PXOPT_LOOKUP_S64(template_stack_id, config->args, "-template_stack_id", false, false);
    PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
    PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    if (template_stack_id && template_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined.");
        return false;
    }
    if (!template_stack_id && !template_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "No template has been defined (-template_stack_id or -template_warp_id)");
        return false;
    }

    if (input_stack_id && input_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined.");
        return false;
    }
    if (!input_stack_id && !input_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "No input has been defined (-input_stack_id or -input_warp_id)");
        return false;
    }

    psArray *list = psArrayAllocEmpty(16); // List of runs, to print

    if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction,
                      input_warp_id ? input_warp_id : PS_MAX_S64,
                      input_stack_id ? input_stack_id : PS_MAX_S64,
                      template_warp_id ? template_warp_id : PS_MAX_S64,
                      template_stack_id ? template_stack_id : PS_MAX_S64,
                      config)) {
        psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
        psFree(list);
        return false;
    }

    if (!diffRunPrintObjects(stdout, list, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(list);
            return false;
    }
    psFree(list);

    return true;
}


static bool definewarpstackMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *expWhere = psMetadataAlloc();
    psMetadata *warpWhere = psMetadataAlloc();
    psMetadata *stackWhere = psMetadataAlloc();

    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
    PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
    PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");

    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);

    // find all things to queue
    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    psString warpQuery = NULL;
    psString stackQuery = NULL;
    psString expQuery = NULL;

    if (psListLength(expWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
        psStringAppend(&expQuery, "\n AND %s", whereClause);
        psFree(whereClause);
    } else {
        expQuery = psStringCopy("\n");
    }
    psFree(expWhere);
    if (psListLength(warpWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
        psStringAppend(&warpQuery, "\n AND %s", whereClause);
        psFree(whereClause);
    } else {
        warpQuery = psStringCopy("\n");
    }
    psFree(warpWhere);

    if (!available) {
        // diff what's available, even if warp run has some faults and is incomplete
        psStringAppend(&warpQuery, " AND warpRun.state = 'full'");
    }

    // don't queue for exposures that have already been diff'd unless requested
    psString diffQuery = NULL;
    if (! (reRun || newTemplates) ) {
        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
    } else {
        diffQuery = psStringCopy("\n");
    }

    if (psListLength(stackWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
        psStringAppend(&stackQuery, "\nAND %s", whereClause);
        psFree(whereClause);
    } else {
        stackQuery = psStringCopy("");
    }
    psFree(stackWhere);

    psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
          case PS_ERR_DB_CLIENT:
            psError(PXTOOLS_ERR_SYS, false, "database error");
            break;
          case PS_ERR_DB_SERVER:
            psError(PXTOOLS_ERR_PROG, false, "database error");
            break;
          default:
            psError(PXTOOLS_ERR_PROG, false, "unknown error");
            break;
        }
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        if (!psDBCommit(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
        return true;
    }

    if (pretend) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
        psFree(output);
        return true;
    }

    // create temporary table
    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }
    psFree(query);
    query = NULL;

    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");

    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
    long numGood = 0;                   // Number of good rows added
    psS64 last_exp_id = 0;
    for (long i = 0; i < output->n; i++) {
        psMetadata *row = output->data[i]; // Output row from query
        bool mdok;                      // Status of MD lookup

        // Take the first warp for each exposure.
        // The list is sorted by exposure id and warp_id and the warps are in descending
        // order.
        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        if (exp_id == last_exp_id) {
            continue;
        }
        last_exp_id = exp_id;

        // clear temporary table
        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psString filter = psMetadataLookupStr(&mdok, row, "filter");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "filter not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warpQuery)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psS64 num = psDBAffectedRows(config->dbh);

        if (num == 0) {
            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id);
            continue;
        }

        if (!available && (num != skycell_count)) {
            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %"
                PRId64 " need %" PRId64, num, skycell_count, warp_id);
            continue;
        }

        // ok we've got one create the diffRun
        diffRunRow *run = diffRunRowAlloc(
                0,          // ID
                "reg",      // state
                workdir,
                label,
                reduction,
                NULL,       // dvodb
                registered,
                tess_id,
                false,                  // bothways
                true,                   // exposure
                false       // magicked
        );

        if (!diffRunInsertObject(config->dbh, run)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(run);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        run->diff_id = psDBLastInsertID(config->dbh);

        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
            return false;
        }
        psFree(query);
        query = NULL;
        psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff");
        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(query);
        query = NULL;

        if (!setdiffRunState(config, run->diff_id, "new", false)) {
            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                run->diff_id);
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psArrayAdd(list, list->n, run);
        numGood++;
    }
    psFree(output);
    psFree(warpQuery);
    psFree(stackQuery);
    psFree(skycell_query);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(list);
        return false;
    }

    if (!diffRunPrintObjects(stdout, list, !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print object");
        psFree(list);
        return false;
    }
    psFree(list);

    return true;
}

static bool definewarpwarpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *selectWhere = psMetadataAlloc();
    psMetadata *insertWhere = psMetadataAlloc();

    // Restrictions for selecting warps
    PXOPT_COPY_S64(config->args, selectWhere, "-warp_id", "inputWarpRun.warp_id", "==");
    PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
    PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "==");
    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "templateRawExp.obs_mode", "==");
    PXOPT_COPY_STR(config->args, selectWhere, "-input_label", "inputWarpRun.label", "==");
    PXOPT_COPY_STR(config->args, selectWhere, "-template_label", "templateWarpRun.label", "==");
    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
    PXOPT_COPY_F32(config->args, selectWhere, "-timediff",
                   "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");

    // Haversine formula for great circle distance
    PXOPT_COPY_F32(config->args, selectWhere, "-distance",
                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");

    PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);

    psMetadataAddF32(selectWhere, PS_LIST_TAIL,
                     "TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs))",
                     PS_META_DUPLICATE_OK, backwards ? "<" : ">", 0.0);

    // Restrictions for inserting skycells
    PXOPT_COPY_F32(config->args, insertWhere, "-good_frac", "inputWarpSkyfile.good_frac", ">=");
    PXOPT_COPY_F32(config->args, insertWhere, "-good_frac", "templateWarpSkyfile.good_frac", ">=");

    // Additional controls
    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);

    // Settings to apply to defined run
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(selectWhere);
        psFree(insertWhere);
        return false;
    }

    if (!rerun) {
        // Need to build table of exposures with diffs
        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
        if (!tempCreate) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            psFree(selectWhere);
            psFree(insertWhere);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
            psFree(tempCreate);
            psFree(selectWhere);
            psFree(insertWhere);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(tempCreate);

        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
        if (!tempInsert) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            psFree(selectWhere);
            psFree(insertWhere);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psString where = psStringCopy(""); // WHERE for insertion
        if (label) {
            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
        }

        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
            psFree(tempInsert);
            psFree(where);
            psFree(selectWhere);
            psFree(insertWhere);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(where);
        psFree(tempInsert);
    }

    // Get list of warps to diff
    psString select = pxDataGet("difftool_definewarpwarp_select.sql");
    if (!select) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        psFree(selectWhere);
        psFree(insertWhere);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }

    psString whereClause = NULL; // The WHERE part of the query

    if (psListLength(selectWhere->list)) {
        psString new = psDBGenerateWhereConditionSQL(selectWhere, NULL);
        psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new);
        psFree(new);
    }
    psFree(selectWhere);

    if (!available) {
        psStringAppend(&whereClause,
                       "\n%s inputWarpRun.state = 'full'"
                       "AND templateWarpRun.state = 'full'",
                       whereClause ? "AND" : "WHERE");
    }

    if (!rerun) {
        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
    }

    if (!p_psDBRunQueryF(config->dbh, select,
                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                         whereClause)) {
        psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
        psFree(select);
        psFree(whereClause);
        psFree(insertWhere);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }
    psFree(select);
    psFree(whereClause);

    psArray *results = p_psDBFetchResult(config->dbh); // Results of query
    if (!results) {
        psErrorCode err = psErrorCodeLast(); // Code for error
        switch (err) {
          case PS_ERR_DB_CLIENT:
            psError(PXTOOLS_ERR_SYS, false, "database error");
            break;
          case PS_ERR_DB_SERVER:
            psError(PXTOOLS_ERR_PROG, false, "database error");
            break;
          default:
            psError(PXTOOLS_ERR_PROG, false, "unknown error");
            break;
        }
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psFree(insertWhere);
        return false;
    }
    if (!psArrayLength(results)) {
        psTrace("difftool", 1, "no rows found");
        psFree(results);
        psFree(insertWhere);
        if (!psDBCommit(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
        return true;
    }

    if (pretend) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, results, "diffRun", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(results);
            return false;
        }
        psFree(results);
        return true;
    }

    psString insert = pxDataGet("difftool_definewarpwarp_insert.sql"); // Insertion for each new run

    if (psListLength(insertWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(insertWhere, NULL);
        psStringAppend(&insert, "\nAND %s", whereClause);
        psFree(whereClause);
    }
    psFree(insertWhere);

    psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print
    long numGood = 0;                   // Number of good rows added
    for (long i = 0; i < results->n; i++) {
        psMetadata *row = results->data[i]; // Result row from query

        psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
        const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
        const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
        if (!input_id || !template || !tess_id) {
            psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
            psFree(list);
            psFree(insert);
            psFree(results);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
                                          tess_id, true, true, false); // Run to insert
        if (!diffRunInsertObject(config->dbh, run)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(run);
            psFree(list);
            psFree(insert);
            psFree(results);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        run->diff_id = psDBLastInsertID(config->dbh); // Difference run identifier

        // Convert identifiers to string, for insertion into query
        psString diff = NULL, input = NULL; // String versions of identifiers
        psStringAppend(&diff, "%" PRId64, run->diff_id);
        psStringAppend(&input, "%" PRId64, input_id);

        if (!p_psDBRunQueryF(config->dbh, insert, diff, input, template, input, template)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(input);
            psFree(diff);
            psFree(run);
            psFree(list);
            psFree(insert);
            psFree(results);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(input);
        psFree(diff);

        if (!setdiffRunState(config, run->diff_id, "new", false)) {
            psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64,
                    run->diff_id);
            psFree(run);
            psFree(list);
            psFree(insert);
            psFree(results);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psArrayAdd(list, list->n, run);
        psFree(run);                    // Drop reference

        numGood++;
    }
    psFree(insert);
    psFree(results);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(list);
        return false;
    }

    if (!diffRunPrintObjects(stdout, list, !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print object");
        psFree(list);
        return false;
    }
    psFree(list);

    return true;
}


static bool pendingcleanuprunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");

    psString query = pxDataGet("difftool_pendingcleanuprun.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupRun", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool pendingcleanupskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    if (diff_id) {
        PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
    }
    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");

    psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupSkyfile", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);
    return true;
}


static bool donecleanupMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("difftool_donecleanup.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffDoneCleanup", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);
    return true;
}

static bool updatediffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");

    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) {
        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
        psFree (where);
        return false;
    }
    psFree (where);

    return true;
}

static bool diffRunComplete(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // look for completed diffRuns
    psString query = pxDataGet("difftool_completed_runs.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }
    for (long i = 0; i < psArrayLength(output); i++) {
        psMetadata *row = output->data[i];

        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
        bool magicked = psMetadataLookupBool(NULL, row, "magicked");

        // set diffRun.state to 'stop'
        if (!setdiffRunState(config, diff_id, "full", magicked)) {
            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                diff_id);
            psFree(output);
            return false;
        }
    }

    return true;
}

bool exportrunMode(pxConfig *config)
{
  typedef struct ExportTable {
    char tableName[80];
    char sqlFilename[80];
  } ExportTable;

  int numExportFiles = 3;

  PS_ASSERT_PTR_NON_NULL(config, NULL);

  PXOPT_LOOKUP_S64(det_id, config->args, "-diff_id", true,  false);
  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);

  FILE *f = fopen (outfile, "w");
  if (f == NULL) {
    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
    return false;
  }

  psMetadata *where = psMetadataAlloc();
  PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");

  ExportTable tables [] = {
    {"diffRun", "difftool_export_run.sql"},
    {"diffInputSkyfile", "difftool_export_input_skyfile.sql"},
    {"diffSkyfile", "difftool_export_skyfile.sql"},
  };

  for (int i=0; i < numExportFiles; i++) {
    psString query = pxDataGet(tables[i].sqlFilename);
    if (!query) {
      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
      return false;
    }

    if (where && psListLength(where->list)) {
      psString whereClause = psDBGenerateWhereSQL(where, NULL);
      psStringAppend(&query, " %s", whereClause);
      psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
      psString limitString = psDBGenerateLimitSQL(limit);
      psStringAppend(&query, " %s", limitString);
      psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
      psError(PS_ERR_UNKNOWN, false, "database error");
      psFree(query);
      return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
      return false;
    }
    if (!psArrayLength(output)) {
      psError(PS_ERR_UNKNOWN, true, "no rows found");
      psFree(output);
      return false;
    }

    if (clean) {
        bool success = true;
        if (!strcmp(tables[i].tableName, "diffRun")) {
            success = pxSetStateCleaned("diffRun", "state", output);
#ifdef notyet
        // diffSkyfile doesn't have dataState yet
        } else if (!strcmp(tables[i].tableName, "diffSkyfile")) {
            success = pxSetStateCleaned("diffSkyfile", "data_state", output);
#endif
        }
        if (!success) {
            psFree(output);
            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
            return false;
        }
    }

    // we must write the export table in non-simple (true) format
    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
      psError(PS_ERR_UNKNOWN, false, "failed to print array");
      psFree(output);
      return false;
    }
    psFree(output);
  }

  fclose (f);

  return true;
}

bool importrunMode(pxConfig *config)
{
  unsigned int nFail;

  int numImportTables = 2;

  char tables[2] [80] = {"diffInputSkyfile", "diffSkyfile"};

  PS_ASSERT_PTR_NON_NULL(config, NULL);

  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);

  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);

  fprintf (stdout, "---- input ----\n");
  psMetadataPrint (stderr, input, 1);

  psMetadataItem *item = psMetadataLookup (input, "diffRun");
  psAssert (item, "entry not in input?");
  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");

  psMetadataItem *entry = psListGet (item->data.list, 0);
  assert (entry);
  assert (entry->type == PS_DATA_METADATA);
  diffRunRow *diffRun = diffRunObjectFromMetadata (entry->data.md);
  diffRunInsertObject (config->dbh, diffRun);

  // fprintf (stdout, "---- diff run ----\n");
  // psMetadataPrint (stderr, entry->data.md, 1);

  for (int i = 0; i < numImportTables; i++) {
    item = psMetadataLookup (input, tables[i]);
    psAssert (item, "entry not in input?");
    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");

    switch (i) {
      case 0:
        for (int i = 0; i < item->data.list->n; i++) {
          entry = psListGet (item->data.list, i);
          assert (entry);
          assert (entry->type == PS_DATA_METADATA);
          diffInputSkyfileRow *diffInputSkyfile = diffInputSkyfileObjectFromMetadata (entry->data.md);
          diffInputSkyfileInsertObject (config->dbh, diffInputSkyfile);

          // fprintf (stdout, "---- row %d ----\n", i);
          // psMetadataPrint (stderr, entry->data.md, 1);
        }
        break;

      case 1:
        for (int i = 0; i < item->data.list->n; i++) {
          entry = psListGet (item->data.list, i);
          assert (entry);
          assert (entry->type == PS_DATA_METADATA);
          diffSkyfileRow *diffSkyfile = diffSkyfileObjectFromMetadata (entry->data.md);
          diffSkyfileInsertObject (config->dbh, diffSkyfile);

          // fprintf (stdout, "---- row %d ----\n", i);
          // psMetadataPrint (stderr, entry->data.md, 1);
        }
        break;
    }
  }

  return true;
}
