/*
 * magicdstool.c
 *
 * Copyright (C) 2006-2007  IfA
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "magicdstool.h"

static bool definebyqueryMode(pxConfig *config);
static psS64 definerunMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool todestreakMode(pxConfig *config);
static bool adddestreakedfileMode(pxConfig *config);
static bool revertdestreakedfileMode(pxConfig *config);
static bool getskycellsMode(pxConfig *config);

static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
static bool magicDSRunComplete(pxConfig *config, bool setmagicked);
static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = magicdstoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY,       definebyqueryMode);
        MODECASE(MAGICDSTOOL_MODE_DEFINERUN,           definerunMode);
        MODECASE(MAGICDSTOOL_MODE_UPDATERUN,           updaterunMode);
        MODECASE(MAGICDSTOOL_MODE_TODESTREAK,          todestreakMode);
        MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE,   adddestreakedfileMode);
        MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode);
        MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS,         getskycellsMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}

static bool definebyqueryMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet");

    return false;
#ifdef notyet

    // Required
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);

    // Optional
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // Create temporary table of the best diffs
    {
        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
        psFree(query);
    }

    // Insert list of best diffs into temporary table
    {
        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        psMetadata *where = psMetadataAlloc();
        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
        PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
        PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");

        psString whereClause = NULL;    // WHERE conditions
        if (psListLength(where->list)) {
            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
            psStringPrepend(&whereClause, "\n AND ");
        }
        psFree(where);

        if (!p_psDBRunQueryF(config->dbh, query, whereClause)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(whereClause);
            psFree(query);
            return false;
        }
        psFree(whereClause);
        psFree(query);
    }

    // Get list of exposures ready to magic
    {
        psString query = pxDataGet("magictool_definebyquery_select.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
        {
            psMetadata *where = psMetadataAlloc();
            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
            PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
            PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");

            if (psListLength(where->list)) {
                magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
                psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
            }
            psFree(where);
        }

        // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
        // told to only take what's available.
        // "new" means we want a new run even if there's already a magic run defined
        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
        PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);

        psString queryWhere = NULL;     // WHERE conditions for entire query
        if (available) {
            psStringAppend(&queryWhere, " WHERE num_done = num_todo");
        }
        if (new) {
            const char *newWhere = " magic_id IS NULL"; // String to add
            if (queryWhere) {
                psStringAppend(&queryWhere, " AND %s", newWhere);
            } else {
                psStringAppend(&queryWhere, " WHERE %s", newWhere);
            }
        }
        if (queryWhere) {
            psStringAppend(&query, " %s", queryWhere);
            psFree(queryWhere);
        }


        if (!p_psDBRunQueryF(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(magicSkyCellNumsWhere);
            psFree(query);
            return false;
        }
        psFree(magicSkyCellNumsWhere);
        psFree(query);
    }

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
          case PS_ERR_DB_CLIENT:
            psError(PXTOOLS_ERR_SYS, false, "database error");
          case PS_ERR_DB_SERVER:
            psError(PXTOOLS_ERR_PROG, false, "database error");
          default:
            psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // Parse the list of exposures ready to magic

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query

    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
    for (long i = 0; i < psArrayLength(output); i++) {
        psMetadata *row = output->data[i]; // Row of interest
        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier

        // create a new magicRun for this group
        magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0);
        if (!run) {
            psAbort("failed to alloc magicRun object");
        }

        if (!magicRunInsertObject(config->dbh, run)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(run);
            psFree(insert);
            psFree(output);
            psFree(list);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier
        run->magic_id = magic_id;

        psArrayAdd(list, list->n, run);
        psFree(run);

        // Create a suitable insertion query for this run
        psString thisInsert = psStringCopy(insert);
        {
            psString idString = NULL;
            psStringAppend(&idString, "%" PRId64, magic_id);
            psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@");
            psFree(idString);
        }
        {
            psString idString = NULL;
            psStringAppend(&idString, "%" PRId64, exp_id);
            psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
            psFree(idString);
        }

        if (!p_psDBRunQueryF(config->dbh, thisInsert, magic_id, exp_id)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(thisInsert);
            psFree(insert);
            psFree(output);
            psFree(list);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(thisInsert);
    }

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(output);

    if (!magicRunPrintObjects(stdout, list, !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print object");
        psFree(list);
        return false;
    }

    psFree(list);

    return true;
#endif // notyet
}

static psS64 definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
    PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false);

    // optional
    PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
    PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false);
    PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);

    psS64 stage_id = 0, cam_id = 0;

    if (!magicDSGetIDs(config, stage, magic_id, &stage_id, &cam_id)) {
        psError(PS_ERR_UNKNOWN, false, "failed to get ids");
        return false;
    }

    magicDSRunRow *run = magicDSRunRowAlloc(
            0,          // ID
            magic_id,
            "new",      // state
            stage,
            stage_id,
            cam_id,
            label,
            outroot,
            recoveryroot,
            re_place,
            remove
    );

    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object");
        return false;
    }
    if (!magicDSRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return false;
    }

    psS64 magic_ds_id = psDBLastInsertID(config->dbh);
    run->magic_ds_id = magic_ds_id;

    if (!magicDSRunPrintObject(stdout, run, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(run);
            return false;
    }

    psFree(run);

    return magic_id;
}


static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);

    if (state) {
        // set detRun.state to state
        return setmagicDSRunState(config, magic_ds_id, state);
    }

    return true;
}


static bool todestreakMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    pxAddLabelSearchArgs (config, where, "-label", "label", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // look for "inputs" that need to processed
    psString query = pxDataGet("magicdstool_todestreak.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " WHERE %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "todestreak", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool
setMagicked(pxConfig *config, psS64 magic_ds_id, psString component)
{
    // first query the magicDSRun to find the stage and the stage_id
    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;

    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magicdstool", PS_LOG_INFO, "magic_ds_id: %" PRId64 " not found", magic_ds_id);
        psFree(output);
        return true;
    }
    if (psArrayLength(output) > 1) {
        psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_ds_id %" PRId64,
            psArrayLength(output), magic_ds_id);
        psFree(output);
        return false;
    }
    psMetadata *row = output->data[0];

    psString stage = psMetadataLookupStr(NULL, row, "stage");
    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");


    // chose the appropriate query based on the stage
    if (!strcmp(stage, "raw")) {
        query = "UPDATE rawImfile SET magicked = 1 where exp_id = %" PRId64 " AND class_id = '%s'";
    } else if (!strcmp(stage, "chip")) {
        query = "UPDATE chipProcessedImfile SET magicked = 1 where chip_id = %" PRId64 " AND class_id = '%s'";
    } else if (!strcmp(stage, "warp")) {
        query = "UPDATE warpSkyfile SET magicked = 1 where warp_id = %" PRId64 " AND skycell_id = '%s'";
    } else if (!strcmp(stage, "diff")) {
        query = "UPDATE diffSkyfile SET magicked = 1 where diff_id = %" PRId64 " AND skycell_id = '%s'";
    } else {
        psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
        psFree(output);
        return false;
    }
    if (!p_psDBRunQueryF(config->dbh, query, stage_id, component)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(output);

    psU64 affected = psDBAffectedRows(config->dbh);
    if (affected != 1) {
        psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
        return false;
    }

    return true;
}

static bool
setRunMagicked(pxConfig *config, psS64 magic_ds_id)
{
    // first query the magicDSRun to find the stage and the stage_id
    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;

    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psError(PS_ERR_UNKNOWN, true, "magicDSRun not found for magic_ds_id %" PRId64, magic_ds_id);
        psFree(output);
        return false;
    }
    if (psArrayLength(output) > 1) {
        psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_ds_id %" PRId64,
            psArrayLength(output), magic_ds_id);
        psFree(output);
        return false;
    }
    psMetadata *row = output->data[0];

    psString stage = psMetadataLookupStr(NULL, row, "stage");
    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");


    // chose the appropriate query based on the stage
    if (!strcmp(stage, "raw")) {
        query = "UPDATE rawExp SET magicked = 1 where exp_id = %" PRId64;
    } else if (!strcmp(stage, "chip")) {
        query = "UPDATE chipRun SET magicked = 1 where chip_id = %" PRId64;
    } else if (!strcmp(stage, "warp")) {
        query = "UPDATE warpRun SET magicked = 1 where warp_id = %" PRId64;
    } else if (!strcmp(stage, "diff")) {
        query = "UPDATE diffRun SET magicked = 1 where diff_id = %" PRId64;
    } else {
        psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
        psFree(output);
        return false;
    }
    if (!p_psDBRunQueryF(config->dbh, query, stage_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(output);

    psU64 affected = psDBAffectedRows(config->dbh);
    if (affected != 1) {
        psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
        return false;
    }

    return true;
}

static bool adddestreakedfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required values
    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);

    // default values
    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false);
    PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false);
    PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false);

    if (setmagicked && (fault != 0)) {
        psError(PS_ERR_UNKNOWN, true, " cannot setmagicked for faulted file");
        return false;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (setmagicked) {
        // set the image file's magicked flag
        if (!setMagicked(config, magic_ds_id, component)) {
            psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
    }

    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault)) {
            // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!magicDSRunComplete(config, setmagicked)) {
            // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(stage, false);
    PS_ASSERT_PTR_NON_NULL(stage_id, false);
    PS_ASSERT_PTR_NON_NULL(cam_id, false);

    if (!strcmp(stage, "diff")) {
        // don't need these ids for diff stage because diff_id is in the magicRun
        *stage_id = 0;
        *cam_id = 0;
        return true;
    }

    int stageNum;
    if (!strcmp(stage, "raw")) {
        stageNum = 0;
    } else if (!strcmp(stage, "chip")) {
        stageNum = 1;
    } else if (!strcmp(stage, "warp")) {
        stageNum = 2;
    } else {
        psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage);
        return false;
    }


    psString query = pxDataGet("magicdstool_getrunids.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (!p_psDBRunQueryF(config->dbh, query, magic_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }
    if (psArrayLength(output) > 1) {
        psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_id %" PRId64,
            psArrayLength(output), magic_id);
        return false;
    }
    psMetadata *row = output->data[0];

    *cam_id = psMetadataLookupS64(NULL, row, "cam_id");
    if (stageNum == 0) {
        *stage_id = psMetadataLookupS64(NULL, row, "exp_id");
    } else if (stageNum == 1) {
        *stage_id = psMetadataLookupS64(NULL, row, "chip_id");
    } else if (stageNum == 2) {
        *stage_id = psMetadataLookupS64(NULL, row, "warp_id");
    }

    return true;
}

static bool magicDSRunComplete(pxConfig *config, bool setmagicked)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // look for completed magicDSRuns
    psString query = pxDataGet("magicdstool_completed_runs.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }
    for (long i = 0; i < psArrayLength(output); i++) {
        psMetadata *row = output->data[i];

        psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");

        // if requested, set stageRun.magicked
        if (setmagicked && !setRunMagicked(config, magic_ds_id)) {
            psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
                magic_ds_id);
            return false;
        }

        // set magicDSRun.state to 'full'
        if (!setmagicDSRunState(config, magic_ds_id, "full")) {
            psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
                magic_ds_id);
            psFree(output);
            return false;
        }
    }


    return true;
}


static bool revertdestreakedfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
    PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");

    psString query = psStringCopy("DELETE FROM magicDSFile WHERE fault != 0");

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "failed to revert");
        return false;
    }
    return true;
}

static bool getskycellsMode(pxConfig *config)
{
    // required
    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");

    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("magicdstool_getskycells.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}

static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
{
    PS_ASSERT_PTR_NON_NULL(state, false);

    // check that state is a valid string value
    if (!(
            (strncmp(state, "new", 4) == 0)
            || (strncmp(state, "full", 5) == 0)
        )
    ) {
        psError(PS_ERR_UNKNOWN, false,
                "invalid magicDSRun state: %s", state);
        return false;
    }

    char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
    if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) {
        psError(PS_ERR_UNKNOWN, false,
                "failed to change state for magic_id %" PRId64, magic_ds_id);
        return false;
    }

    return true;
}

