Index: trunk/ippTools/src/magicdstool.c =================================================================== --- trunk/ippTools/src/magicdstool.c (revision 24709) +++ trunk/ippTools/src/magicdstool.c (revision 24882) @@ -2,5 +2,5 @@ * magicdstool.c * - * Copyright (C) 2006-2007 IfA + * Copyright (C) 2006-2009 IfA * * This program is free software; you can redistribute it and/or modify it @@ -27,4 +27,5 @@ #include #include +#include #include "pxtools.h" @@ -97,119 +98,88 @@ static bool definebyqueryMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet"); - - return false; -#ifdef notyet // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + // Optional PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); - - // Optional - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - // Create temporary table of the best diffs - { - psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + query = pxDataGet("magicdstool_definebyquery_raw.sql"); + break; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definebyquery_chip.sql"); + break; + case IPP_STAGE_CAMERA: + query = pxDataGet("magicdstool_definebyquery_camera.sql"); + break; + case IPP_STAGE_WARP: + query = pxDataGet("magicdstool_definebyquery_warp.sql"); + break; + case IPP_STAGE_DIFF: + query = pxDataGet("magicdstool_definebyquery_diff.sql"); + break; + case IPP_STAGE_FAKE: + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + dry_run = true; + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun ? "\n1 " : "\n0 ")) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - } - - // Insert list of best diffs into temporary table - { - psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - psString whereClause = NULL; // WHERE conditions - if (psListLength(where->list)) { - whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); - } - psFree(where); - - if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereClause); - psFree(query); - return false; - } - psFree(whereClause); - psFree(query); - } - - // Get list of exposures ready to magic - { - psString query = pxDataGet("magictool_definebyquery_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString magicSkyCellNumsWhere = NULL; // WHERE conditions for magicSkyCellNums - { - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - if (psListLength(where->list)) { - magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&magicSkyCellNumsWhere, "\n AND "); - } - psFree(where); - } - - // "available" means only concern ourselves with exposures that have all diffs completed, unless we're - // told to only take what's available. - // "new" means we want a new run even if there's already a magic run defined - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); - PXOPT_LOOKUP_BOOL(new, config->args, "-new", false); - - psString queryWhere = NULL; // WHERE conditions for entire query - if (available) { - psStringAppend(&queryWhere, " WHERE num_done = num_todo"); - } - if (new) { - const char *newWhere = " magic_id IS NULL"; // String to add - if (queryWhere) { - psStringAppend(&queryWhere, " AND %s", newWhere); - } else { - psStringAppend(&queryWhere, " WHERE %s", newWhere); - } - } - if (queryWhere) { - psStringAppend(&query, " %s", queryWhere); - psFree(queryWhere); - } - - - if (!p_psDBRunQueryF(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(magicSkyCellNumsWhere); - psFree(query); - return false; - } - psFree(magicSkyCellNumsWhere); - psFree(query); - } + return false; + } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -233,72 +203,69 @@ } - // Parse the list of exposures ready to magic - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + // Parse the list of runs ready to be destreaked + + if (!dry_run && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; // Row of interest - psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); + + // if workdir is not supplied use the magicRun's + if (!workdir) { + workdir = magicRunWorkdir; + } + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0); + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + outroot, + recoveryroot, + re_place, + 0); // remove + + psFree(outroot); if (!run) { - psAbort("failed to alloc magicRun object"); - } - - if (!magicRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!dry_run) { + if (!magicDSRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; } - return false; - } - - psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier - run->magic_id = magic_id; + } psArrayAdd(list, list->n, run); psFree(run); - - // Create a suitable insertion query for this run - psString thisInsert = psStringCopy(insert); - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, magic_id); - psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); - psFree(idString); - } - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, exp_id); - psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); - psFree(idString); - } - - if (!p_psDBRunQueryF(config->dbh, thisInsert, magic_id, exp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(thisInsert); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(thisInsert); - } - - if (!psDBCommit(config->dbh)) { + } + + if (!dry_run && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -306,5 +273,5 @@ psFree(output); - if (!magicRunPrintObjects(stdout, list, !simple)) { + if (!magicDSRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -315,5 +282,4 @@ return true; -#endif // notyet } @@ -405,4 +371,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -501,27 +468,33 @@ psMetadata *row = output->data[0]; - psString stage = psMetadataLookupStr(NULL, row, "stage"); + psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); - if (!strcmp(stage, "camera")) { - // no there is no magicked column in camProcessedExp + ippStage stageNum = ippStringToStage(stage); + + // chose the appropriate query based on the stage + switch (stageNum) { + case IPP_STAGE_RAW: + query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CHIP: + query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CAMERA: + // no there is no magicked column in camProcessedExp so we have nothing to do psFree(output); return true; - } - - // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { - query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "warp")) { + case IPP_STAGE_WARP: query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; - } else if (!strcmp(stage, "diff")) { + break; + case IPP_STAGE_DIFF: query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; - } else { + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); return false; } + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -571,17 +544,24 @@ psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + ippStage stageNum = ippStringToStage(stage); // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { + switch (stageNum) { + case IPP_STAGE_RAW: query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64; - } else if (!strcmp(stage, "chip")) { + break; + case IPP_STAGE_CHIP: query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64; - } else if (!strcmp(stage, "camera")) { + break; + case IPP_STAGE_CAMERA: query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64; - } else if (!strcmp(stage, "warp")) { + break; + case IPP_STAGE_WARP: query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64; - } else if (!strcmp(stage, "diff")) { + break; + case IPP_STAGE_DIFF: query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64; - } else { + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); @@ -671,25 +651,9 @@ PS_ASSERT_PTR_NON_NULL(cam_id, false); - if (!strcmp(stage, "diff")) { - // don't need these ids for diff stage because diff_id is in the magicRun - *stage_id = 0; - *cam_id = 0; - return true; - } - - int stageNum; - if (!strcmp(stage, "raw")) { - stageNum = 0; - } else if (!strcmp(stage, "chip")) { - stageNum = 1; - } else if (!strcmp(stage, "camera")) { - stageNum = 2; - } else if (!strcmp(stage, "warp")) { - stageNum = 3; - } else { - psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage); - return false; - } - + int stageNum = ippStringToStage(stage);; + if (stageNum == IPP_STAGE_NONE) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); + return false; + } psString query = pxDataGet("magicdstool_getrunids.sql"); @@ -724,12 +688,20 @@ *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - if (stageNum == 0) { + switch (stageNum) { + case IPP_STAGE_RAW: *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - } else if (stageNum == 1) { + break; + case IPP_STAGE_CHIP: *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - } else if (stageNum == 2) { + break; + case IPP_STAGE_CAMERA: *stage_id = *cam_id; - } else if (stageNum == 3) { + return true; + case IPP_STAGE_WARP: *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); + break; + case IPP_STAGE_DIFF: + *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); + break; }