Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 26911) +++ trunk/ippTools/share/Makefile.am (revision 26912) @@ -195,4 +195,5 @@ magicdstool_definebyquery_warp.sql \ magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_chip.sql \ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ Index: trunk/ippTools/share/magicdstool_definecopy_chip.sql =================================================================== --- trunk/ippTools/share/magicdstool_definecopy_chip.sql (revision 26912) +++ trunk/ippTools/share/magicdstool_definecopy_chip.sql (revision 26912) @@ -0,0 +1,33 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'chip' AS stage, + chipRun.chip_id AS stage_id, + camRun.cam_id, + chipRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun + ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun AS oldCamRun USING(cam_id) + -- This ends the usual JOIN down to the raw level + JOIN chipRun + ON chipRun.exp_id = magicRun.exp_id + AND chipRun.chip_id != oldCamRun.chip_id -- Not allowed to copy yourself + JOIN camRun ON + camRun.chip_id = chipRun.chip_id + JOIN rawExp + ON rawExp.exp_id = chipRun.exp_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'chip' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND chipRun.magicked = 0 + AND chipRun.state = 'full' + AND camRun.state = 'full' Index: trunk/ippTools/src/magicdstool.c =================================================================== --- trunk/ippTools/src/magicdstool.c (revision 26911) +++ trunk/ippTools/src/magicdstool.c (revision 26912) @@ -33,4 +33,5 @@ static bool definebyqueryMode(pxConfig *config); +static bool definecopyMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todestreakMode(pxConfig *config); @@ -65,4 +66,5 @@ switch (config->mode) { MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MAGICDSTOOL_MODE_DEFINECOPY, definecopyMode); MODECASE(MAGICDSTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); @@ -127,5 +129,5 @@ ippStage stageNum = ippStringToStage(stage); - + psString query = NULL; switch (stageNum) { @@ -249,5 +251,5 @@ psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); - + // if workdir is not supplied use the magicRun's if (!workdir) { @@ -302,4 +304,224 @@ if (!dry_run && !psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicDSRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); + + return true; +} + + +// XXX This currently allows multiple destreak runs to be queued on the same exposure if there are multiple +// magicRuns selected! +static bool definecopyMode(pxConfig *config) +{ + // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // Optional + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + pretend = true; + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "chipRun.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_S32(config->args, where, "-streaks_max", "magicMask.streaks", "<="); + + pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label + psString labelName = NULL; // Name of label + psStringAppend(&labelName, "%sRun.label", stage); + pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label + psFree(labelName); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Raw stage is not appropriate for a copied destreak"); + return false; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definecopy_chip.sql"); + break; + case IPP_STAGE_CAMERA: + case IPP_STAGE_WARP: + case IPP_STAGE_DIFF: + case IPP_STAGE_FAKE: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "%s has not been coded.", stage); + return false; + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_DATA, true, "Stacks do not need to be destreaked"); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + const char *rerun_flag = rerun ? "\n" : ""; // String to give query to activate (or not) rerun + + if (stageNum != IPP_STAGE_DIFF) { + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + } else { + // diff stage query has two types bothways and !bothways + // so we need to send the rerun flag and the where data twice + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // Parse the list of runs ready to be destreaked + + if (!pretend && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); + + // create a new magicRun for this group + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + inv_magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, + outroot, + recoveryroot, + noreplace ? 0 :1, // re_place + 0, // remove + 0, // fault + note); // remove + + psFree(outroot); + if (!run) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!pretend) { + if (!magicDSRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; + } + + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!pretend && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -829,6 +1051,6 @@ for (int i=0; idata[i]; - psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - psString old_state = psMetadataLookupStr(NULL, row, "state"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); psString new_state; if (!strcmp(old_state, "goto_censored")) { Index: trunk/ippTools/src/magicdstool.h =================================================================== --- trunk/ippTools/src/magicdstool.h (revision 26911) +++ trunk/ippTools/src/magicdstool.h (revision 26912) @@ -26,4 +26,5 @@ MAGICDSTOOL_MODE_NONE = 0x0, MAGICDSTOOL_MODE_DEFINEBYQUERY, + MAGICDSTOOL_MODE_DEFINECOPY, MAGICDSTOOL_MODE_UPDATERUN, MAGICDSTOOL_MODE_TODESTREAK, Index: trunk/ippTools/src/magicdstoolConfig.c =================================================================== --- trunk/ippTools/src/magicdstoolConfig.c (revision 26911) +++ trunk/ippTools/src/magicdstoolConfig.c (revision 26912) @@ -66,8 +66,31 @@ psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one arleady exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -definecopy + psMetadata *definecopyArgs = psMetadataAlloc(); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage_label", PS_META_DUPLICATE_OK, "search by stageRun.label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-magic_label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); + psMetadataAddS32(definecopyArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definecopyArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -updaterun @@ -166,4 +189,6 @@ PXOPT_ADD_MODE("-definebyquery", "create magic de-streak runs from magic runs", MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definecopy", "create magic de-streak runs from magic runs for different path", + MAGICDSTOOL_MODE_DEFINECOPY, definecopyArgs); PXOPT_ADD_MODE("-updaterun", "update state of magic de-streak run", MAGICDSTOOL_MODE_UPDATERUN, updaterunArgs);