Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 23742) +++ trunk/ippTools/share/Makefile.am (revision 23765) @@ -92,4 +92,11 @@ difftool_skyfile.sql \ difftool_todiffskyfile.sql \ + disttool_definebyquery_camera.sql \ + disttool_definebyquery_chip.sql \ + disttool_definebyquery_diff.sql \ + disttool_definebyquery_fake.sql \ + disttool_definebyquery_raw.sql \ + disttool_definebyquery_stack.sql \ + disttool_definebyquery_warp.sql \ disttool_pendingcomponent.sql \ disttool_processedcomponent.sql \ Index: trunk/ippTools/share/disttool_definebyquery_camera.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_camera.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_camera.sql (revision 23765) @@ -0,0 +1,18 @@ +SELECT + 'cam' as stage, + camRun.cam_id as stage_id, + rawExp.exp_name as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM camRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'camera' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'camera' AND camRun.cam_id = distRun.stage_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL -- no existing distRun + AND ((!distTarget.clean AND camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_definebyquery_chip.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_chip.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_chip.sql (revision 23765) @@ -0,0 +1,17 @@ +SELECT + 'chip' as stage, + chipRun.chip_id as stage_id, + rawExp.exp_name as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM chipRun +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'chip' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distTarget.stage = 'chip' AND distRun.stage_id = chipRun.chip_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL -- no existing distRun + AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_definebyquery_diff.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_diff.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_diff.sql (revision 23765) @@ -0,0 +1,25 @@ +SELECT DISTINCT + 'diff' as stage, + diffRun.diff_id AS stage_id, + rawExp.exp_name as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM diffRun +JOIN diffInputSkyfile using(diff_id) +JOIN warpRun on diffInputSkyfile.warp1 = warpRun.warp_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(exp_id, chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'diff' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'diff' AND (distRun.stage_id = diff_id) + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned')) + + + Index: trunk/ippTools/share/disttool_definebyquery_fake.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_fake.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_fake.sql (revision 23765) @@ -0,0 +1,18 @@ +SELECT + 'fake' as stage, + fakeRun.fake_id AS stage_id, + rawExp.exp_name as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM fakeRun +JOIN camRun USING(cam_id) -- ON fakeRun.cam_id = camRun.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'fake' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'fake' AND (distRun.stage_id = fake_id) +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_definebyquery_raw.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_raw.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_raw.sql (revision 23765) @@ -0,0 +1,17 @@ +SELECT DISTINCT -- we use distinct to handle multiple chip runs. Distribution will find right chipRun + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawExp.exp_name AS run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM rawExp +JOIN chipRun USING(exp_id) +JOIN distTarget ON distTarget.obs_mode = rawExp.obs_mode AND distTarget.stage = 'raw' +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'raw' AND distRun.stage_id = exp_id + -- JOIN hook for magicked stuff %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL -- no existing distRun + Index: trunk/ippTools/share/disttool_definebyquery_select.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_select.sql (revision 23742) +++ (revision ) @@ -1,124 +1,0 @@ --- Maybe it would be better to split the query into multiple modes --- the way it is outstanding rawExp's to bundle will block other stages --- from being processed -SELECT - stage, - stage_id, - obs_mode, - clean -FROM ( - SELECT DISTINCT -- we use distinct to handle multiple chip runs. Distribution will find right one - 'raw' AS stage, - rawExp.exp_id AS stage_id, - rawExp.obs_mode, - distTarget.clean - FROM rawExp - JOIN chipRun USING(exp_id) - JOIN distTarget ON distTarget.obs_mode = rawExp.obs_mode AND distTarget.stage = 'raw' - JOIN rcInterest USING(target_id) - LEFT JOIN distRun ON distRun.stage = 'raw' AND distRun.stage_id = exp_id - WHERE distTarget.state = 'enabled' - AND rcInterest.state = 'enabled' - AND chipRun.state = 'full' -- Note: we need a completed chip stage magicDSRun because we - -- need the mask file to NAN masked pixels - -- to support cameras that don't need magic disttool adds the magicked restriction - -- \nAND rawExp.magicked AND chipRun.magicked here: - -- raw magicked HOOK %s - AND distRun.dist_id IS NULL -- no existing distRun - -- raw where hook %s - -UNION - SELECT - 'chip' as stage, - chipRun.chip_id as stage_id, - rawExp.obs_mode, - distTarget.clean - FROM chipRun - JOIN rawExp USING(exp_id) - JOIN distTarget ON distTarget.stage = 'chip' AND rawExp.obs_mode = distTarget.obs_mode - JOIN rcInterest USING(target_id) - LEFT JOIN distRun ON distTarget.stage = 'chip' AND distRun.stage_id = chipRun.chip_id - WHERE distTarget.state = 'enabled' - AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun - AND ((!distTarget.clean AND (chipRun.state = 'full') - -- If camera requires files to be magicked before distibution - -- the following gets added by disttool - AND chipRun.magicked - -- chip magicked HOOK %s - ) - OR (distTarget.clean AND chipRun.state = 'cleaned')) - -- chip where hook %s - -UNION - SELECT - 'cam' as stage, - camRun.cam_id as stage_id, - rawExp.obs_mode, - distTarget.clean - FROM camRun - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - JOIN distTarget ON distTarget.stage = 'cam' AND rawExp.obs_mode = distTarget.obs_mode - JOIN rcInterest USING(target_id) - LEFT JOIN distRun ON distRun.stage = 'cam' AND camRun.cam_id = distRun.stage_id - WHERE distTarget.state = 'enabled' - AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun - AND ((!distTarget.clean AND camRun.state = 'full' - -- need magicked chip run because camera mask are destreaked by chip destreaking - -- the following line - -- AND chipRun.magicked - -- cam magicked HOOK %s - ) - OR (distTarget.clean AND camRun.state = 'cleaned') - ) - -- cam where hook %s - -UNION - SELECT - 'fake' as stage, - fakeRun.fake_id as stage_id, - rawExp.obs_mode, - distTarget.clean - FROM fakeRun - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - JOIN distTarget ON rawExp.obs_mode = distTarget.obs_mode AND distTarget.stage = 'fake' - JOIN rcInterest USING(target_id) - LEFT JOIN distRun ON distRun.stage = 'fake' AND camRun.cam_id = distRun.stage_id - WHERE distTarget.state = 'enabled' - AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun - AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned')) - -- fake where hook %s - -UNION - SELECT - 'warp' as stage, - warpRun.warp_id AS stage_id, - rawExp.obs_mode, - distTarget.clean - FROM warpRun - JOIN fakeRun USING(fake_id) - JOIN camRun ON fakeRun.cam_id = camRun.cam_id - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - JOIN distTarget ON distTarget.stage = 'warp' AND rawExp.obs_mode = distTarget.obs_mode - JOIN rcInterest USING(target_id) - LEFT JOIN distRun ON distRun.stage = 'warp' AND (distRun.stage_id = warp_id) - WHERE distTarget.state = 'enabled' - AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL - AND ((warpRun.state = 'full' - -- AND warpRun.magicked - -- warp magicked hook %s - ) - OR (distTarget.clean AND warpRun.state = 'cleaned') - ) - -- warp where hook %s -) as foo - - - Index: trunk/ippTools/share/disttool_definebyquery_stack.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_stack.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_stack.sql (revision 23765) @@ -0,0 +1,23 @@ +SELECT DISTINCT + 'stack' as stage, + stackRun.stack_id AS stage_id, + -- run tag in the form 'stack.$skycell_id.$stack_id' + CONCAT('stack.', stackRun.skycell_id, '.', convert(stackRun.stack_id, CHAR)) as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM stackRun +JOIN stackInputSkyfile using(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'stack' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'stack' AND (distRun.stage_id = stack_id) + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_definebyquery_warp.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_warp.sql (revision 23765) +++ trunk/ippTools/share/disttool_definebyquery_warp.sql (revision 23765) @@ -0,0 +1,22 @@ +SELECT + 'warp' as stage, + warpRun.warp_id AS stage_id, + rawExp.exp_name as run_tag, + rawExp.obs_mode, + distTarget.target_id, + distTarget.clean +FROM warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun ON fakeRun.cam_id = camRun.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'warp' AND rawExp.obs_mode = distTarget.obs_mode +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage = 'warp' AND (distRun.stage_id = warp_id) +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned')) + + + Index: trunk/ippTools/src/disttool.c =================================================================== --- trunk/ippTools/src/disttool.c (revision 23742) +++ trunk/ippTools/src/disttool.c (revision 23765) @@ -132,6 +132,232 @@ static bool definebyqueryMode(pxConfig *config) { - psError(PS_ERR_PROGRAMMING, true, "-definebyquery is not implemented yet"); - return false; + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + + // optional + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); + + PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); + + // select arguments + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-target_id", "distTarget.target_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); + + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + psString query = NULL; + psString runType = NULL; + psString runJoinStr = NULL; + if (!strcmp(stage, "raw")) { + runType = "rawExp"; + runJoinStr = "rawExp.exp_id"; + query = pxDataGet("disttool_definebyquery_raw.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + } else if (!strcmp(stage, "chip")) { + runType = "chipRun"; + runJoinStr = "chipRun.chip_id"; + query = pxDataGet("disttool_definebyquery_chip.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (chipRun.label = '%s')", label); + } + } else if (!strcmp(stage, "camera")) { + runType = "camRun"; + query = pxDataGet("disttool_definebyquery_camera.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (camRun.label = '%s')", label); + } + } else if (!strcmp(stage, "fake")) { + runType = "fakeRun"; + query = pxDataGet("disttool_definebyquery_fake.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (fakeRun.label = '%s')", label); + } + // fake stage doesn't require magic + no_magic = true; + } else if (!strcmp(stage, "warp")) { + runType = "warpRun"; + runJoinStr = "warpRun.warp_id"; + query = pxDataGet("disttool_definebyquery_warp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (warpRun.label = '%s')", label); + } + + } else if (!strcmp(stage, "diff")) { + runType = "diffRun"; + runJoinStr = "diffRun.diff_id"; + query = pxDataGet("disttool_definebyquery_diff.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (diffRun.label = '%s')", label); + } + + } else if (!strcmp(stage, "stack")) { + runType = "stackRun"; + query = pxDataGet("disttool_definebyquery_stack.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (stackRun.label = '%s')", label); + } + // stack stage doesn't require magic (perhaps let the script do this? + no_magic = true; + } else { + psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString joinHook = NULL; + + if (!no_magic) { + psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", runType); + + // is selecting by magic_ds_id really interesting? + if (magic_ds_id) { + if (strcmp(stage, "camera")) { + // stage other than camera + if (!runJoinStr) { + psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); + psFree(query); + return false; + } + psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" + " AND magicDSRun.stage_id = %s", runJoinStr); + } else { + // camera masks are magicked when the chipRun is magicked + // XXX: This is confusing. Is it dangerous? + // Maybe I should add a magicked bit to camRun. Note this isn't + psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'" + " AND magicDSRun.stage_id = chipRun.chip_id"); + } + psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); + } + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, joinHook ? joinHook : "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psFree(joinHook); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // XXX Remove this debugging output at some point + if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + if (dry_run) { + psFree(output); + return true; + } + + for (long i=0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psString stage = psMetadataLookupStr(NULL, md, "stage"); + psString run_tag = psMetadataLookupStr(NULL, md, "run_tag"); + psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); + bool clean = psMetadataLookupBool(NULL, md, "clean"); + + psString outroot = NULL; + psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); + + if (!distRunInsert(config->dbh, + 0, // dist_id + target_id, + stage, + stage_id, + set_label, + outroot, + clean, + no_magic, + "new", + NULL, // time_stamp + 0 // fault + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(outroot); + psFree(output); + return false; + } + psFree(outroot); + } + + psFree(output); + + return true; } Index: trunk/ippTools/src/disttoolConfig.c =================================================================== --- trunk/ippTools/src/disttoolConfig.c (revision 23742) +++ trunk/ippTools/src/disttoolConfig.c (revision 23765) @@ -42,4 +42,24 @@ return NULL; } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage", 0, "define stage for bundle (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false); + + // select args + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "select by obs_mode", NULL); + + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -definerun @@ -107,4 +127,5 @@ psMetadata *modes = psMetadataAlloc(); + PXOPT_ADD_MODE("-definebyquery", "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-definerun", "", DISTTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs);