Index: trunk/ippTools/configure.ac =================================================================== --- trunk/ippTools/configure.ac (revision 20934) +++ trunk/ippTools/configure.ac (revision 20973) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.46]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.47]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 20934) +++ trunk/ippTools/share/Makefile.am (revision 20973) @@ -70,5 +70,8 @@ dettool_toresidimfile.sql \ dettool_tostacked.sql \ - difftool_definebyquery.sql \ + difftool_completed_runs.sql \ + difftool_definebyquery_part1.sql \ + difftool_definebyquery_part2.sql \ + difftool_definebyquery_temp_create.sql \ difftool_donecleanup.sql \ difftool_inputskyfile.sql \ Index: trunk/ippTools/share/difftool_completed_runs.sql =================================================================== --- trunk/ippTools/share/difftool_completed_runs.sql (revision 20973) +++ trunk/ippTools/share/difftool_completed_runs.sql (revision 20973) @@ -0,0 +1,17 @@ +SELECT DISTINCT + diff_id +FROM ( + SELECT + COUNT(diffInputSkyfile.skycell_id), COUNT(diffSkyfile.skycell_id), + diffSkyfile.* + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + LEFT JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE + diffRun.state = 'new' + GROUP BY + diffInputSkyfile.diff_id + HAVING + COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) + AND SUM(diffSkyfile.fault) = 0 + ) as Foo Index: trunk/ippTools/share/difftool_definebyquery.sql =================================================================== --- trunk/ippTools/share/difftool_definebyquery.sql (revision 20934) +++ trunk/ippTools/share/difftool_definebyquery.sql (revision 20973) @@ -9,7 +9,7 @@ warpsToDiff.good_frac, warpsToDiff.diff_id, - warpsToDiff.kind, current_stack_id, - best_stack_id + best_stack_id, + exp_id FROM ( -- Get list of warps that can be diffed, with any associated diff @@ -23,6 +23,6 @@ warpRun.label as warp_label, diffInputs.diff_id, - diffInputs.kind, - diffTemplates.stack_id AS current_stack_id + diffInputs.stack2 AS current_stack_id, + rawExp.exp_id FROM warpSkyfile JOIN warpRun USING(warp_id) @@ -33,11 +33,7 @@ -- Check if it has an associated diff LEFT JOIN diffInputSkyfile AS diffInputs - ON diffInputs.warp_id = warpSkyfile.warp_id + ON diffInputs.warp1 = warpSkyfile.warp_id AND diffInputs.skycell_id = warpSkyfile.skycell_id - AND diffInputs.template = 0 -- only join input files - -- Get the stack_id currently used as a template, if any - LEFT JOIN diffInputSkyfile AS diffTemplates - ON diffTemplates.diff_id = diffInputs.diff_id - AND diffTemplates.template != 0 -- only join template files + AND diffInputs.stack2 IS NOT NULL WHERE warpSkyfile.fault = 0 Index: trunk/ippTools/share/difftool_definebyquery_part1.sql =================================================================== --- trunk/ippTools/share/difftool_definebyquery_part1.sql (revision 20973) +++ trunk/ippTools/share/difftool_definebyquery_part1.sql (revision 20973) @@ -0,0 +1,22 @@ + SELECT + exp_id, + warp_id, + rawExp.filter, + warpRun.label, + warpRun.tess_id, + count(skycell_id) as skycell_count + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING (chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN diffRun USING(exp_id) + + WHERE + warpSkyfile.ignored = 0 + -- warp where hook %s + -- exp where hook %s + -- diff where hook %s + + GROUP BY exp_id, warp_id DESC Index: trunk/ippTools/share/difftool_definebyquery_part2.sql =================================================================== --- trunk/ippTools/share/difftool_definebyquery_part2.sql (revision 20973) +++ trunk/ippTools/share/difftool_definebyquery_part2.sql (revision 20973) @@ -0,0 +1,33 @@ +-- insert skycells to diff +INSERT INTO skycellsToDiff + SELECT + 0, -- diff_id + warpSkyfile.skycell_id, + warp_id, -- warp1 + NULL, -- stack1 + NULL, -- warp2 + max_stack_id, -- stack2 + warpSkyfile.tess_id + FROM warpSkyfile + JOIN warpRun USING(warp_id) + LEFT JOIN ( + SELECT + MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment + stackRun.skycell_id, + stackRun.tess_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + GROUP BY + skycell_id, + filter + ) as bestStacks + USING(skycell_id) + WHERE + warp_id = %lld + AND ignored = 0 + AND filter = '%s' + AND warpSkyfile.tess_id = bestStacks.tess_id + -- (good fraction test goes here for example also above) + -- warp where hook %s + -- stacks where hook %s Index: trunk/ippTools/share/difftool_definebyquery_temp_create.sql =================================================================== --- trunk/ippTools/share/difftool_definebyquery_temp_create.sql (revision 20973) +++ trunk/ippTools/share/difftool_definebyquery_temp_create.sql (revision 20973) @@ -0,0 +1,9 @@ +CREATE TEMPORARY TABLE skycellsToDiff ( +diff_id BIGINT, +skycell_id VARCHAR(64), +warp1 BIGINT, +stack1 BIGINT, +warp2 BIGINT, +stack2 BIGINT, +tess_id VARCHAR(64) +) ENGINE=MEMORY; Index: trunk/ippTools/share/difftool_inputskyfile.sql =================================================================== --- trunk/ippTools/share/difftool_inputskyfile.sql (revision 20934) +++ trunk/ippTools/share/difftool_inputskyfile.sql (revision 20973) @@ -1,12 +1,13 @@ SELECT * FROM - (SELECT + (SELECT + -- warp input diffRun.diff_id, - diffRun.skycell_id, - diffRun.tess_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, 0 as stack_id, warpSkyfile.warp_id, warpSkyfile.uri, warpSkyfile.path_base, - diffInputSkyfile.template, + 0 as template, rawExp.camera FROM diffRun @@ -14,9 +15,47 @@ USING(diff_id) JOIN warpSkyfile - ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + ON diffInputSkyfile.warp1 = warpSkyfile.warp_id AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id AND diffInputSkyfile.tess_id = warpSkyfile.tess_id JOIN warpRun - ON diffInputSkyfile.warp_id = warpRun.warp_id + ON diffInputSkyfile.warp1 = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + WHERE + diffRun.state = 'new' + AND warpRun.state = 'full' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + -- where hook %s + UNION + SELECT + -- warp template + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + 0 as stack_id, + warpSkyfile.warp_id, + warpSkyfile.uri, + warpSkyfile.path_base, + 0 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp2 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON diffInputSkyfile.warp2 = warpRun.warp_id JOIN fakeRun USING(fake_id) @@ -38,12 +77,13 @@ UNION SELECT + -- stack input diffRun.diff_id, - diffRun.skycell_id, - diffRun.tess_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, stackSumSkyfile.stack_id, 0 as warp_id, stackSumSkyfile.uri, stackSumSkyfile.path_base, - diffInputSkyfile.template, + 1 as template, rawExp.camera FROM diffRun @@ -51,25 +91,31 @@ USING(diff_id) JOIN stackSumSkyfile - ON diffInputSkyfile.stack_id = stackSumSkyfile.stack_id - JOIN stackInputSkyfile - ON diffInputSkyfile.stack_id = stackInputSkyfile.stack_id - JOIN warpRun - ON stackInputSkyfile.warp_id = warpRun.warp_id - JOIN fakeRun - USING(fake_id) - JOIN camRun - USING(cam_id) - JOIN chipRun - USING(chip_id) - JOIN chipProcessedImfile - USING(chip_id) + ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id JOIN rawExp - ON chipRun.exp_id = rawExp.exp_id + USING(exp_id) WHERE diffRun.state = 'new' - AND warpRun.state = 'full' - AND fakeRun.state = 'full' - AND camRun.state = 'full' - AND chipRun.state = 'full' + -- where hook %s + UNION + SELECT + -- stack template + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + stackSumSkyfile.stack_id, + 0 as warp_id, + stackSumSkyfile.uri, + stackSumSkyfile.path_base, + 1 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id + JOIN rawExp + USING(exp_id) + WHERE + diffRun.state = 'new' -- where hook %s ) as Foo Index: trunk/ippTools/share/difftool_skyfile.sql =================================================================== --- trunk/ippTools/share/difftool_skyfile.sql (revision 20934) +++ trunk/ippTools/share/difftool_skyfile.sql (revision 20973) @@ -1,15 +1,18 @@ SELECT - diffRun.skycell_id, + diffSkyfile.*, diffRun.tess_id, diffRun.state, - diffSkyfile.*, - (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0, - (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0, - (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1, - (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1 + warp1, + stack1, + warp2, + stack2 +-- (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id +-- AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0, +-- (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id +-- AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0, +-- (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id +-- AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1, +-- (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id +-- AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1 FROM diffRun JOIN diffSkyfile @@ -17,13 +20,6 @@ JOIN diffInputSkyfile ON diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.template = 0 JOIN warpRun - USING(warp_id) -JOIN fakeRun - USING(fake_id) -JOIN camRun - USING(cam_id) -JOIN chipRun - USING(chip_id) + ON warpRun.warp_id = diffInputSkyfile.warp1 JOIN rawExp USING(exp_id) Index: trunk/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- trunk/ippTools/share/difftool_todiffskyfile.sql (revision 20934) +++ trunk/ippTools/share/difftool_todiffskyfile.sql (revision 20973) @@ -6,48 +6,56 @@ 0 as diff_skyfile_id, diffRun.workdir, - diffRun.skycell_id, + diffInputSkyfile.skycell_id, diffRun.tess_id, diffRun.label, diffRun.state FROM diffRun +JOIN diffInputSkyfile USING(diff_id) + -- Get list of templates for each diffRun -JOIN diffInputSkyfile AS diffTemplateSkyfile - ON diffRun.diff_id = diffTemplateSkyfile.diff_id - AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id - AND diffTemplateSkyfile.template = 1 +-- JOIN diffInputSkyfile AS diffTemplateSkyfile +-- ON diffRun.diff_id = diffTemplateSkyfile.diff_id +-- AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id +-- AND diffTemplateSkyfile.template = 1 -- Get list of inputs for each diffRun -JOIN diffInputSkyfile - ON diffRun.diff_id = diffInputSkyfile.diff_id - AND diffRun.skycell_id = diffInputSkyfile.skycell_id - AND diffInputSkyfile.template = 0 +-- JOIN diffInputSkyfile +-- ON diffRun.diff_id = diffInputSkyfile.diff_id +-- AND diffRun.skycell_id = diffInputSkyfile.skycell_id +-- AND diffInputSkyfile.template = 0 + -- Get warp templates LEFT JOIN warpRun AS warpTemplateRun - ON warpTemplateRun.warp_id = diffTemplateSkyfile.warp_id - AND diffTemplateSkyfile.warp_id IS NOT NULL + ON warpTemplateRun.warp_id = diffInputSkyfile.warp2 + AND diffInputSkyfile.warp2 IS NOT NULL LEFT JOIN warpSkyfile AS warpTemplateSkyfile ON warpTemplateSkyfile.warp_id = warpTemplateRun.warp_id - AND warpTemplateSkyfile.skycell_id = diffTemplateSkyfile.skycell_id + AND warpTemplateSkyfile.skycell_id = diffInputSkyfile.skycell_id + -- Get warp inputs LEFT JOIN warpRun - ON warpRun.warp_id = diffInputSkyfile.warp_id - AND diffInputSkyfile.warp_id IS NOT NULL + ON warpRun.warp_id = diffInputSkyfile.warp1 + AND diffInputSkyfile.warp1 IS NOT NULL LEFT JOIN warpSkyfile ON warpSkyfile.warp_id = warpRun.warp_id AND warpSkyfile.skycell_id = diffInputSkyfile.skycell_id + -- Get stack templates LEFT JOIN stackRun AS stackTemplateRun - ON stackTemplateRun.stack_id = diffTemplateSkyfile.stack_id + ON stackTemplateRun.stack_id = diffInputSkyfile.stack2 + AND diffInputSkyfile.stack2 IS NOT NULL LEFT JOIN stackSumSkyfile AS stackTemplateSkyfile ON stackTemplateSkyfile.stack_id = stackTemplateRun.stack_id + -- Get stack inputs LEFT JOIN stackRun - ON stackRun.stack_id = diffInputSkyfile.stack_id - AND diffInputSkyfile.warp_id IS NULL + ON stackRun.stack_id = diffInputSkyfile.stack1 + AND diffInputSkyfile.stack1 IS NOT NULL LEFT JOIN stackSumSkyfile ON stackSumSkyfile.stack_id = stackRun.stack_id - AND diffInputSkyfile.warp_id IS NULL + -- Get what's already been processed LEFT JOIN diffSkyfile ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id WHERE -- Ready to be processed @@ -58,19 +66,19 @@ ) -- Ensure input warps are available - AND (diffInputSkyfile.warp_id IS NULL + AND (diffInputSkyfile.warp1 IS NULL OR (warpRun.state = 'full' AND warpSkyfile.fault = 0 AND warpSkyfile.ignored = 0)) -- Ensure input stacks are available - AND (diffInputSkyfile.stack_id IS NULL + AND (diffInputSkyfile.stack1 IS NULL OR (stackRun.state = 'full' AND stackSumSkyfile.fault = 0)) -- Ensure template warps are available - AND (diffTemplateSkyfile.warp_id IS NULL + AND (diffInputSkyfile.warp2 IS NULL OR (warpTemplateRun.state = 'full' AND warpTemplateSkyfile.fault = 0 AND warpTemplateSkyfile.ignored = 0)) -- Ensure template stacks are available - AND (diffTemplateSkyfile.stack_id IS NULL + AND (diffInputSkyfile.stack2 IS NULL OR (stackTemplateRun.state = 'full' AND stackTemplateSkyfile.fault = 0)) Index: trunk/ippTools/share/magicdstool_completed_runs.sql =================================================================== --- trunk/ippTools/share/magicdstool_completed_runs.sql (revision 20934) +++ trunk/ippTools/share/magicdstool_completed_runs.sql (revision 20973) @@ -65,15 +65,13 @@ JOIN magicRun USING (magic_id) JOIN magicInputSkyfile USING(magic_id) - -- Do we really need to join back to diffInputSkyfile here? - JOIN diffRun USING(diff_id) - JOIN diffSkyfile USING(diff_id) - JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffInputSkyfile.skycell_id + AND magicDSFile.component = diffSkyfile.skycell_id WHERE magicDSRun.state = 'run' AND magicDSRun.stage = 'diff' - AND diffRun.state = 'full' AND diffSkyfile.fault = 0 GROUP BY Index: trunk/ippTools/share/magicdstool_getrunids.sql =================================================================== --- trunk/ippTools/share/magicdstool_getrunids.sql (revision 20934) +++ trunk/ippTools/share/magicdstool_getrunids.sql (revision 20973) @@ -8,9 +8,14 @@ FROM magicRun JOIN magicInputSkyfile USING(magic_id) -JOIN diffInputSkyfile USING(diff_id) -JOIN warpRun USING(warp_id) +JOIN diffSkyfile + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id +JOIN warpRun + ON diffInputSkyfile.warp1 = warp_id JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) -WHERE template = 0 -AND magic_id = %ld +WHERE magic_id = %ld Index: trunk/ippTools/share/magicdstool_getskycells.sql =================================================================== --- trunk/ippTools/share/magicdstool_getskycells.sql (revision 20934) +++ trunk/ippTools/share/magicdstool_getskycells.sql (revision 20973) @@ -1,20 +1,20 @@ SELECT DISTINCT diffSkyfile.diff_id, - diffRun.skycell_id, + diffSkyfile.skycell_id, diffSkyfile.uri, diffSkyfile.path_base FROM magicDSRun JOIN magicRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile USING(diff_id) +JOIN magicInputSkyfile USING(magic_id, diff_id) +JOIN diffSkyfile + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id JOIN diffInputSkyfile - ON diffInputSkyfile.diff_id = diffRun.diff_id - AND diffInputSkyfile.skycell_id = diffRun.skycell_id + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id -- Want input warps only - AND diffInputSkyfile.warp_id IS NOT NULL - AND diffInputSkyfile.template = 0 + AND diffInputSkyfile.warp1 IS NOT NULL JOIN warpSkyCellMap - ON warpSkyCellMap.warp_id = diffInputSkyfile.warp_id + ON warpSkyCellMap.warp_id = diffInputSkyfile.warp1 AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id JOIN warpSkyfile @@ -23,5 +23,4 @@ AND warpSkyfile.ignored = 0 WHERE - diffRun.state = 'full' - AND diffSkyfile.fault = 0 + diffSkyfile.fault = 0 AND magic_ds_id = %lld Index: trunk/ippTools/share/magicdstool_todestreak.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak.sql (revision 20934) +++ trunk/ippTools/share/magicdstool_todestreak.sql (revision 20973) @@ -106,6 +106,6 @@ magicMask.uri as streaks_uri, stage, - diff_id as stage_id, - diffInputSkyfile.skycell_id as component, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, diffSkyfile.uri, diffSkyfile.path_base, @@ -120,14 +120,13 @@ JOIN magicDSRun USING(magic_id) JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile USING(diff_id) -JOIN diffInputSkyfile USING(diff_id) +JOIN diffSkyfile + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffInputSkyfile.skycell_id + AND magicDSFile.component = diffSkyfile.skycell_id WHERE magicDSRun.state = 'run' AND magicDSRun.stage = 'diff' - AND diffRun.state = 'full' AND diffSkyfile.fault = 0 AND magicDSFile.component IS NULL Index: trunk/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 20934) +++ trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 20973) @@ -5,5 +5,6 @@ diff_id, skycell_id -FROM magicBestDiffs +FROM diffSkyfile WHERE - exp_id = @EXP_ID@ -- Update this with the appropriate exp_id + diff_id = @DIFF_ID@ -- Update this with the appropriate diff_id + AND fault = 0 Index: trunk/ippTools/share/magictool_definebyquery_select.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_select.sql (revision 20934) +++ trunk/ippTools/share/magictool_definebyquery_select.sql (revision 20973) @@ -1,38 +1,9 @@ -- Get a list of exposures on which magic may be performed -SELECT DISTINCT +SELECT exp_id, - filter, - num_todo, - num_done, - MAX(magic_id) -FROM ( - -- Number of skycells as a function of exposure - SELECT - exp_id, - filter, - COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo - FROM rawExp - JOIN chipRun USING(exp_id) - JOIN camRun USING(chip_id) - JOIN fakeRun USING(cam_id) - JOIN warpRun USING(fake_id) - JOIN warpSkyCellMap USING(warp_id) - JOIN warpSkyfile USING(warp_id, skycell_id) - JOIN diffInputSkyfile USING(warp_id,skycell_id) - JOIN diffRun USING(diff_id) - WHERE - warpSkyfile.ignored = 0 - -- magicSkycellNums WHERE hook %s - GROUP BY - exp_id - ) AS magicSkycellNums -JOIN ( - -- Number of completed diffs for an exposure - SELECT - exp_id, - COUNT(diff_id) AS num_done - FROM magicBestDiffs - GROUP BY - exp_id - ) AS magicDiffNums USING(exp_id) + MAX(diffRun.diff_id) AS diff_id +FROM diffRun +JOIN rawExp USING(exp_id) LEFT JOIN magicRun USING(exp_id) +-- WHERE hook %s +GROUP BY exp_id Index: trunk/ippTools/share/magictool_definebyquery_temp_insert.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 20934) +++ trunk/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 20973) @@ -14,7 +14,6 @@ JOIN warpSkyfile USING(warp_id, skycell_id) JOIN diffInputSkyfile - ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + ON diffInputSkyfile.warp1 = warpSkyfile.warp_id AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id - AND diffInputSkyfile.template = 0 -- selecting inputs only JOIN diffRun USING(diff_id) JOIN diffSkyfile USING(diff_id) Index: trunk/ippTools/share/magictool_inputs.sql =================================================================== --- trunk/ippTools/share/magictool_inputs.sql (revision 20934) +++ trunk/ippTools/share/magictool_inputs.sql (revision 20973) @@ -10,9 +10,10 @@ diffSkyfile.path_base, diffSkyfile.fault -FROM magicInputSkyfile +FROM magicRun +JOIN magicInputSkyfile +USING(magic_id) JOIN diffSkyfile - USING(diff_id) -JOIN magicRun - USING(magic_id) + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id UNION -- Merged skycells @@ -21,5 +22,5 @@ magicRun.state, magicTree.node, - 0, + 0, -- no diff_id magicNodeResult.uri, NULL, -- magicNodeResult doesn't have a path_base Index: trunk/ippTools/share/magictool_inputskyfile.sql =================================================================== --- trunk/ippTools/share/magictool_inputskyfile.sql (revision 20934) +++ trunk/ippTools/share/magictool_inputskyfile.sql (revision 20973) @@ -2,9 +2,10 @@ magicInputSkyfile.*, diffSkyfile.uri -FROM magicInputSkyfile -JOIN magicRun +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) JOIN diffSkyfile - USING(diff_id) + ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id WHERE magicRun.state = 'run' Index: trunk/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 20934) +++ trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 20973) @@ -11,5 +11,7 @@ JOIN magicRun USING(magic_id) JOIN magicInputSkyfile USING(magic_id, node) -JOIN diffSkyfile USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id JOIN rawExp USING(exp_id) LEFT JOIN magicNodeResult Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 20934) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 20973) @@ -238,7 +238,10 @@ fault SMALLINT NOT NULL, epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + raw_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), KEY(fault), + KEY(raw_image_id), UNIQUE KEY(exp_id, tmp_class_id), FOREIGN KEY (exp_id, tmp_class_id) @@ -330,7 +333,10 @@ path_base VARCHAR(255), fault SMALLINT NOT NULL, + chip_image_id BIGINT AUTO_INCREMENT, + magicked BIGINT, PRIMARY KEY(chip_id, exp_id, class_id), KEY(data_state), KEY(fault), + KEY(chip_image_id), FOREIGN KEY (chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id), @@ -815,5 +821,4 @@ end_stage VARCHAR(64), registered DATETIME, - magiced TINYINT, PRIMARY KEY(warp_id), KEY(warp_id), @@ -824,5 +829,4 @@ KEY(label), KEY(end_stage), - KEY(magiced), INDEX(warp_id, fake_id), FOREIGN KEY (fake_id) @@ -861,7 +865,10 @@ ignored TINYINT, fault SMALLINT, + warp_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), KEY(ignored), KEY(fault), + KEY(warp_image_id), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) @@ -941,34 +948,38 @@ dvodb VARCHAR(255), registered DATETIME, - skycell_id VARCHAR(64), tess_id VARCHAR(64), + exp_id BIGINT, PRIMARY KEY(diff_id), KEY(diff_id), KEY(state), - KEY(skycell_id), - KEY(tess_id) + KEY(tess_id), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffInputSkyfile ( diff_id BIGINT, - template TINYINT, - stack_id BIGINT, - warp_id BIGINT, skycell_id VARCHAR(64), + warp1 BIGINT, + stack1 BIGINT, + warp2 BIGINT, + stack2 BIGINT, tess_id VARCHAR(64), - kind VARCHAR(64), - PRIMARY KEY(diff_id, template), - KEY(stack_id), - KEY(warp_id), + PRIMARY KEY(diff_id, skycell_id), + KEY(warp1), + KEY(warp2), + KEY(stack1), + KEY(stack2), KEY(skycell_id), KEY(tess_id), - KEY(kind), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), - FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), - FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id) + FOREIGN KEY (warp1, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id), + FOREIGN KEY (warp2, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id), + FOREIGN KEY (stack1) REFERENCES stackSumSkyfile(stack_id), + FOREIGN KEY (stack2) REFERENCES stackSumSkyfile(stack_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffSkyfile ( diff_id BIGINT, + skycell_id VARCHAR(64), uri VARCHAR(255), path_base VARCHAR(255), @@ -993,7 +1004,10 @@ good_frac FLOAT, fault SMALLINT, - PRIMARY KEY(diff_id), + diff_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, + PRIMARY KEY(diff_id, skycell_id), KEY(good_frac), KEY(fault), + KEY(diff_image_id), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1002,4 +1016,5 @@ magic_id BIGINT AUTO_INCREMENT, exp_id BIGINT, + diff_id BIGINT, state VARCHAR(64), workdir VARCHAR(255), @@ -1015,5 +1030,6 @@ KEY(label), KEY(fault), - FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1022,7 +1038,7 @@ diff_id BIGINT, node VARCHAR(64), - PRIMARY KEY(magic_id, diff_id), + PRIMARY KEY(magic_id, diff_id, node), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), - FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: trunk/ippTools/share/warptool_tooverlap.sql =================================================================== --- trunk/ippTools/share/warptool_tooverlap.sql (revision 20934) +++ trunk/ippTools/share/warptool_tooverlap.sql (revision 20973) @@ -7,6 +7,5 @@ rawExp.camera, exp_id, - exp_tag, - warpRun.magiced + exp_tag FROM warpRun JOIN fakeRun Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 20934) +++ trunk/ippTools/src/chiptool.c (revision 20973) @@ -484,5 +484,7 @@ n_cr, path_base, - code + code, + 0, // chip_image_id + 0 // magic_ds_id )) { // rollback Index: trunk/ippTools/src/difftool.c =================================================================== --- trunk/ippTools/src/difftool.c (revision 20934) +++ trunk/ippTools/src/difftool.c (revision 20973) @@ -47,4 +47,5 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state); +static bool diffRunComplete(pxConfig *config); # define MODECASE(caseName, func) \ @@ -108,8 +109,8 @@ // required options PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); - PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); // default @@ -120,11 +121,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - skycell_id, - tess_id + tess_id, + exp_id ); if (!run) { @@ -178,26 +179,32 @@ // optional - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false); - - // defaults to false - PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); - - // must provide either stack_id or warp_id but not BOTH - if (!(stack_id || warp_id)) { - psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); - return false; - } - if (stack_id && warp_id) { - psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); - return false; - } - - // if a warp_id was provided we need to lookup the skycell_id and tess_id - // from the warpRun - psString skycell_id = NULL; + PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false); + PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false); + PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false); + PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + // must provide either stack1 or warp1 but not BOTH + if (!(stack1 || warp1)) { + psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified"); + return false; + } + if (stack1 && warp1) { + psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified"); + return false; + } + // must provide either stack2 or warp2 but not BOTH + if (!(stack2 || warp2)) { + psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified"); + return false; + } + if (stack2 && warp2) { + psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified"); + return false; + } + + // if a warp1 was provided we need to lookup the and tess_id from the diffRun psString tess_id = NULL; - if (warp_id) { + if (warp1) { if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -217,5 +224,4 @@ diffRunRow *run = diffRunObjectFromMetadata(output->data[0]); - skycell_id = run->skycell_id; tess_id = run->tess_id; } @@ -228,10 +234,10 @@ if (!diffInputSkyfileInsert(config->dbh, diff_id, - template, - stack_id ? stack_id : PS_MAX_S64, // defined or NULL - warp_id ? warp_id : PS_MAX_S64, // defined or NULL skycell_id, - tess_id, - kind + warp1 ? warp1 : PS_MAX_S64, // defined or NULL + stack1 ? stack1 : PS_MAX_S64, // defined or NULL + warp2 ? warp2 : PS_MAX_S64, // defined or NULL + stack2 ? stack2 : PS_MAX_S64, // defined or NULL + tess_id )) { if (!psDBRollback(config->dbh)) { @@ -298,7 +304,6 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - PXOPT_COPY_STR(config->args, where, "-kind", "kind", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -349,5 +354,5 @@ } - if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) { + if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(whereClause); @@ -471,4 +476,5 @@ PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); @@ -503,4 +509,5 @@ if (!diffSkyfileInsert(config->dbh, diff_id, + skycell_id, uri, path_base, @@ -524,5 +531,7 @@ hostname, good_frac, - code + code, + 0, // diff_image_id + 0 // magic_ds_id )) { if (!psDBRollback(config->dbh)) { @@ -533,5 +542,5 @@ } - if (!setdiffRunState(config, diff_id, "full")) { + if (!diffRunComplete(config)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -546,4 +555,5 @@ return false; } + return true; @@ -762,4 +772,5 @@ psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none + psS64 exp_id, // exposure id for input_warp_id (if defined) pxConfig *config // Configuration ) @@ -786,11 +797,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - skycell_id, - tess_id + tess_id, + exp_id ); @@ -817,27 +828,10 @@ if (!diffInputSkyfileInsert(config->dbh, run->diff_id, - true, + skycell_id, + input_warp_id, + input_stack_id, + template_warp_id, template_stack_id, - template_warp_id, - skycell_id, - tess_id, - NULL // kind - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // Input - if (!diffInputSkyfileInsert(config->dbh, - run->diff_id, - false, - input_stack_id, - input_warp_id, - skycell_id, - tess_id, - NULL // kind + tess_id )) { if (!psDBRollback(config->dbh)) { @@ -887,4 +881,5 @@ PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -906,4 +901,9 @@ psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No input has been defined (-input_stack_id or -input_warp_id)"); + return false; + } + if (input_warp_id && !exp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "-exp_id is required with -input_warp_id."); return false; } @@ -916,4 +916,5 @@ template_warp_id ? template_warp_id : PS_MAX_S64, template_stack_id ? template_stack_id : PS_MAX_S64, + exp_id, config)) { psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); @@ -937,7 +938,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *expWhere = psMetadataAlloc(); psMetadata *warpWhere = psMetadataAlloc(); psMetadata *stackWhere = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "=="); @@ -945,5 +948,4 @@ PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-kind", "warpsToDiff.kind", "=="); PXOPT_COPY_F32(config->args, warpWhere, "-good_frac", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); @@ -955,7 +957,9 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // find all things to queue - psString query = pxDataGet("difftool_definebyquery.sql"); + psString query = pxDataGet("difftool_definebyquery_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -963,15 +967,16 @@ } - if (newTemplates) { - // Warps that haven't been diffed OR warps that can take a newer template in the diff - psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); - } else { - // Only warps that haven't been diffed - psStringAppend(&query, " WHERE diff_id IS NULL"); - } - psString warpQuery = NULL; psString stackQuery = NULL; - + psString expQuery = NULL; + + if (psListLength(expWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL); + psStringAppend(&expQuery, "\n AND %s", whereClause); + psFree(whereClause); + } else { + expQuery = psStringCopy("\n"); + } + psFree(expWhere); if (psListLength(warpWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); @@ -979,11 +984,24 @@ psFree(whereClause); } else { - warpQuery = psStringCopy(""); + warpQuery = psStringCopy("\n"); } psFree(warpWhere); + + if (!available) { + // diff what's available, even if warp run has some faults and is incomplete + psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); + } + + // don't queue for exposures that have already been diff'd unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\nAND diff_id IS NULL"); + } else { + diffQuery = psStringCopy("\n"); + } if (psListLength(stackWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL); - psStringAppend(&stackQuery, "\n AND %s", whereClause); + psStringAppend(&stackQuery, "\nAND %s", whereClause); psFree(whereClause); } else { @@ -992,7 +1010,8 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, stackQuery); - - if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) { + psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + + + if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1021,43 +1040,156 @@ } + // create temporary table + query = pxDataGet("difftool_definebyquery_temp_create.sql"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + query = NULL; + + psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql"); + psArray *list = psArrayAllocEmpty(16); // List of runs, to print long numGood = 0; // Number of good rows added + psS64 last_exp_id = 0; for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; // Output row from query - - // Selected parameters bool mdok; // Status of MD lookup - const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); - if (!mdok || !skycell_id) { - psWarning("skycell_id not found --- ignoring row %ld", i); + + // Take the first warp for each exposure. + // The list is sorted by exposure id and warp_id and the warps are in descending + // order. + psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + return false; + } + if (exp_id == last_exp_id) { continue; } - const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); - if (!mdok || !tess_id) { - psWarning("tess_id not found --- ignoring row %ld", i); - continue; - } - psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); - if (!mdok) { - psWarning("stack_id not found --- ignoring row %ld", i); - continue; + last_exp_id = exp_id; + + // clear temporary table + if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; } psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); if (!mdok) { - psWarning("warp_id not found --- ignoring row %ld", i); + psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psString filter = psMetadataLookupStr(&mdok, row, "filter"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "filter not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, filter, warpQuery, stackQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 num = psDBAffectedRows(config->dbh); + + if (num == 0) { + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); continue; } - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, - PS_MAX_S64, PS_MAX_S64, stack_id, config)) { - psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, - warp_id, stack_id); - psErrorClear(); + if (!available && (num != skycell_count)) { + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" + PRId64 " need %" PRId64, num, skycell_count, warp_id); continue; } + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label, + reduction, + NULL, // dvodb + registered, + tess_id, + exp_id + ); + + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + run->diff_id = psDBLastInsertID(config->dbh); + + psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + psFree(query); + query = NULL; + psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + psFree(query); + query = NULL; + + if (!setdiffRunState(config, run->diff_id, "new")) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + + psArrayAdd(list, list->n, run); numGood++; } psFree(output); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); if (!diffRunPrintObjects(stdout, list, !simple)) { @@ -1274,2 +1406,47 @@ } +static bool diffRunComplete(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + + // set diffRun.state to 'stop' + if (!setdiffRunState(config, diff_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + return false; + } + } + + return true; +} + Index: trunk/ippTools/src/difftoolConfig.c =================================================================== --- trunk/ippTools/src/difftoolConfig.c (revision 20934) +++ trunk/ippTools/src/difftoolConfig.c (revision 20973) @@ -48,5 +48,4 @@ psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); @@ -65,5 +64,4 @@ psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID", 0); psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID", 0); - psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "define kind", NULL); psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template", 0, "this sky cell file is the subtrahend", false); @@ -74,5 +72,4 @@ psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); - psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template", 0, "find only subtrahend", false); psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-input", 0, "find only minuend", false); @@ -90,4 +87,5 @@ psMetadata *adddiffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define warp ID (required)", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell of file (required)", 0); psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); @@ -147,6 +145,6 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); @@ -155,5 +153,4 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); @@ -161,4 +158,6 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: trunk/ippTools/src/magicdstool.c =================================================================== --- trunk/ippTools/src/magicdstool.c (revision 20934) +++ trunk/ippTools/src/magicdstool.c (revision 20973) @@ -469,5 +469,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "todestreak", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); Index: trunk/ippTools/src/magictool.c =================================================================== --- trunk/ippTools/src/magictool.c (revision 20934) +++ trunk/ippTools/src/magictool.c (revision 20973) @@ -128,48 +128,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // Create temporary table of the best diffs - { - psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - } - - // Insert list of best diffs into temporary table - { - psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - psString whereClause = NULL; // WHERE conditions - if (psListLength(where->list)) { - whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereClause); - psFree(query); - return false; - } - psFree(whereClause); - psFree(query); - } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); + // Get list of exposures ready to magic @@ -181,49 +141,41 @@ } - psString magicSkyCellNumsWhere = NULL; // WHERE conditions for magicSkyCellNums - { - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - if (psListLength(where->list)) { - magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&magicSkyCellNumsWhere, "\n AND "); - } - psFree(where); - } - - // "available" means only concern ourselves with exposures that have all diffs completed, unless we're - // told to only take what's available. - // "new" means we want a new run even if there's already a magic run defined + // "available" means queue magic run even though the diffRun has skycells that did not complete + // "rerun" means we want a new run even if there's already a magic run defined for the exposure PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); - PXOPT_LOOKUP_BOOL(new, config->args, "-new", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); psString queryWhere = NULL; // WHERE conditions for entire query - if (available) { - psStringAppend(&queryWhere, " WHERE num_done = num_todo"); - } - if (new) { + if (!available) { + psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'"); + } else { + // what if no skycells for the diff run completed? + } + if (!rerun) { const char *newWhere = " magic_id IS NULL"; // String to add if (queryWhere) { psStringAppend(&queryWhere, " AND %s", newWhere); } else { - psStringAppend(&queryWhere, " WHERE %s", newWhere); + psStringAppend(&queryWhere, "\nWHERE %s", newWhere); } } - if (queryWhere) { - psStringAppend(&query, " %s", queryWhere); + // now add the user specified qualifiers + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause); + psFree(whereClause); + } + psFree(where); + if (!queryWhere) { + psStringAppend(&queryWhere, " "); + } + + if (!p_psDBRunQuery(config->dbh, query, queryWhere)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(queryWhere); - } - - - if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(magicSkyCellNumsWhere); psFree(query); return false; } - psFree(magicSkyCellNumsWhere); + psFree(queryWhere); psFree(query); } @@ -262,7 +214,8 @@ psMetadata *row = output->data[i]; // Row of interest psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "run", workdir, "dirty", label, dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); @@ -297,6 +250,6 @@ { psString idString = NULL; - psStringAppend(&idString, "%" PRId64, exp_id); - psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); + psStringAppend(&idString, "%" PRId64, diff_id); + psStringSubstitute(&thisInsert, idString, "@DIFF_ID@"); psFree(idString); } @@ -340,4 +293,5 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // optional @@ -350,4 +304,5 @@ 0, // ID exp_id, + diff_id, "reg", // state workdir, Index: trunk/ippTools/src/magictoolConfig.c =================================================================== --- trunk/ippTools/src/magictoolConfig.c (revision 20934) +++ trunk/ippTools/src/magictoolConfig.c (revision 20973) @@ -55,7 +55,6 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: trunk/ippTools/src/pxwarp.c =================================================================== --- trunk/ippTools/src/pxwarp.c (revision 20934) +++ trunk/ippTools/src/pxwarp.c (revision 20973) @@ -147,6 +147,5 @@ tess_id, end_stage, - NULL, // registered - false // magiced + NULL // registered )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 20934) +++ trunk/ippTools/src/regtool.c (revision 20973) @@ -299,5 +299,6 @@ hostname, code, - NULL + NULL, + 0 )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 20934) +++ trunk/ippTools/src/warptool.c (revision 20973) @@ -134,5 +134,4 @@ PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -154,6 +153,5 @@ tess_id, end_stage, - registered, - magiced + registered ); if (!warpRun) { @@ -246,5 +244,4 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -942,5 +939,7 @@ ymax, !accept, - code + code, + 0, // warp_image_id + 0 // magic_ds_id )) { if (!psDBRollback(config->dbh)) { Index: trunk/ippTools/src/warptoolConfig.c =================================================================== --- trunk/ippTools/src/warptoolConfig.c (revision 20934) +++ trunk/ippTools/src/warptoolConfig.c (revision 20973) @@ -103,5 +103,4 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -116,5 +115,4 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);