Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 25428) +++ trunk/ippTools/share/Makefile.am (revision 25429) @@ -110,5 +110,11 @@ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ - disttool_pendingcomponent.sql \ + disttool_pending_camera.sql \ + disttool_pending_chip.sql \ + disttool_pending_diff.sql \ + disttool_pending_fake.sql \ + disttool_pending_raw.sql \ + disttool_pending_stack.sql \ + disttool_pending_warp.sql \ disttool_pendingfileset.sql \ disttool_pendingdest.sql \ Index: trunk/ippTools/share/disttool_pending_camera.sql =================================================================== --- trunk/ippTools/share/disttool_pending_camera.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_camera.sql (revision 25429) @@ -0,0 +1,31 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + camProcessedExp.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + camRun.state, + camRun.state AS data_state, + camProcessedExp.quality, + distRun.no_magic, + chipRun.magicked +FROM distRun +JOIN camRun ON camRun.cam_id = distRun.stage_id +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +-- JOIN chipProcessedImfile USING(exp_id, chip_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +-- AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'camera' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) + AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_pending_chip.sql =================================================================== --- trunk/ippTools/share/disttool_pending_chip.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_chip.sql (revision 25429) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + chipProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + chipProcessedImfile.path_base, + chipProcessedImfile.path_base as chip_path_base, + chipRun.state, + chipProcessedImfile.data_state, + chipProcessedImfile.quality, + distRun.no_magic, + chipProcessedImfile.magicked +FROM distRun +JOIN chipRun ON chipRun.chip_id = distRun.stage_id +JOIN rawExp using(exp_id) +JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'chip' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) + AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_pending_diff.sql =================================================================== --- trunk/ippTools/share/disttool_pending_diff.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_diff.sql (revision 25429) @@ -0,0 +1,37 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + diffSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + diffSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + diffRun.state, + -- data_state doesn't exist yet + -- diffSkyfile.data_state, + 'full' AS data_state, + diffSkyfile.quality, + distRun.no_magic, + diffSkyfile.magicked +FROM distRun +JOIN diffRun ON stage_id = diff_id +JOIN diffSkyfile using(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND diffSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'diff' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) + AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_pending_fake.sql =================================================================== --- trunk/ippTools/share/disttool_pending_fake.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_fake.sql (revision 25429) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + fakeProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fakeProcessedImfile.path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + fakeRun.state, + fakeRun.state AS data_state, + 0 as quality, + distRun.no_magic, + 0 +FROM distRun +JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id +JOIN fakeProcessedImfile USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND fakeProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'fake' + AND distComponent.dist_id IS NULL Index: trunk/ippTools/share/disttool_pending_raw.sql =================================================================== --- trunk/ippTools/share/disttool_pending_raw.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_raw.sql (revision 25429) @@ -0,0 +1,58 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + rawImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + rawImfile.uri as path_base, -- change this once rawImfile has path_base + chipProcessedImfile.path_base as chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked +FROM distRun +JOIN rawExp ON rawExp.exp_id = stage_id +JOIN rawImfile USING(exp_id) +JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked +JOIN chipProcessedImfile + USING(chip_id, class_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.stage = 'raw' + AND distComponent.dist_id IS NULL + AND (rawExp.magicked OR distRun.no_magic) +UNION + -- raw stage clean (dbinfo only) +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + CAST(NULL AS CHAR(255)) AS path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawExp.magicked +FROM distRun +JOIN rawExp ON exp_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +WHERE + distRun.state = 'new' + AND distRun.stage = 'raw' + AND distRun.clean + AND distComponent.dist_id IS NULL Index: trunk/ippTools/share/disttool_pending_stack.sql =================================================================== --- trunk/ippTools/share/disttool_pending_stack.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_stack.sql (revision 25429) @@ -0,0 +1,48 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + stage, + stage_id, + stackRun.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + stackSumSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + stackRun.state, + -- data_state doesn't exist yet + -- stackSumSkyfile.data_state, + 'full' AS data_state, + stackSumSkyfile.quality, + 1 AS no_magic, + 0 AS magicked +FROM distRun +JOIN stackRun + ON stage_id = stack_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'stack' + AND distComponent.dist_id IS NULL + AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: trunk/ippTools/share/disttool_pending_warp.sql =================================================================== --- trunk/ippTools/share/disttool_pending_warp.sql (revision 25429) +++ trunk/ippTools/share/disttool_pending_warp.sql (revision 25429) @@ -0,0 +1,32 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + warpSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + warpSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + warpRun.state, + warpSkyfile.data_state, + warpSkyfile.quality, + distRun.no_magic, + warpSkyfile.magicked +FROM distRun +JOIN warpRun ON stage_id = warp_id +JOIN warpSkyfile using(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND warpSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'warp' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) + AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) Index: trunk/ippTools/share/disttool_pendingcomponent.sql =================================================================== --- trunk/ippTools/share/disttool_pendingcomponent.sql (revision 25428) +++ (revision ) @@ -1,283 +1,0 @@ -SELECT * FROM ( - -- raw stage -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - rawImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - rawImfile.uri as path_base, -- change this once rawImfile has path_base - chipProcessedImfile.path_base as chip_path_base, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawImfile.magicked -FROM distRun -JOIN rawExp ON rawExp.exp_id = stage_id -JOIN rawImfile USING(exp_id) -JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked -JOIN chipProcessedImfile - USING(chip_id, class_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND rawImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.clean = 0 - AND distRun.stage = 'raw' - AND distComponent.dist_id IS NULL - AND (rawExp.magicked OR distRun.no_magic) - -- where hook 1 %s -UNION - -- raw stage clean (dbinfo only) -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - NULL, - NULL, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawExp.magicked -FROM distRun -JOIN rawExp ON exp_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id -WHERE - distRun.state = 'new' - AND distRun.stage = 'raw' - AND distRun.clean - AND distComponent.dist_id IS NULL - -- where hook 2 %s - --- chip stage -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - chipProcessedImfile.path_base, - chipProcessedImfile.path_base as chip_path_base, - chipRun.state, - chipProcessedImfile.data_state, - chipProcessedImfile.quality, - distRun.no_magic, - chipProcessedImfile.magicked -FROM distRun -JOIN chipRun ON chipRun.chip_id = distRun.stage_id -JOIN rawExp using(exp_id) -JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'chip' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) - AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) - -- where hook 3 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, --- chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - camProcessedExp.path_base, - NULL as chip_path_base, - camRun.state, - NULL, - camProcessedExp.quality, - distRun.no_magic, - chipRun.magicked -FROM distRun -JOIN camRun ON camRun.cam_id = distRun.stage_id -JOIN camProcessedExp USING(cam_id) -JOIN chipRun USING(chip_id) --- JOIN chipProcessedImfile USING(exp_id, chip_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id --- AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'camera' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) - AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) - -- where hook 4 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - fakeProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - fakeProcessedImfile.path_base, - NULL, - fakeRun.state, - NULL, - 0 as quality, - distRun.no_magic, - 0 -FROM distRun -JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id -JOIN fakeProcessedImfile USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id, exp_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND fakeProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'fake' - AND distComponent.dist_id IS NULL - -- where hook 5 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - warpSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - warpSkyfile.path_base, - NULL as chip_path_base, - warpRun.state, - warpSkyfile.data_state, - warpSkyfile.quality, - distRun.no_magic, - warpSkyfile.magicked -FROM distRun -JOIN warpRun ON stage_id = warp_id -JOIN warpSkyfile using(warp_id) -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun ON camRun.chip_id = chipRun.chip_id -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND warpSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'warp' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) - AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) - -- where hook 6 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - diffSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - diffSkyfile.path_base, - NULL as chip_path_base, - diffRun.state, - -- data_state doesn't exist yet - -- diffSkyfile.data_state, - 'full' AS data_state, - diffSkyfile.quality, - distRun.no_magic, - diffSkyfile.magicked -FROM distRun -JOIN diffRun ON stage_id = diff_id -JOIN diffSkyfile using(diff_id) -JOIN diffInputSkyfile USING(diff_id, skycell_id) -JOIN warpRun - ON warpRun.warp_id = diffInputSkyfile.warp1 -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND diffSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'diff' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) - AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) - -- where hook 7 %s -UNION -SELECT DISTINCT - distRun.dist_id, - distRun.label, - stage, - stage_id, - stackRun.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - stackSumSkyfile.path_base, - NULL as chip_path_base, - stackRun.state, - -- data_state doesn't exist yet - -- stackSumSkyfile.data_state, - 'full' AS data_state, - stackSumSkyfile.quality, - 1 AS no_magic, - 0 AS magicked -FROM distRun -JOIN stackRun - ON stage_id = stack_id -JOIN stackSumSkyfile - USING(stack_id) -JOIN stackInputSkyfile - USING(stack_id) -JOIN warpSkyfile - ON stackInputSkyfile.warp_id = warpSkyfile.warp_id - AND stackRun.skycell_id = warpSkyfile.skycell_id - AND stackRun.tess_id = warpSkyfile.tess_id -JOIN warpRun - ON warpRun.warp_id = warpSkyfile.warp_id -JOIN fakeRun - USING(fake_id) -JOIN camRun - USING(cam_id) -JOIN chipRun - ON camRun.chip_id = chipRun.chip_id -JOIN rawExp - USING (exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND stackRun.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'stack' - AND distComponent.dist_id IS NULL - AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) - -- where hook 8 %s -) as Foo