SELECT * FROM (
    -- raw stage
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    rawImfile.class_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    rawImfile.uri as path_base,         -- change this once rawImfile has path_base
    chipProcessedImfile.path_base as chip_path_base,
    NULL as state,
    NULL as data_state,
    0 as quality,
    distRun.no_magic,
    rawImfile.magicked
FROM distRun
JOIN rawExp ON exp_id = stage_id
JOIN (                      -- find the last satisfactory chip run 
                            -- note this requires that a chip run have been completed even when 
                            -- magic is not required for the run
    SELECT
        exp_id,
        MAX(chip_id) AS chip_id
    FROM chipRun
    JOIN distRun ON stage_id = exp_id AND stage = 'raw'
    WHERE 
        chipRun.state = 'full'
        AND chipRun.exp_id = exp_id
        AND (chipRun.magicked OR distRun.no_magic)
        GROUP BY exp_id
    ) AS bestChipRun 
    USING(exp_id)
JOIN rawImfile USING(exp_id)
JOIN chipProcessedImfile
    USING(exp_id, chip_id, class_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND rawImfile.class_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.clean = 0
    AND distRun.stage = 'raw'
    AND distComponent.dist_id IS NULL
    AND (rawExp.magicked OR distRun.no_magic)
    -- where hook 1 %s
UNION
    -- raw stage clean (dbinfo only)
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    'exposure' AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    NULL,
    NULL,
    NULL as state,
    NULL as data_state,
    0 as quality,
    distRun.no_magic,
    rawExp.magicked
FROM distRun
JOIN rawExp ON exp_id = stage_id
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'raw'
    AND distRun.clean
    AND distComponent.dist_id IS NULL
    -- where hook 2 %s

-- chip stage
UNION
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    chipProcessedImfile.class_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    chipProcessedImfile.path_base,
    chipProcessedImfile.path_base as chip_path_base,
    chipRun.state,
    chipProcessedImfile.data_state,
    chipProcessedImfile.quality,
    distRun.no_magic,
    chipProcessedImfile.magicked
FROM distRun
JOIN chipRun ON chipRun.chip_id = distRun.stage_id
JOIN rawExp using(exp_id)
JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND chipProcessedImfile.class_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'chip'
    AND distComponent.dist_id IS NULL
    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))
    -- where hook 3 %s
UNION
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    'exposure' AS component,
--    chipProcessedImfile.class_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    camProcessedExp.path_base,
    NULL as chip_path_base,
    camRun.state,
    NULL,
    camProcessedExp.quality,
    distRun.no_magic,
    chipRun.magicked
FROM distRun
JOIN camRun ON camRun.cam_id = distRun.stage_id
JOIN camProcessedExp USING(cam_id)
JOIN chipRun USING(chip_id)
-- JOIN chipProcessedImfile USING(exp_id, chip_id)
JOIN rawExp using(exp_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
--    AND chipProcessedImfile.class_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'camera'
    AND distComponent.dist_id IS NULL
    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
    -- where hook 4 %s
UNION
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    fakeProcessedImfile.class_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    fakeProcessedImfile.path_base,
    NULL,
    fakeRun.state,
    NULL,
    0 as quality,
    distRun.no_magic,
    0
FROM distRun
JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id
JOIN fakeProcessedImfile USING(fake_id)
JOIN camRun USING(cam_id)
JOIN chipRun USING(chip_id, exp_id)
JOIN rawExp using(exp_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND fakeProcessedImfile.class_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'fake'
    AND distComponent.dist_id IS NULL
    -- where hook 5 %s
UNION
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    warpSkyfile.skycell_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    warpSkyfile.path_base,
    NULL as chip_path_base,
    warpRun.state,
    warpSkyfile.data_state,
    warpSkyfile.quality,
    distRun.no_magic,
    warpSkyfile.magicked
FROM distRun
JOIN warpRun ON stage_id = warp_id
JOIN warpSkyfile using(warp_id)
JOIN fakeRun USING(fake_id)
JOIN camRun USING(cam_id)
JOIN chipRun ON camRun.chip_id = chipRun.chip_id
JOIN rawExp using(exp_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND warpSkyfile.skycell_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'warp'
    AND distComponent.dist_id IS NULL
    AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))
    -- where hook 6 %s
UNION
SELECT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    diffSkyfile.skycell_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    diffSkyfile.path_base,
    NULL as chip_path_base,
    diffRun.state,
    -- data_state doesn't exist yet
    -- diffSkyfile.data_state,
    'full' AS data_state,
    diffSkyfile.quality,
    distRun.no_magic,
    diffSkyfile.magicked
FROM distRun
JOIN diffRun ON stage_id = diff_id
JOIN diffSkyfile using(diff_id)
JOIN diffInputSkyfile USING(diff_id, skycell_id)
JOIN warpRun
    ON warpRun.warp_id = diffInputSkyfile.warp1
JOIN fakeRun USING(fake_id)
JOIN camRun USING(cam_id)
JOIN chipRun USING(chip_id)
JOIN rawExp USING(exp_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND diffSkyfile.skycell_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'diff'
    AND distComponent.dist_id IS NULL
    AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
    -- where hook 7 %s
UNION
SELECT DISTINCT
    distRun.dist_id,
    distRun.label,
    stage,
    stage_id,
    stackRun.skycell_id AS component,
    clean,
    rawExp.camera,
    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    stackSumSkyfile.path_base,
    NULL as chip_path_base,
    stackRun.state,
    -- data_state doesn't exist yet
    -- stackSumSkyfile.data_state,
    'full' AS data_state,
    stackSumSkyfile.quality,
    1 AS no_magic,
    0 AS magicked
FROM distRun
JOIN stackRun
    ON stage_id = stack_id
JOIN stackSumSkyfile
    USING(stack_id)
JOIN stackInputSkyfile
    USING(stack_id)
JOIN warpSkyfile
    ON  stackInputSkyfile.warp_id = warpSkyfile.warp_id
    AND stackRun.skycell_id       = warpSkyfile.skycell_id
    AND stackRun.tess_id          = warpSkyfile.tess_id
JOIN warpRun
    ON warpRun.warp_id = warpSkyfile.warp_id
JOIN fakeRun
    USING(fake_id)
JOIN camRun
    USING(cam_id)
JOIN chipRun
    ON camRun.chip_id = chipRun.chip_id
JOIN rawExp 
     USING (exp_id)
LEFT JOIN distComponent 
    ON distRun.dist_id = distComponent.dist_id 
    AND stackRun.skycell_id = distComponent.component
WHERE
    distRun.state = 'new'
    AND distRun.stage = 'stack'
    AND distComponent.dist_id IS NULL
    AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned'))
    -- where hook 8 %s
) as Foo
