Index: trunk/ippTools/share/disttool_definebyquery_select.sql =================================================================== --- trunk/ippTools/share/disttool_definebyquery_select.sql (revision 23390) +++ trunk/ippTools/share/disttool_definebyquery_select.sql (revision 23394) @@ -1,2 +1,5 @@ +-- Maybe it would be better to split the query into multiple modes +-- the way it is outstanding rawExp's to bundle will block other stages +-- from being processed SELECT stage, @@ -5,8 +8,7 @@ clean FROM ( - SELECT + SELECT DISTINCT -- we use distinct to handle multiple chip runs. Distribution will find right one 'raw' AS stage, rawExp.exp_id AS stage_id, - chipRun.chip_id AS chip_id, rawExp.obs_mode, distTarget.clean @@ -16,105 +18,108 @@ JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage = 'raw' AND distRun.stage_id = exp_id - WHERE - chipRun.state = 'full' -- XXX: we need a completed chip stage magicDSRun because we --- AND rawExp.magicked and chipRun.magicked -- to support other cameras need to add this optionally - -- need the mask file + WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND chipRun.state = 'full' -- XXX: we need a completed chip stage magicDSRun because we + -- need the mask file to NAN masked pixels + -- to support cameras that don't need magic disttool adds the magicked restriction + -- \nAND rawExp.magicked AND chipRun.magicked here: + AND rawExp.magicked AND chipRun.magicked + -- raw magicked HOOK %s AND distRun.dist_id IS NULL -- no existing distRun + -- raw where hook %s UNION SELECT + 'chip' as stage, chipRun.chip_id as stage_id, - 'chip' as stage, rawExp.obs_mode, distTarget.clean FROM chipRun JOIN rawExp USING(exp_id) - JOIN distTarget ON rawExp.obs_mode = distTarget.obs_mode AND distTarget.stage = 'chip' + JOIN distTarget ON distTarget.stage = 'chip' AND rawExp.obs_mode = distTarget.obs_mode JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distTarget.stage = 'chip' AND distRun.stage_id = chipRun.chip_id WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND (chipRun.magicked > 0 OR rawExp.magicked > 0) - AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned')) AND distRun.dist_id IS NULL -- no existing distRun + AND ((!distTarget.clean AND (chipRun.state = 'full') + -- If camera requires files to be magicked before distibution + -- the following gets added by disttool + -- AND chipRun.magicked + -- chip magicked HOOK %s + ) + OR (distTarget.clean AND chipRun.state = 'cleaned')) + -- chip where hook %s UNION SELECT - max(magic_ds_id) AS magic_ds_id, -- This assumes that the last destreak run is the best 'cam' as stage, camRun.cam_id as stage_id, rawExp.obs_mode, distTarget.clean - FROM magicDSRun - JOIN chipRun ON chipRun.chip_id = magicDSRun.stage_id AND magicDSRun.stage = 'chip' - JOIN camRun USING(chip_id) + FROM camRun + JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) - JOIN distTarget ON rawExp.obs_mode = distTarget.obs_mode AND distTarget.stage = 'cam' + JOIN distTarget ON distTarget.stage = 'cam' AND rawExp.obs_mode = distTarget.obs_mode JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage = 'cam' AND camRun.cam_id = distRun.stage_id - WHERE magicDSRun.state = 'full' - -- replace hook %s - -- AND re_place > 0 -- if re_place is 0 we need to get destreaked image from backup - -- if re_place is 1 we get it from the stageRun - -- This is really an issue for development, not real operations - AND ((camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned')) + WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' AND distRun.dist_id IS NULL -- no existing distRun - GROUP BY chipRun.chip_id + AND ((!distTarget.clean AND camRun.state = 'full' + -- need magicked chip run because camera mask are destreaked by chip destreaking + -- the following line + -- AND chipRun.magicked + -- cam magicked HOOK %s + ) + OR (distTarget.clean AND camRun.state = 'cleaned') + ) + -- cam where hook %s UNION - -- we really don't require a magicDSRun for fake stage do we? SELECT - max(magic_ds_id) AS magic_ds_id, -- This assumes that the last destreak run is the best 'fake' as stage, fakeRun.fake_id as stage_id, rawExp.obs_mode, distTarget.clean - FROM magicDSRun - JOIN chipRun ON chipRun.chip_id = magicDSRun.stage_id AND magicDSRun.stage = 'chip' - JOIN camRun USING(chip_id) - JOIN fakeRun ON fakeRun.cam_id = camRun.cam_id + FROM fakeRun + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) JOIN distTarget ON rawExp.obs_mode = distTarget.obs_mode AND distTarget.stage = 'fake' JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage = 'fake' AND camRun.cam_id = distRun.stage_id - WHERE magicDSRun.state = 'full' - -- replace hook %s - -- AND re_place > 0 -- if re_place is 0 we need to get destreaked image from backup - -- if re_place is 1 we get it from the stageRun - -- This is really an issue for development, not real operations + WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL -- no existing distRun AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned')) - AND distRun.dist_id IS NULL -- no existing distRun - - GROUP BY chipRun.chip_id -- by chip run to find the latest magicDSRun + -- fake where hook %s UNION SELECT - max(magic_ds_id) AS magic_ds_id, 'warp' as stage, warpRun.warp_id AS stage_id, - obs_mode, + rawExp.obs_mode, distTarget.clean - FROM magicDSRun - JOIN warpRun ON warpRun.warp_id = magicDSRun.stage_id AND magicDSRun.stage = 'warp' + FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun ON fakeRun.cam_id = camRun.cam_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) - JOIN distTarget USING(obs_mode, stage) + JOIN distTarget ON distTarget.stage = 'warp' AND rawExp.obs_mode = distTarget.obs_mode JOIN rcInterest USING(target_id) - LEFT JOIN distRun USING(stage, stage_id) - WHERE magicDSRun.state = 'full' - -- replace hook %s - -- AND re_place > 0 -- if re_place is 0 we need to get destreaked image from backup - -- if re_place is 1 we get it from the stageRun - -- This is really an issue for development, not real operations - AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned')) - AND distRun.dist_id IS NULL -- no existing distRun - GROUP BY warpRun.warp_id - - + LEFT JOIN distRun ON distRun.stage = 'warp' AND (distRun.stage_id = warp_id) + WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((warpRun.state = 'full' + -- AND warpRun.magicked + -- warp magicked hook %s + ) + OR (distTarget.clean AND warpRun.state = 'cleaned') + ) + -- warp where hook %s ) as foo -