Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 25418) +++ trunk/ippTools/share/Makefile.am (revision 25419) @@ -165,4 +165,5 @@ magictool_revertnode.sql \ magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ magicdstool_definebyquery_raw.sql \ magicdstool_definebyquery_chip.sql \ @@ -172,5 +173,9 @@ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ - magicdstool_todestreak.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ magicdstool_toremove.sql \ magicdstool_torestore.sql \ Index: trunk/ippTools/share/magicdstool_completedrevert.sql =================================================================== --- trunk/ippTools/share/magicdstool_completedrevert.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_completedrevert.sql (revision 25419) @@ -0,0 +1,10 @@ +SELECT + magic_ds_id, + state +FROM magicDSRun +LEFT JOIN magicDSFile USING(magic_ds_id) +WHERE + magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored' +GROUP BY magic_ds_id +HAVING COUNT(component) = 0 + Index: trunk/ippTools/share/magicdstool_todestreak.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak.sql (revision 25418) +++ (revision ) @@ -1,228 +1,0 @@ -SELECT * -FROM ( -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - rawImfile.uri as uri, - NULL as path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = rawImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'raw' - AND magicDSFile.component IS NULL -UNION - -- chipProcesedImfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - chipProcessedImfile.uri, - chipProcessedImfile.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN chipRun ON chip_id = stage_id -JOIN chipProcessedImfile USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = chipProcessedImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'chip' - AND chipRun.state = 'full' - AND chipProcessedImfile.fault = 0 - AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL -UNION - -- camProcessedExp -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - 'exposure' as component, - NULL AS uri, - camProcessedExp.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camRun ON magicDSRun.stage_id = camRun.cam_id -JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id -JOIN chipRun USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'camera' - AND camRun.state = 'full' - AND chipRun.state = 'full' - AND chipRun.magicked - AND camProcessedExp.fault = 0 - AND camProcessedExp.quality = 0 - AND magicDSFile.component IS NULL -UNION --- warpSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - warpSkyfile.skycell_id as component, - warpSkyfile.uri, - warpSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN warpRun ON warp_id = stage_id -JOIN warpSkyfile USING(warp_id) -JOIN rawExp ON magicRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = warpSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'warp' - AND warpRun.state = 'full' - AND warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - AND magicDSFile.component IS NULL -UNION --- regular diffSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways = 0 - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL --- bothways diffSkyfiles -UNION -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL -) as Foo Index: trunk/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 25419) @@ -0,0 +1,37 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + 'exposure' as component, + CAST(NULL AS CHAR(255)) AS uri, + camProcessedExp.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun ON magicDSRun.stage_id = camRun.cam_id +JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND chipRun.magicked + AND camProcessedExp.fault = 0 + AND camProcessedExp.quality = 0 + AND magicDSFile.component IS NULL Index: trunk/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 25419) @@ -0,0 +1,36 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun ON chip_id = stage_id +JOIN chipProcessedImfile USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'chip' + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND magicDSFile.component IS NULL Index: trunk/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 25419) @@ -0,0 +1,82 @@ +SELECT * FROM ( +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +) as magicDSRun +-- we need the following so this query is compatible with the other stages +WHERE magic_ds_id IS NOT NULL Index: trunk/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 25419) @@ -0,0 +1,31 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + rawImfile.uri as uri, + CAST(NULL AS CHAR(255)) as path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + AND magicDSFile.component IS NULL Index: trunk/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 25419) +++ trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 25419) @@ -0,0 +1,35 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + warpSkyfile.skycell_id as component, + warpSkyfile.uri, + warpSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN warpRun ON warp_id = stage_id +JOIN warpSkyfile USING(warp_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND magicDSFile.component IS NULL Index: trunk/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_camera.sql (revision 25418) +++ trunk/ippTools/share/magicdstool_torevert_camera.sql (revision 25419) @@ -19,4 +19,5 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'camera' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: trunk/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 25418) +++ trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 25419) @@ -18,4 +18,5 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: trunk/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_diff.sql (revision 25418) +++ trunk/ippTools/share/magicdstool_torevert_diff.sql (revision 25419) @@ -18,4 +18,5 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'diff' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: trunk/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_raw.sql (revision 25418) +++ trunk/ippTools/share/magicdstool_torevert_raw.sql (revision 25419) @@ -18,4 +18,5 @@ AND rawImfile.tmp_class_id = summitImfile.class_id) WHERE magicDSRun.stage = 'raw' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: trunk/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_warp.sql (revision 25418) +++ trunk/ippTools/share/magicdstool_torevert_warp.sql (revision 25419) @@ -21,4 +21,5 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'warp' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: trunk/ippTools/share/magictool_censor_chip.sql =================================================================== --- trunk/ippTools/share/magictool_censor_chip.sql (revision 25418) +++ trunk/ippTools/share/magictool_censor_chip.sql (revision 25419) @@ -1,7 +1,7 @@ UPDATE chipRun JOIN chipProcessedImfile USING(chip_id, exp_id) - JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id + JOIN magicDSRun ON stage_id = chip_id AND stage = 'chip' AND re_place JOIN magicDSFile USING(magic_ds_id) - JOIN magicRun USING(magic_id, exp_id) + JOIN magicRun using(magic_id) SET chipProcessedImfile.magicked = 0, chipRun.magicked = 0, Index: trunk/ippTools/share/magictool_censor_diff.sql =================================================================== --- trunk/ippTools/share/magictool_censor_diff.sql (revision 25418) +++ trunk/ippTools/share/magictool_censor_diff.sql (revision 25419) @@ -1,7 +1,7 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) +UPDATE diffRun + JOIN diffSkyfile USING(diff_id) + JOIN magicRun USING(diff_id) + JOIN magicDSRun ON stage = 'diff' AND stage_id = diff_id AND re_place JOIN magicDSFile USING(magic_ds_id) - JOIN diffRun ON magic_ds_id = magicked - JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id SET diffRun.magicked = 0, diffSkyfile.magicked = 0, Index: trunk/ippTools/share/magictool_censor_raw.sql =================================================================== --- trunk/ippTools/share/magictool_censor_raw.sql (revision 25418) +++ trunk/ippTools/share/magictool_censor_raw.sql (revision 25419) @@ -2,9 +2,9 @@ JOIN magicDSRun USING(magic_id) JOIN magicDSFile USING(magic_ds_id) - JOIN rawExp ON rawExp.magicked = magic_ds_id - JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id + JOIN rawImfile ON rawImfile.magicked = magic_ds_id +-- JOIN rawExp ON rawExp.magicked = magic_ds_id SET rawImfile.magicked = 0, - rawExp.magicked = 0, - magicDSRun.state = 'censored', +-- rawExp.magicked = 0, +-- magicDSRun.state = 'censored', magicDSFile.fault = 42