Index: /branches/eam_branch_20080719/ippTools/configure.ac =================================================================== --- /branches/eam_branch_20080719/ippTools/configure.ac (revision 18922) +++ /branches/eam_branch_20080719/ippTools/configure.ac (revision 18923) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.38]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.39]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: /branches/eam_branch_20080719/ippTools/doc/magic_flow.txt =================================================================== --- /branches/eam_branch_20080719/ippTools/doc/magic_flow.txt (revision 18923) +++ /branches/eam_branch_20080719/ippTools/doc/magic_flow.txt (revision 18923) @@ -0,0 +1,65 @@ +### Set up a run: +magictool -definerun -workdir /path/to/workdir +magictool -addinputskyfile -magic_id 6 -diff_id 1 -node 5.1 +magictool -addinputskyfile -magic_id 6 -diff_id 2 -node 5.2 +magictool -addinputskyfile -magic_id 6 -diff_id 3 -node 5.3 +magictool -addinputskyfile -magic_id 6 -diff_id 4 -node 5.4 +magictool -updaterun -magic_id 6 -state run + +### Set up a run in a single call: +magictool -queue -select_XXX + +### Generate a processing tree +magictool -totree +#magic_tree.pl --magic_id X [camera, tess_id, ra0, dec0] +* magictool -inputskyfile -magic_id 6 +* magictool -inputtree -magic_id 6 -dep_file magic_dep.mdc + +### Process the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5.1 +# dep STR NULL +# done S64 0 +# uri STR NULL +#END +#magic_process.pl --magic_id 6 --node 5.1 +magictool -inputs -magic_id 6 -node 5.1 + +magictool -addresult -magic_id 6 -node 5.1 -uri foo +magictool -addresult -magic_id 6 -node 5.2 -uri bar +magictool -addresult -magic_id 6 -node 5.3 -uri baz +magictool -addresult -magic_id 6 -node 5.4 -uri qix + +### Merging the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5A +# uri MULTI +# uri STR foo +# uri STR bar +#END +#magic_process.pl --magic_id 6 --node 5A +magictool -inputs -magic_id 6 -node 5A + +magictool -addresult -magic_id 6 -node 5A -uri foobar +magictool -addresult -magic_id 6 -node 5B -uri bazqix + +### Final merge +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR root +#END +#magic_process.pl --magic_id 6 --node root +magictool -inputs -magic_id 6 -node root + +magictool -addresult -magic_id 6 -node root -uri foobarbazqix + +### Generate the mask description +magictool -tomask +magic_mask.pl --magic_id X +magictool -inputs -magic_id 6 -node root +magictool -addmask -magic_id 6 -uri foobarbazqixqit Index: /branches/eam_branch_20080719/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branch_20080719/ippTools/share/Makefile.am (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/Makefile.am (revision 18923) @@ -58,5 +58,5 @@ dettool_tostacked.sql \ difftool_inputskyfile.sql \ - difftool_queue.sql \ + difftool_definebyquery.sql \ difftool_revertdiffskyfile.sql \ difftool_skyfile.sql \ @@ -75,8 +75,10 @@ flatcorr_find_processedimfiles.sql \ flatcorr_pending.sql \ - magictool_create_tmp_warpcomplete.sql \ - magictool_find_complete_diffed_exposures.sql \ - magictool_find_complete_warpruns.sql \ - magictool_find_unmagiced.sql \ + magictool_definebyquery_temp_create.sql \ + magictool_definebyquery_temp_insert.sql \ + magictool_definebyquery_temp_insert_groupby.sql \ + magictool_definebyquery_select_part1.sql \ + magictool_definebyquery_select_part2.sql \ + magictool_definebyquery_insert.sql \ magictool_inputskyfile.sql \ magictool_totree.sql \ Index: /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql (revision 18923) @@ -6,4 +6,5 @@ chipProcessedImfile.bg_mean_stdev, chipProcessedImfile.path_base, + rawExp.exp_id, rawExp.exp_tag, rawExp.exp_name, Index: /branches/eam_branch_20080719/ippTools/share/difftool_definebyquery.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/difftool_definebyquery.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/difftool_definebyquery.sql (revision 18923) @@ -0,0 +1,60 @@ +-- Get list of warps that can be diffed, with any associated diff, +-- and the best stack to use as a template +-- Warps without an existing diff can be identified by a NULL diff_id +SELECT + warpsToDiff.warp_id, + warpsToDiff.skycell_id, + warpsToDiff.tess_id, + warpsToDiff.filter, + warpsToDiff.good_frac, + warpsToDiff.diff_id, + warpsToDiff.kind, + current_stack_id, + best_stack_id +FROM ( + -- Get list of warps that can be diffed, with any associated diff + SELECT + warpSkyfile.warp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + warpSkyfile.good_frac, + filter, + diffInputs.diff_id, + diffInputs.kind, + diffTemplates.stack_id AS current_stack_id + FROM warpSkyfile + JOIN warpRun USING(warp_id, tess_id) + JOIN fakeRun USING(fake_id, tess_id) + JOIN camRun USING(cam_id, tess_id) + JOIN chipRun USING(chip_id, tess_id) + JOIN rawExp USING(exp_id, tess_id) + -- Check if it has an associated diff + LEFT JOIN diffInputSkyfile AS diffInputs + ON diffInputs.warp_id = warpSkyfile.warp_id + AND diffInputs.skycell_id = warpSkyfile.skycell_id + AND diffInputs.tess_id = warpSkyfile.tess_id + AND diffInputs.template = 0 -- only join input files + -- Get the stack_id currently used as a template, if any + LEFT JOIN diffInputSkyfile AS diffTemplates + ON diffTemplates.diff_id = diffInputs.diff_id + AND diffTemplates.template != 0 -- only join template files + WHERE + warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 + ) AS warpsToDiff +-- Get best stack as a function of skycell_id, filter +JOIN ( + SELECT + MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment + skycell_id, + tess_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'stop' + AND stackSumSkyfile.fault = 0 + GROUP BY + skycell_id, + tess_id, + filter + ) AS stacksForDiff USING(skycell_id, tess_id, filter) Index: /branches/eam_branch_20080719/ippTools/share/faketool_find_pendingexp.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/faketool_find_pendingexp.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/faketool_find_pendingexp.sql (revision 18923) @@ -1,5 +1,5 @@ -- this query is used by both camtool -pendingexp & camtool -addprocessedexp it -- does a little more work then is necessary for -addprocessed but it seems --- "cleaner" to use the same query for both cases +-- "cleaner" to use the same query for both cases SELECT * FROM (SELECT @@ -29,5 +29,5 @@ AND chipRun.state = 'full' AND fakeMask.label IS NULL - AND fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.fake_id IS NOT NULL GROUP BY fakeRun.fake_id Index: /branches/eam_branch_20080719/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/faketool_pendingimfile.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/faketool_pendingimfile.sql (revision 18923) @@ -1,6 +1,6 @@ -SELECT DISTINCT * FROM - -- the subselect is so where criteria can be specified without knowing - -- which table the field came from - (SELECT +SELECT DISTINCT * FROM ( + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + SELECT fakeRun.*, chipProcessedImfile.exp_id, @@ -29,16 +29,13 @@ rawExp.filelevel FROM fakeRun - JOIN camRun - USING(cam_id) - JOIN camProcessedExp - USING(cam_id) - JOIN chipRun - USING(chip_id) - JOIN chipProcessedImfile - USING(chip_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN chipProcessedImfile USING(chip_id) JOIN rawExp ON chipProcessedImfile.exp_id = rawExp.exp_id LEFT JOIN fakeProcessedImfile ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND chipProcessedImfile.class_id = fakeProcessedImfile.class_id LEFT JOIN fakeMask ON fakeRun.label = fakeMask.label @@ -47,3 +44,3 @@ AND fakeProcessedImfile.class_id IS NULL AND fakeMask.label IS NULL - ) as foo + ) as fakePendingImfile Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery.sql (revision 18923) @@ -0,0 +1,109 @@ +-- This is a sequence of SQL for searching for exposures on which +-- magic may be run, and populating the run. This file is provided as +-- a convenience, and is intended solely for playing around and +-- testing. Updates here will not affect the operation of magictool. +-- Conversely, it may not be up to date with the actual queries used +-- by magictool. + +-------------------------------------------------------------------- +-- NOTE: THIS FILE IS NOT USED BY magictool +-------------------------------------------------------------------- + +-- magictool_definebyquery_temp_create.sql +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; + + +-- magictool_definebyquery_temp_insert.sql +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id, tess_id) +JOIN camRun USING(chip_id, tess_id) +JOIN fakeRun USING(cam_id, tess_id) +JOIN warpRun USING(fake_id, tess_id) +JOIN warpSkyCellMap USING(warp_id, tess_id) +JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 +----- AND warpSkyfile.good_frac >= 0.5 +-- magictool_definebyquery_temp_insert_groupby.sql +GROUP BY + exp_id, + warpSkyfile.skycell_id +; + + +-- magictool_definebyquery_select_part1.sql +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +----- AND warpSkyfile.good_frac > 0.5 +-- INSERT HERE magictool_definebyquery_select_part2.sql +-- magictool_definebyquery_select_part2.sql +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL +; + +-- magictool_definebyquery_insert.sql +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + CONCAT(tess_id, '.', skycell_id) AS node +FROM magicBestDiffs +; Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_insert.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_insert.sql (revision 18923) @@ -0,0 +1,9 @@ +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + skycell_id +FROM magicBestDiffs +WHERE + exp_id = @EXP_ID@ -- Update this with the appropriate exp_id Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part1.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part1.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part1.sql (revision 18923) @@ -0,0 +1,25 @@ +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- INSERT HERE magictool_definebyquery_select_part2.sql Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part2.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part2.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_part2.sql (revision 18923) @@ -0,0 +1,18 @@ +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_test.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_test.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_select_test.sql (revision 18923) @@ -0,0 +1,46 @@ +-- This is the combination of the two parts of the query to get a list +-- of exposures to be magic-ed + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO magictool_definebyquery_select_part1.sql +-- AND magictool_definebyquery_select_part2.sql +-------------------------------------------------------------------------------- + +-- magictool_definebyquery_select_part1.sql +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- magictool_definebyquery_select_part2.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_create.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_create.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_create.sql (revision 18923) @@ -0,0 +1,7 @@ +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 18923) @@ -0,0 +1,22 @@ +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id, tess_id) +JOIN camRun USING(chip_id, tess_id) +JOIN fakeRun USING(cam_id, tess_id) +JOIN warpRun USING(fake_id, tess_id) +JOIN warpSkyCellMap USING(warp_id, tess_id) +JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 Index: /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 18923) +++ /branches/eam_branch_20080719/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 18923) @@ -0,0 +1,3 @@ +GROUP BY + exp_id, + warpSkyfile.skycell_id Index: anches/eam_branch_20080719/ippTools/share/magictool_find_complete_diffed_exposures.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_find_complete_diffed_exposures.sql (revision 18922) +++ (revision ) @@ -1,29 +1,0 @@ -SELECT - warp_id, - diff_id, - skycell_id, - tess_id -FROM diffSkyfile -JOIN - (SELECT - warp_id, - diff_id, - skycell_id, - tess_id, - diffSkyfile.fault - FROM warpComplete - LEFT JOIN diffInputSkyfile - USING(warp_id, skycell_id, tess_id) - LEFT JOIN diffSkyfile - USING(diff_id) - WHERE - (diffInputSkyfile.template = 1 || diffInputSkyfile.template IS NULL) - GROUP BY - warpComplete.warp_id - HAVING - SUM(diffSkyfile.fault) = 0 - AND COUNT(warpComplete.warp_id) = COUNT(diffInputSkyfile.warp_id) - AND COUNT(warpComplete.skycell_id) = COUNT(diffInputSkyfile.skycell_id) - AND COUNT(warpComplete.tess_id) = COUNT(diffInputSkyfile.tess_id) - ) as Foo - USING(diff_id) Index: anches/eam_branch_20080719/ippTools/share/magictool_find_complete_warpruns.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_find_complete_warpruns.sql (revision 18922) +++ (revision ) @@ -1,37 +1,0 @@ --- find stopped warp runsthat have had all of their skycell's warped without --- any faults -INSERT INTO warpComplete -SELECT - warp_id, - skycell_id, - tess_id -FROM warpSkyCellMap -JOIN (SELECT warp_id, ignored - FROM (SELECT - warpSkyCellMap.warp_id, - warpSkyCellMap.skycell_id, - warpSkyCellMap.tess_id, - warpSkyfile.warp_id as w2, - warpSkyfile.skycell_id as sc2, - warpSkyfile.tess_id as t2, - warpSkyfile.ignored, - warpSkyfile.fault - FROM warpRun - JOIN warpSkyCellMap - USING(warp_id, tess_id) - LEFT JOIN warpSkyfile - USING(warp_id, skycell_id, tess_id) - WHERE - warpRun.state = 'stop' - GROUP BY - warpRun.warp_id - HAVING - SUM(warpSkyfile.fault) = 0 - AND COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id) - AND COUNT(warpSkyCellMap.skycell_id) = COUNT(warpSkyfile.skycell_id) - AND COUNT(warpSkyCellMap.tess_id) = COUNT(warpSkyfile.tess_id) - ) as Foo - WHERE - ignored = 0 - ) as Bar - USING (warp_id) Index: anches/eam_branch_20080719/ippTools/share/magictool_find_unmagiced.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_find_unmagiced.sql (revision 18922) +++ (revision ) @@ -1,47 +1,0 @@ --- find warp_id that have had all of their skycell's differenced but have not --- yet been magic'd -SELECT - warp_id -FROM - (SELECT - warpRun.warp_id, - warpSkyfile.fault as f1, - diffSkyfile.fault as f2, - warpSkyCellMap.skycell_id as wscm_s, - warpSkyfile.skycell_id as wsf_s, - diffInputSkyfile.skycell_id as s2 - FROM warpRun - -- find all of the skycells that came out of a single exposure - JOIN warpSkyCellMap - USING(warp_id, tess_id) - LEFT JOIN warpSkyfile - USING(warp_id, skycell_id, tess_id) - -- find out which skycells have been diff'd - LEFT JOIN diffInputSkyfile - USING(warp_id, skycell_id, tess_id) - LEFT JOIN diffRun - USING(diff_id) - LEFT JOIN diffSkyfile - USING(diff_id) - -- find out which diff'd skycells have been added to a magic run - LEFT JOIN magicInputSkyfile - USING(diff_id) - WHERE - warpRun.state = 'stop' - AND (diffRun.state = 'stop' OR diffRun.state IS NULL) - -- AND (warpSkyfile.fault = 0 || warpSkyfile.fault IS NULL) - AND (diffInputSkyfile.template = 1 || diffInputSkyfile.template IS NULL) - AND (diffInputSkyfile.kind = 'warped' || diffInputSkyfile.kind IS NULL) - -- AND (diffSkyfile.fault = 0 || diffSkyfile.fault IS NULL) - AND magicInputSkyfile.diff_id IS NULL - AND warpSkyfile.warp_id IS NOT NULL - AND diffInputSkyfile.warp_id IS NOT NULL - GROUP BY - warpRun.warp_id - -- warpRun.warp_id, warpSkyCellMap.skycell_id, warpSkyCellMap.tess_id - HAVING - COUNT(warpSkyCellMap.skycell_id) = COUNT(warpSkyfile.skycell_id) - AND COUNT(warpSkyCellMap.skycell_id) = COUNT(diffInputSkyfile.skycell_id) - AND SUM(warpSkyfile.fault) = 0 - AND SUM(diffSkyfile.fault) = 0 -) as Foo Index: /branches/eam_branch_20080719/ippTools/share/magictool_inputs.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_inputs.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/magictool_inputs.sql (revision 18923) @@ -7,4 +7,5 @@ magicInputSkyfile.node, diffSkyfile.uri, + diffSkyfile.path_base, diffSkyfile.fault FROM magicInputSkyfile @@ -20,4 +21,5 @@ magicTree.node, magicNodeResult.uri, + NULL, -- magicNodeResult doesn't have a path_base magicNodeResult.fault FROM magicTree Index: /branches/eam_branch_20080719/ippTools/share/magictool_tomask.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_tomask.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/magictool_tomask.sql (revision 18923) @@ -1,10 +1,12 @@ SELECT magic_id, + exp_id, + camera, + magicRun.workdir, uri FROM magicRun -JOIN magicTree - USING(magic_id) -LEFT JOIN magicNodeResult - USING(magic_id, node) +JOIN rawExp USING(exp_id) +JOIN magicTree USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) WHERE magicRun.state = 'run' Index: /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_inputs.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_inputs.sql (revision 18923) @@ -1,14 +1,14 @@ SELECT magicTree.*, + rawExp.camera, + diffSkyfile.path_base, -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int magicNodeResult.magic_id IS TRUE as done FROM magicTree -JOIN magicRun - USING(magic_id) -JOIN magicInputSkyfile - USING(magic_id, node) -JOIN diffSkyfile - using(diff_id) +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id, node) +JOIN diffSkyfile USING(diff_id) +JOIN rawExp USING(exp_id) LEFT JOIN magicNodeResult ON magicTree.magic_id = magicNodeResult.magic_id Index: /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_tree.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/magictool_toprocess_tree.sql (revision 18923) @@ -1,4 +1,7 @@ SELECT magicTree.*, + exp_id, + camera, + magicRun.workdir, -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int @@ -6,6 +9,6 @@ magicNodeResult.fault IS TRUE as bad FROM magicTree -JOIN magicRun - USING(magic_id) +JOIN magicRun USING(magic_id) +JOIN rawExp USING(exp_id) LEFT JOIN magicNodeResult ON magicTree.magic_id = magicNodeResult.magic_id Index: /branches/eam_branch_20080719/ippTools/share/magictool_totree.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/magictool_totree.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/magictool_totree.sql (revision 18923) @@ -1,5 +1,12 @@ SELECT - magic_id + magic_id, + exp_id, + camera, + workdir, + ra, + decl, + tess_id FROM magicRun +JOIN rawExp USING(exp_id) LEFT JOIN magicTree USING(magic_id) Index: /branches/eam_branch_20080719/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/pxadmin_create_tables.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/pxadmin_create_tables.sql (revision 18923) @@ -235,60 +235,60 @@ CREATE TABLE chipRun ( chip_id BIGINT AUTO_INCREMENT, - exp_id BIGINT, - state VARCHAR(64), - workdir VARCHAR(255), - workdir_state VARCHAR(64), - label VARCHAR(64), + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), reduction VARCHAR(64), - expgroup VARCHAR(64), - dvodb VARCHAR(255), - tess_id VARCHAR(64), - end_stage VARCHAR(64), - PRIMARY KEY(chip_id), - KEY(chip_id), KEY(exp_id), - KEY(state), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(chip_id), + KEY(chip_id), KEY(exp_id), + KEY(state), KEY(workdir_state), - KEY(label), - KEY(expgroup), - KEY(end_stage), - INDEX(chip_id, exp_id), - FOREIGN KEY (exp_id) + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(chip_id, exp_id), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE chipProcessedImfile ( - chip_id BIGINT, - exp_id BIGINT, - class_id VARCHAR(64), - uri VARCHAR(255), - bg FLOAT, - bg_stdev FLOAT, - bg_mean_stdev FLOAT, - bias FLOAT, - bias_stdev FLOAT, - fringe_0 FLOAT, - fringe_1 FLOAT, - fringe_2 FLOAT, - sigma_ra FLOAT, - sigma_dec FLOAT, - ap_resid FLOAT, - ap_resid_stdev FLOAT, - zp_mean FLOAT, - zp_stdev FLOAT, - fwhm_major FLOAT, - fwhm_minor FLOAT, - dtime_detrend FLOAT, - dtime_photom FLOAT, - dtime_astrom FLOAT, - hostname VARCHAR(64), - n_stars INT, - n_extended INT, - n_cr INT, - n_astrom INT, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(chip_id, exp_id, class_id), - KEY(fault), - FOREIGN KEY (chip_id, exp_id) + chip_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(chip_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id), FOREIGN KEY (exp_id, class_id) @@ -297,66 +297,66 @@ CREATE TABLE chipMask ( - label VARCHAR(64), - PRIMARY KEY(label)) + label VARCHAR(64), + PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE camRun ( - cam_id BIGINT AUTO_INCREMENT, - chip_id BIGINT, - state VARCHAR(64), - workdir VARCHAR(255), - workdir_state VARCHAR(64), - label VARCHAR(64), - reduction VARCHAR(64), - expgroup VARCHAR(64), - dvodb VARCHAR(255), - tess_id VARCHAR(64), - end_stage VARCHAR(64), - PRIMARY KEY(cam_id), - KEY(cam_id), - KEY(chip_id), - KEY(state), - KEY(workdir_state), - KEY(label), - KEY(expgroup), - KEY(end_stage), - INDEX(cam_id, chip_id), - FOREIGN KEY (chip_id) - REFERENCES chipRun(chip_id)) + cam_id BIGINT AUTO_INCREMENT, + chip_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(cam_id), + KEY(cam_id), + KEY(chip_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(cam_id, chip_id), + FOREIGN KEY (chip_id) + REFERENCES chipRun(chip_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE camProcessedExp ( cam_id BIGINT, - uri VARCHAR(255), - bg FLOAT, - bg_stdev FLOAT, - bg_mean_stdev FLOAT, - bias FLOAT, - bias_stdev FLOAT, - fringe_0 FLOAT, - fringe_1 FLOAT, - fringe_2 FLOAT, - sigma_ra FLOAT, - sigma_dec FLOAT, - ap_resid FLOAT, - ap_resid_stdev FLOAT, - zp_mean FLOAT, - zp_stdev FLOAT, - fwhm_major FLOAT, - fwhm_minor FLOAT, - dtime_detrend FLOAT, - dtime_photom FLOAT, - dtime_astrom FLOAT, + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, hostname VARCHAR(64), - n_stars INT, - n_extended INT, - n_cr INT, - n_astrom INT, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(cam_id), - KEY(fault), - FOREIGN KEY (cam_id) - REFERENCES camRun(cam_id)) + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(cam_id), + KEY(fault), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -367,47 +367,47 @@ CREATE TABLE fakeRun ( - fake_id BIGINT AUTO_INCREMENT, - cam_id BIGINT, - state VARCHAR(64), - workdir VARCHAR(255), - label VARCHAR(64), - reduction VARCHAR(64), - expgroup VARCHAR(64), - dvodb VARCHAR(255), - tess_id VARCHAR(64), - end_stage VARCHAR(64), - epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(fake_id), - KEY(cam_id), - KEY(state), - KEY(label), - KEY(expgroup), - KEY(end_stage), - INDEX(fake_id, cam_id), - FOREIGN KEY (cam_id) - REFERENCES camRun(cam_id)) + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE fakeProcessedImfile ( - fake_id BIGINT AUTO_INCREMENT, - exp_id BIGINT, - class_id VARCHAR(64), - uri VARCHAR(255), - dtime_fake FLOAT, + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, hostname VARCHAR(64), - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(fake_id, exp_id, class_id), - KEY(fault), - FOREIGN KEY (fake_id) + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) REFERENCES fakeRun(fake_id), - FOREIGN KEY (exp_id, class_id) - REFERENCES rawImfile(exp_id, class_id)) + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE fakeMask ( - label VARCHAR(64), - PRIMARY KEY(label)) + label VARCHAR(64), + PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -419,103 +419,103 @@ state VARCHAR(64), filelevel VARCHAR(64), - workdir VARCHAR(255), - camera VARCHAR(64), - telescope VARCHAR(64), - exp_type VARCHAR(64), - reduction VARCHAR(64), - filter VARCHAR(64), - airmass_min FLOAT, - airmass_max FLOAT, - exp_time_min FLOAT, - exp_time_max FLOAT, - ccd_temp_min FLOAT, - ccd_temp_max FLOAT, - posang_min DOUBLE, - posang_max DOUBLE, - registered DATETIME, - time_begin DATETIME, - time_end DATETIME, - use_begin DATETIME, - use_end DATETIME, - solang_min FLOAT, - solang_max FLOAT, - label VARCHAR(64), - parent INT, - PRIMARY KEY(det_id), - KEY(det_id), - KEY(iteration), - KEY(det_type), - KEY(mode), - KEY(state), - KEY(label), - KEY(parent), - INDEX(det_id, iteration)) + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + parent INT, + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + KEY(parent), + INDEX(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detInputExp ( -det_id BIGINT, -iteration INT, -exp_id BIGINT, -include TINYINT, -PRIMARY KEY(det_id, iteration, exp_id), -INDEX(det_id, exp_id), -INDEX(det_id, iteration), -FOREIGN KEY (det_id) -REFERENCES detRun(det_id), -FOREIGN KEY (exp_id) -REFERENCES rawExp(exp_id)) +det_id BIGINT, +iteration INT, +exp_id BIGINT, +include TINYINT, +PRIMARY KEY(det_id, iteration, exp_id), +INDEX(det_id, exp_id), +INDEX(det_id, iteration), +FOREIGN KEY (det_id) +REFERENCES detRun(det_id), +FOREIGN KEY (exp_id) +REFERENCES rawExp(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detProcessedImfile ( det_id BIGINT, - exp_id BIGINT, - class_id VARCHAR(64), - uri VARCHAR(255), - recipe VARCHAR(64), - bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - fringe_0 DOUBLE, - fringe_1 DOUBLE, - fringe_2 DOUBLE, - user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, exp_id, class_id), - KEY(fault), - INDEX(det_id, class_id), - INDEX(det_id, exp_id), - FOREIGN KEY (det_id, exp_id) - REFERENCES detInputExp(det_id, exp_id), - FOREIGN KEY (exp_id, class_id) + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (exp_id, class_id) REFERENCES rawImfile(exp_id, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detProcessedExp ( - det_id BIGINT, - exp_id BIGINT, - recipe VARCHAR(64), - bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - fringe_0 DOUBLE, - fringe_1 DOUBLE, - fringe_2 DOUBLE, - user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, exp_id), - KEY(fault), - FOREIGN KEY (det_id, exp_id) - REFERENCES detInputExp(det_id, exp_id), - FOREIGN KEY (det_id, exp_id) + det_id BIGINT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id), + KEY(fault), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) REFERENCES detProcessedImfile(det_id, exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -528,114 +528,114 @@ recipe VARCHAR(64), bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, iteration, class_id), - KEY(fault), - FOREIGN KEY (det_id, iteration) - REFERENCES detInputExp(det_id, iteration), - FOREIGN KEY (det_id, class_id) + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, class_id) REFERENCES detProcessedImfile(det_id, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detNormalizedStatImfile ( - det_id BIGINT, - iteration INT, - class_id VARCHAR(64), - norm FLOAT, - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, iteration, class_id), - KEY(fault), - FOREIGN KEY (det_id, iteration) - REFERENCES detInputExp(det_id, iteration), - FOREIGN KEY (det_id, iteration, class_id) + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration, class_id) REFERENCES detStackedImfile(det_id, iteration, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detNormalizedImfile ( - det_id BIGINT, - iteration INT, - class_id VARCHAR(64), - uri VARCHAR(255), - bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, iteration, class_id), - KEY(fault), - INDEX(det_id, iteration), - FOREIGN KEY (det_id) - REFERENCES detInputExp(det_id), - FOREIGN KEY (det_id, iteration, class_id) + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id) + REFERENCES detInputExp(det_id), + FOREIGN KEY (det_id, iteration, class_id) REFERENCES detNormalizedStatImfile(det_id, iteration, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detNormalizedExp ( - det_id BIGINT, - iteration INT, - recipe VARCHAR(64), - bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, iteration), - KEY(fault), - FOREIGN KEY (det_id, iteration) - REFERENCES detInputExp(det_id, iteration), - FOREIGN KEY (det_id, iteration) + det_id BIGINT, + iteration INT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) REFERENCES detNormalizedImfile(det_id, iteration) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE detResidImfile ( - det_id BIGINT, - iteration INT, - exp_id BIGINT, - class_id VARCHAR(64), - uri VARCHAR(255), - recipe VARCHAR(64), - bg DOUBLE, - bg_stdev DOUBLE, - bg_mean_stdev DOUBLE, - bg_skewness DOUBLE, - bg_kurtosis DOUBLE, - bin_stdev DOUBLE, - fringe_0 DOUBLE, - fringe_1 DOUBLE, - fringe_2 DOUBLE, - fringe_resid_0 DOUBLE, - fringe_resid_1 DOUBLE, - fringe_resid_2 DOUBLE, + det_id BIGINT, + iteration INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, user_1 DOUBLE, - user_2 DOUBLE, - user_3 DOUBLE, - user_4 DOUBLE, - user_5 DOUBLE, - path_base VARCHAR(255), - fault SMALLINT NOT NULL, - PRIMARY KEY(det_id, iteration, exp_id, class_id), - KEY(fault), - INDEX(det_id, iteration, exp_id), - FOREIGN KEY (det_id, iteration, exp_id) - REFERENCES detInputExp(det_id, iteration, exp_id), - FOREIGN KEY (det_id, exp_id, class_id) - REFERENCES detProcessedImfile(det_id, exp_id, class_id), - FOREIGN KEY (det_id, iteration) + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, exp_id, class_id) + REFERENCES detProcessedImfile(det_id, exp_id, class_id), + FOREIGN KEY (det_id, iteration) REFERENCES detNormalizedExp(det_id, iteration) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -646,27 +646,27 @@ CREATE TABLE warpRun ( - warp_id BIGINT AUTO_INCREMENT, - fake_id BIGINT, - mode VARCHAR(64), - state VARCHAR(64), - workdir VARCHAR(255), - workdir_state VARCHAR(64), - label VARCHAR(64), - dvodb VARCHAR(255), - tess_id VARCHAR(64), - end_stage VARCHAR(64), - registered DATETIME, - magiced TINYINT, - PRIMARY KEY(warp_id), - KEY(warp_id), - KEY(fake_id), - KEY(mode), - KEY(state), - KEY(workdir_state), - KEY(label), - KEY(end_stage), - KEY(magiced), - INDEX(warp_id, fake_id), - FOREIGN KEY (fake_id) + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) REFERENCES fakeRun(fake_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -685,24 +685,24 @@ CREATE TABLE warpSkyfile ( - warp_id BIGINT, - skycell_id VARCHAR(64), - tess_id VARCHAR(64), - uri VARCHAR(255), - path_base VARCHAR(255), - bg DOUBLE, - bg_stdev DOUBLE, - dtime_warp FLOAT, - hostname VARCHAR(64), - good_frac FLOAT, - xmin INT, - xmax INT, - ymin INT, - ymax INT, - ignored TINYINT, - fault SMALLINT, - PRIMARY KEY(warp_id, skycell_id, tess_id), - KEY(good_frac), - KEY(ignored), KEY(fault), - FOREIGN KEY (warp_id, skycell_id, tess_id) + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -714,160 +714,232 @@ CREATE TABLE stackRun ( - stack_id BIGINT AUTO_INCREMENT, - state VARCHAR(64), - workdir VARCHAR(255), - dvodb VARCHAR(255), - registered DATETIME, - skycell_id VARCHAR(64), - tess_id VARCHAR(64), - filter VARCHAR(64), - PRIMARY KEY(stack_id), - KEY(stack_id), - KEY(state), - KEY(skycell_id), - KEY(tess_id) + stack_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(64), + PRIMARY KEY(stack_id), + KEY(stack_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE stackInputSkyfile ( - stack_id BIGINT, - warp_id BIGINT, - PRIMARY KEY(stack_id, warp_id), - FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), - FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id) + stack_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(stack_id, warp_id), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), + FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE stackSumSkyfile ( - stack_id BIGINT, - uri VARCHAR(255), - path_base VARCHAR(255), - bg DOUBLE, - bg_stdev DOUBLE, - dtime_stack FLOAT, - hostname VARCHAR(64), - good_frac FLOAT, - fault SMALLINT, - PRIMARY KEY(stack_id), - KEY(dtime_stack), - KEY(good_frac), - KEY(fault), - FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id) + stack_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_stack FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(stack_id), + KEY(dtime_stack), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffRun ( - diff_id BIGINT AUTO_INCREMENT, - state VARCHAR(64), - workdir VARCHAR(255), - dvodb VARCHAR(255), - registered DATETIME, - skycell_id VARCHAR(64), - tess_id VARCHAR(64), - PRIMARY KEY(diff_id), - KEY(diff_id), - KEY(state), - KEY(skycell_id), - KEY(tess_id) + diff_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + PRIMARY KEY(diff_id), + KEY(diff_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffInputSkyfile ( - diff_id BIGINT, - template TINYINT, - stack_id BIGINT, - warp_id BIGINT, - skycell_id VARCHAR(64), - tess_id VARCHAR(64), - kind VARCHAR(64), - PRIMARY KEY(diff_id, template), - KEY(stack_id), - KEY(warp_id), - KEY(skycell_id), - KEY(tess_id), - KEY(kind), - FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), - FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), - FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id) + diff_id BIGINT, + template TINYINT, + stack_id BIGINT, + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + kind VARCHAR(64), + PRIMARY KEY(diff_id, template), + KEY(stack_id), + KEY(warp_id), + KEY(skycell_id), + KEY(tess_id), + KEY(kind), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), + FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), + FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffSkyfile ( - diff_id BIGINT, - uri VARCHAR(255), - path_base VARCHAR(255), - bg DOUBLE, - bg_stdev DOUBLE, - stamps_num INT, - stamps_rms FLOAT, - sources INT, - dtime_diff FLOAT, - hostname VARCHAR(64), - good_frac FLOAT, - fault SMALLINT, - PRIMARY KEY(diff_id), - KEY(good_frac), - KEY(fault), - FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) + diff_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + stamps_num INT, + stamps_rms FLOAT, + sources INT, + dtime_diff FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(diff_id), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE magicRun ( - magic_id BIGINT AUTO_INCREMENT, - state VARCHAR(64), - workdir VARCHAR(255), - workdir_state VARCHAR(255), - label VARCHAR(64), - dvodb VARCHAR(255), - registered DATETIME, - fault SMALLINT, - PRIMARY KEY(magic_id), - KEY(magic_id), - KEY(state), - KEY(workdir_state), - KEY(label), - KEY(fault) + magic_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(255), + label VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + fault SMALLINT, + PRIMARY KEY(magic_id), + KEY(magic_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(fault), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE magicInputSkyfile ( - magic_id BIGINT, - diff_id BIGINT, - node VARCHAR(64), - PRIMARY KEY(magic_id, diff_id), - FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), - FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) + magic_id BIGINT, + diff_id BIGINT, + node VARCHAR(64), + PRIMARY KEY(magic_id, diff_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE magicTree ( - magic_id BIGINT, - node VARCHAR(64), - dep VARCHAR(64), - KEY(magic_id), - KEY(node), - KEY(dep), - INDEX(magic_id, node), - FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) + magic_id BIGINT, + node VARCHAR(64), + dep VARCHAR(64), + KEY(magic_id), + KEY(node), + KEY(dep), + INDEX(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE magicNodeResult ( - magic_id BIGINT, - node VARCHAR(64), - uri VARCHAR(255), - fault SMALLINT, - PRIMARY KEY(magic_id, node), - FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), - FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node), - KEY(fault) + magic_id BIGINT, + node VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node), + KEY(fault) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE magicMask ( - magic_id BIGINT, - uri VARCHAR(255), - streaks INT, - fault SMALLINT, - PRIMARY KEY(magic_id), - FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), - KEY(fault) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -CREATE TABLE calDB (cal_id BIGINT AUTO_INCREMENT, dvodb VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE calRun (cal_id BIGINT AUTO_INCREMENT, region VARCHAR(64), last_step VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id), KEY(last_step), FOREIGN KEY (cal_id) REFERENCES calDB(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE flatcorrRun (corr_id BIGINT AUTO_INCREMENT, dvodb VARCHAR(64), filter VARCHAR(64), state VARCHAR(64), workdir VARCHAR(255), label VARCHAR(64), stats VARCHAR(255), region VARCHAR(255), PRIMARY KEY(corr_id), KEY(corr_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE flatcorrExp (corr_id BIGINT, chip_id BIGINT, PRIMARY KEY(corr_id, chip_id), FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE pstampDataStore (ds_id BIGINT AUTO_INCREMENT, state VARCHAR(64), lastFileset VARCHAR(64), outProduct VARCHAR(64) UNIQUE, uri VARCHAR(255), PRIMARY KEY(ds_id), KEY(ds_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE pstampRequest (req_id BIGINT AUTO_INCREMENT, ds_id BIGINT, state VARCHAR(64), outFileset VARCHAR(64) UNIQUE, uri VARCHAR(255), resultsFile SMALLINT, PRIMARY KEY(req_id), KEY(req_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE pstampJob (job_id BIGINT AUTO_INCREMENT, req_id BIGINT, state VARCHAR(64), jobType VARCHAR(16), result INT, uri VARCHAR(255), outputBase VARCHAR(255), args VARCHAR(511), PRIMARY KEY(job_id, req_id), KEY(job_id), FOREIGN KEY (req_id) REFERENCES pstampRequest(req_id)) ENGINE=innodb DEFAULT CHARSET=latin1 + magic_id BIGINT, + uri VARCHAR(255), + streaks INT, + fault SMALLINT, + PRIMARY KEY(magic_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calDB ( + cal_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calRun ( + cal_id BIGINT AUTO_INCREMENT, + region VARCHAR(64), + last_step VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id), + KEY(last_step), + FOREIGN KEY (cal_id) REFERENCES calDB(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + filter VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + stats VARCHAR(255), + region VARCHAR(255), + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrExp ( + corr_id BIGINT, + chip_id BIGINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDataStore ( + ds_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + lastFileset VARCHAR(64), + outProduct VARCHAR(64) UNIQUE, + uri VARCHAR(255), + PRIMARY KEY(ds_id), + KEY(ds_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampRequest ( + req_id BIGINT AUTO_INCREMENT, + ds_id BIGINT, + state VARCHAR(64), + name VARCHAR(64) UNIQUE, + reqType VARCHAR(16), + outProduct VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(req_id), + KEY(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampJob ( + job_id BIGINT AUTO_INCREMENT, + req_id BIGINT, + rownum VARCHAR(64), + state VARCHAR(64), + jobType VARCHAR(16), + fault SMALLINT, + uri VARCHAR(255), + outputBase VARCHAR(255), + args VARCHAR(511), + PRIMARY KEY(job_id, req_id), + KEY(job_id), + FOREIGN KEY (req_id) REFERENCES pstampRequest(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1 Index: /branches/eam_branch_20080719/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/warptool_tooverlap.sql (revision 18922) +++ /branches/eam_branch_20080719/ippTools/share/warptool_tooverlap.sql (revision 18923) @@ -7,4 +7,5 @@ rawExp.camera, exp_id, + exp_tag, warpRun.magiced FROM warpRun Index: /branches/eam_branch_20080719/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/chiptool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/chiptool.c (revision 18923) @@ -522,7 +522,7 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.chip_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); @@ -534,5 +534,5 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "chipProcessedImfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -628,4 +628,5 @@ PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "="); if (!psListLength(where->list) @@ -782,5 +783,5 @@ return false; } - + if (where) { psString whereClause = psDBGenerateWhereSQL(where, "chipUnmask"); Index: /branches/eam_branch_20080719/ippTools/src/difftool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/difftool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/difftool.c (revision 18923) @@ -652,5 +652,6 @@ // Generate a single populated run -static bool populatedrun(const char *workdir, // Working directory +static bool populatedrun(psArray *list, // List of runs, to print + const char *workdir, // Working directory const char *skycell_id, // Skycell identifier const char *tess_id, // Tessellation identifier @@ -759,8 +760,6 @@ } - if (!diffRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - return false; + if (list) { + psArrayAdd(list, list->n, run); } @@ -782,4 +781,5 @@ PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); if (template_stack_id && template_warp_id) { @@ -803,10 +803,23 @@ } - return populatedrun(workdir, skycell_id, tess_id, - input_warp_id ? input_warp_id : PS_MAX_S64, - input_stack_id ? input_stack_id : PS_MAX_S64, - template_warp_id ? template_warp_id : PS_MAX_S64, - template_stack_id ? template_stack_id : PS_MAX_S64, - config); + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + + if (!populatedrun(list, workdir, skycell_id, tess_id, + input_warp_id ? input_warp_id : PS_MAX_S64, + input_stack_id ? input_stack_id : PS_MAX_S64, + template_warp_id ? template_warp_id : PS_MAX_S64, + template_stack_id ? template_stack_id : PS_MAX_S64, + config)) { + psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); return true; @@ -820,16 +833,17 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-kind", "kind", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "good_frac", ">="); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); // find all things to queue - psString query = pxDataGet("difftool_queue.sql"); + psString query = pxDataGet("difftool_definebyquery.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -837,15 +851,14 @@ } - if (template) { - // list only faulted rows - psStringAppend(&query, " %s", " AND diffInputSkyfile.template != 0"); + if (newTemplates) { + // Warps that haven't been diffed OR warps that can take a newer template in the diff + psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); } else { - // don't list faulted rows - psStringAppend(&query, " %s", - " AND (diffInputSkyfile.template = 0 OR diffInputSkyfile.template IS NULL)"); + // Only warps that haven't been diffed + psStringAppend(&query, " WHERE diff_id IS NULL"); } if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpsToDiff"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -880,4 +893,5 @@ } + psArray *list = psArrayAllocEmpty(16); // List of runs, to print long numGood = 0; // Number of good rows added for (long i = 0; i < output->n; i++) { @@ -896,5 +910,5 @@ continue; } - psS64 stack_id = psMetadataLookupS64(&mdok, row, "stack_id"); + psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); if (!mdok) { psWarning("stack_id not found --- ignoring row %ld", i); @@ -907,5 +921,6 @@ } - if (!populatedrun(workdir, skycell_id, tess_id, warp_id, PS_MAX_S64, PS_MAX_S64, stack_id, config)) { + if (!populatedrun(list, workdir, skycell_id, tess_id, warp_id, PS_MAX_S64, PS_MAX_S64, + stack_id, config)) { psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, warp_id, stack_id); @@ -916,8 +931,12 @@ numGood++; } - - psLogMsg("difftool", PS_LOG_INFO, "%ld runs added.", numGood); - psFree(output); + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); return true; Index: /branches/eam_branch_20080719/ippTools/src/faketool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/faketool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/faketool.c (revision 18923) @@ -341,7 +341,7 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-fake_id", "fake_id", "=="); - PXOPT_COPY_STR(config->args, where, "-cam_id", "cam_id", "=="); - PXOPT_COPY_STR(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); psString query = pxDataGet("faketool_find_pendingexp.sql"); @@ -798,5 +798,5 @@ return false; } - + if (where) { psString whereClause = psDBGenerateWhereSQL(where, "fakeUnmask"); Index: /branches/eam_branch_20080719/ippTools/src/faketoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/faketoolConfig.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/faketoolConfig.c (revision 18923) @@ -233,4 +233,6 @@ // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake ID", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); Index: /branches/eam_branch_20080719/ippTools/src/magictool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/magictool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/magictool.c (revision 18923) @@ -110,12 +110,16 @@ PS_ASSERT_PTR_NON_NULL(config, false); + // Required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + + // Optional PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - // create warped skycells temp table + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // Create temporary table of the best diffs { - psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -127,13 +131,35 @@ return false; } - } - - // find warped skycells + psFree(query); + } + + // Insert list of best diffs into temporary table { - psString query = pxDataGet("magictool_find_complete_warpruns.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(query); + return false; + } + + psStringAppend(&query, " %s", groupby); + psFree(groupby); if (!p_psDBRunQuery(config->dbh, query)) { @@ -145,11 +171,33 @@ } - // find the diff_id's of the warped skycells + // Get list of exposures ready to magic { - psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql"); + psString query = pxDataGet("magictool_definebyquery_select_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + } + + psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, " %s", part2); + psFree(part2); if (!p_psDBRunQuery(config->dbh, query)) { @@ -180,91 +228,82 @@ } - // pre-allocate enough hash buckets for 1 unique warp_id per row - psHash *groups = psHashAlloc(psArrayLength(output)); - // iterate over result set, and group by warp_id + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < psArrayLength(output); i++) { - // lookup the key we're grouping by - bool status = false; - psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id"); - if (!status) { - psAbort("failed to lookup value for warp_id"); - } - - psString key = psDBIntToString(warp_id); - // look to see if there is a bin for this key - psArray *groupBin = psHashLookup(groups, key); - if (!groupBin) { - // if not, create one - groupBin = psArrayAlloc(0); - psHashAdd(groups, key, groupBin); - } - psFree(key); - - // add this row to the bin - psArrayAdd(groupBin, 0, output->data[i]); - psFree(groupBin); - } - psFree(output); - - // create a magic run - psArray *grouped = psHashToArray(groups); - psFree(groups); - - // iterate over array of grouped warp_ids - for (long i = 0; i < psArrayLength(grouped); i++) { - psArray *group = grouped->data[i]; + psMetadata *row = output->data[i]; // Row of interest + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc( - 0, // ID - "reg", // state - workdir, // workdir - "dirty", // workdir_state - label, // label - dvodb, // dvodb - registered, // registered - 0 // fault - ); + magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); } + if (!magicRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - psFree(grouped); - return false; - } - - // get the assigned warp_id - psS64 magic_id = psDBLastInsertID(config->dbh); - - // insert all rows in this bin into the new magicRun - for (long j = 0; j < psArrayLength(group); j++) { - psMetadata *row = group->data[j]; - - bool status = false; - psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - psString node = psMetadataLookupStr(&status, row, "skycell_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + return false; + } + + psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_id = magic_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, magic_id); + psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); + psFree(idString); + } + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, exp_id); + psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); + psFree(idString); + } + + if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - - if (!magicInputSkyfileInsert(config->dbh, - magic_id, - diff_id, - node)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(grouped); - return false; - } - } - - // set magicRun.state to 'run' - return setmagicRunState(config, magic_id, "run"); - } - psFree(grouped); + return false; + } + psFree(thisInsert); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); return true; @@ -277,4 +316,5 @@ // required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // optional @@ -286,4 +326,5 @@ magicRunRow *run = magicRunRowAlloc( 0, // ID + exp_id, "reg", // state workdir, @@ -658,4 +699,9 @@ } + if (!node->data) { + // It's a leaf node, not done + return false; + } + if (psMetadataLookupBool(NULL, node->data, "done")) { // It's already done @@ -671,4 +717,10 @@ while ((child = psListGetAndIncrement(iter))) { psMetadata *data = child->data; + if (!data) { + // Child is a leaf node, not done + psFree(iter); + psFree(work); + return false; + } bool status = false; @@ -812,7 +864,7 @@ // find the root of the tree - pxNode *root = psHashLookup(forest, "root"); + pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root")); psFree(forest); - // pxTreePrint(stdout, root); + // pxTreePrint(stdout, root); // crawl through the tree and looking for nodes with children that are all Index: /branches/eam_branch_20080719/ippTools/src/magictoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/magictoolConfig.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/magictoolConfig.c (revision 18923) @@ -48,13 +48,17 @@ // -definebyquery - psMetadata *queueArgs = psMetadataAlloc(); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id (required)", 0); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); @@ -152,5 +156,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs); Index: /branches/eam_branch_20080719/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pstamptool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pstamptool.c (revision 18923) @@ -250,5 +250,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-ds_id", "ds_id", "=="); + PXOPT_COPY_STR(config->args, where, "-req_id", "req_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -625,6 +625,6 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); - + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); Index: /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c (revision 18923) @@ -119,5 +119,5 @@ // -pendingjob psMetadata *pendingjobArgs = psMetadataAlloc(); - psMetadataAddStr(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", NULL); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); psMetadataAddU64(pendingjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: /branches/eam_branch_20080719/ippTools/src/pxadmin.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pxadmin.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pxadmin.c (revision 18923) @@ -128,11 +128,14 @@ psString dbName = psMetadataLookupStr(&status, config->modules->complete, "DBNAME"); - fprintf (stdout, "*** delete the chip tables from database %s? ***\n", dbName); - fprintf (stdout, "*** to delete the tables, answer YES, and give password ***\n"); - fprintf (stdout, "*** WARNING: this action is permanent ***\n\n"); - - fprintf (stdout, "delete the chip tables (YES/[n]): "); - fgets (line, 128, stdin); - sscanf (line, "%s", answer); + fprintf(stdout, "*** delete the tables from database %s? ***\n", dbName); + fprintf(stdout, "*** to delete the tables, answer YES, and give password ***\n"); + fprintf(stdout, "*** WARNING: this action is permanent ***\n\n"); + + fprintf(stdout, "delete the tables (YES/[n]): "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); if (strcmp (answer, "YES")) { psError(PS_ERR_UNKNOWN, true, "tables NOT deleleted"); @@ -140,7 +143,10 @@ } - fprintf (stdout, "enter dbh connection password: "); - fgets (line, 128, stdin); - sscanf (line, "%s", answer); + fprintf(stdout, "enter dbh connection password: "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); psString dbPassword = psMetadataLookupStr(&status, config->modules->complete, "DBPASSWORD"); Index: /branches/eam_branch_20080719/ippTools/src/pxtree.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pxtree.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pxtree.c (revision 18923) @@ -88,5 +88,5 @@ ? node->parent->name : NULL); - psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); pxNode *child = NULL; while ((child = psListGetAndIncrement(iter))) { @@ -120,5 +120,5 @@ } - psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); pxNode *child = NULL; while ((child = psListGetAndIncrement(iter))) { @@ -215,4 +215,6 @@ // get from a hashlookup psFree(node); + } else if (!node->data) { + pxNodeAddData(node, data); } Index: /branches/eam_branch_20080719/ippTools/src/pzgetexp.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pzgetexp.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pzgetexp.c (revision 18923) @@ -74,8 +74,14 @@ PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + //optional + PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + // find last fileset/exp_name (if we have one) bool haveLastFileSet = false; psString lastFileSet = NULL; - { + + // -all means "request all known filesets" + if (!all) { char *query = "SELECT * from summitExp WHERE camera = \"%s\" and TELESCOPE = \"%s\" ORDER BY dateobs DESC LIMIT 1"; if (!p_psDBRunQuery(config->dbh, query, camera, telescope)) { @@ -110,4 +116,7 @@ } else { psStringAppend(&cmd, "%s --uri %s", PRODUCT_LS_CMD, uri); + } + if (timeout) { + psStringAppend(&cmd, " --timeout %d", timeout); } Index: /branches/eam_branch_20080719/ippTools/src/pzgetexpConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pzgetexpConfig.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/pzgetexpConfig.c (revision 18923) @@ -2,5 +2,5 @@ * pzgetexpConfig.c * - * Copyright (C) 2006 Joshua Hoblitt + * Copyright (C) 2006-2008 Joshua Hoblitt * * This program is free software; you can redistribute it and/or modify it @@ -51,4 +51,9 @@ psMetadataAddStr(args , PS_LIST_TAIL, "-telescope", 0, "telescope name (required)", ""); + psMetadataAddS32(args, PS_LIST_TAIL, "-timeout", 0, + "HTTP timeout", 0); + psMetadataAddBool(args, PS_LIST_TAIL, "-all", 0, + "download ALL filesets", 0); + bool status = false; @@ -61,5 +66,5 @@ fprintf(stderr, "error parsing arguments\n"); printf("\nPan-STARRS Phase Z Get Exposures Tool\n"); - printf("Usage: %s -uri -inst -telescope \n\n", + printf("Usage: %s -uri -inst -telescope [-timeout ]\n\n", argv[0]); psArgumentHelp(args); Index: /branches/eam_branch_20080719/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/stacktool.c (revision 18922) +++ /branches/eam_branch_20080719/ippTools/src/stacktool.c (revision 18923) @@ -107,5 +107,5 @@ psMetadata *whereRE = psMetadataAlloc(); // rawExp where psMetadata *whereWSF = psMetadataAlloc(); // warpSkyfile where - psMetadata *having = psMetadataAlloc(); // having + psMetadata *having = psMetadataAlloc(); // HAVING clause // map -inst -> camera, etc PXOPT_COPY_STR(config->args, whereRE, "-select_inst", "camera", "=="); @@ -128,10 +128,9 @@ PXOPT_COPY_STR(config->args, having, "-select_skycell_id", "skycell_id", "=="); + PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); PXOPT_LOOKUP_S32(randomLimit, config->args, "-random", false, false); - PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); - PXOPT_COPY_S32(config->args, having, "-min_new", "(num_warp - num_stack)", ">="); - + PXOPT_LOOKUP_S32(min_new, config->args, "-min_new", false, false); PXOPT_LOOKUP_F32(min_frac, config->args, "-min_frac", false, false); @@ -178,21 +177,34 @@ psFree(groupby); - if (psListLength(having->list)) { - psString havingClause = psDBGenerateWhereConditionSQL(having, NULL); - psStringAppend(&select, " HAVING %s", havingClause); - psFree(havingClause); - } - - // Gotta handle this one explicitly --- want to avoid divisions - if (isfinite(min_frac)) { + // Restriction on aggregated quantities using HAVING + { + psString havingClause = NULL; // HAVING string if (psListLength(having->list)) { - psStringAppend(&select, " AND"); - } else { - psStringAppend(&select, " HAVING"); - } - psStringAppend(&select, " num_warp >= %f * num_stack", (double)min_frac); - } - + havingClause = psDBGenerateWhereConditionSQL(having, NULL); + } + + if (min_new > 0) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + psStringAppend(&havingClause, + " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", + min_new, min_new); + } + if (isfinite(min_frac)) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + // Avoiding division by zero + psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", + (double)min_frac); + } + if (havingClause) { + psStringAppend(&select, " HAVING %s", havingClause); + psFree(havingClause); + } + } psFree(having); + if (!p_psDBRunQuery(config->dbh, select)) { @@ -239,4 +251,5 @@ } + psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; // Row from select @@ -249,4 +262,5 @@ psFree(output); psFree(insert); + psFree(list); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -260,4 +274,5 @@ psFree(output); psFree(insert); + psFree(list); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -271,4 +286,5 @@ psFree(output); psFree(insert); + psFree(list); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -296,4 +312,5 @@ psFree(run); psFree(insert); + psFree(list); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -306,13 +323,5 @@ run->stack_id = stack_id; - if (!stackRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - psFree(insert); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psArrayAdd(list, list->n, run); psFree(run); @@ -331,4 +340,5 @@ psFree(insert); psFree(output); + psFree(list); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -338,11 +348,18 @@ psFree(thisInsert); } + psFree(output); if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(output); + psFree(list); + return false; + } + + if (!stackRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + return false; + } + + psFree(list); return true;