Index: trunk/ippTools/doc/magic_flow.txt =================================================================== --- trunk/ippTools/doc/magic_flow.txt (revision 18663) +++ trunk/ippTools/doc/magic_flow.txt (revision 18663) @@ -0,0 +1,65 @@ +### Set up a run: +magictool -definerun -workdir /path/to/workdir +magictool -addinputskyfile -magic_id 6 -diff_id 1 -node 5.1 +magictool -addinputskyfile -magic_id 6 -diff_id 2 -node 5.2 +magictool -addinputskyfile -magic_id 6 -diff_id 3 -node 5.3 +magictool -addinputskyfile -magic_id 6 -diff_id 4 -node 5.4 +magictool -updaterun -magic_id 6 -state run + +### Set up a run in a single call: +magictool -queue -select_XXX + +### Generate a processing tree +magictool -totree +#magic_tree.pl --magic_id X [camera, tess_id, ra0, dec0] +* magictool -inputskyfile -magic_id 6 +* magictool -inputtree -magic_id 6 -dep_file magic_dep.mdc + +### Process the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5.1 +# dep STR NULL +# done S64 0 +# uri STR NULL +#END +#magic_process.pl --magic_id 6 --node 5.1 +magictool -inputs -magic_id 6 -node 5.1 + +magictool -addresult -magic_id 6 -node 5.1 -uri foo +magictool -addresult -magic_id 6 -node 5.2 -uri bar +magictool -addresult -magic_id 6 -node 5.3 -uri baz +magictool -addresult -magic_id 6 -node 5.4 -uri qix + +### Merging the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5A +# uri MULTI +# uri STR foo +# uri STR bar +#END +#magic_process.pl --magic_id 6 --node 5A +magictool -inputs -magic_id 6 -node 5A + +magictool -addresult -magic_id 6 -node 5A -uri foobar +magictool -addresult -magic_id 6 -node 5B -uri bazqix + +### Final merge +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR root +#END +#magic_process.pl --magic_id 6 --node root +magictool -inputs -magic_id 6 -node root + +magictool -addresult -magic_id 6 -node root -uri foobarbazqix + +### Generate the mask description +magictool -tomask +magic_mask.pl --magic_id X +magictool -inputs -magic_id 6 -node root +magictool -addmask -magic_id 6 -uri foobarbazqixqit Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 18662) +++ trunk/ippTools/share/Makefile.am (revision 18663) @@ -75,8 +75,9 @@ flatcorr_find_processedimfiles.sql \ flatcorr_pending.sql \ - magictool_create_tmp_warpcomplete.sql \ - magictool_find_complete_diffed_exposures.sql \ - magictool_find_complete_warpruns.sql \ - magictool_find_unmagiced.sql \ + magictool_definebyquery_temp_create.sql \ + magictool_definebyquery_temp_insert.sql \ + magictool_definebyquery_select_part1.sql \ + magictool_definebyquery_select_part2.sql \ + magictool_definebyquery_insert.sql \ magictool_inputskyfile.sql \ magictool_totree.sql \ Index: trunk/ippTools/share/magictool_definebyquery.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery.sql (revision 18663) @@ -0,0 +1,97 @@ +-- This is a sequence of SQL for searching for exposures on which +-- magic may be run, and populating the run. This file is provided as +-- a convenience, and is intended solely for playing around and +-- testing. Updates here will not affect the operation of magictool. +-- Conversely, it may not be up to date with the actual queries used +-- by magictool. + +-------------------------------------------------------------------- +-- NOTE: THIS FILE IS NOT USED BY magictool +-------------------------------------------------------------------- + + +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; + + +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id, tess_id) +JOIN camRun USING(chip_id, tess_id) +JOIN fakeRun USING(cam_id, tess_id) +JOIN warpRun USING(fake_id, tess_id) +JOIN warpSkyCellMap USING(warp_id, tess_id) +JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + warpSkyfile.good_frac > 0.2 -- XXX Must update! + AND diffSkyfile.good_frac > 0.2 -- XXX Must update! + AND diffSkyfile.fault = 0 +GROUP BY + exp_id, + warpSkyfile.skycell_id +; + + +-- Get list of exposures ready for magic +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + AND warpSkyfile.good_frac > 0.2 -- XXX Must update! + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +WHERE + num_done = num_todo +; + + +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + 12345, + diff_id, + CONCAT(tess_id, '.', skycell_id) AS node +FROM magicBestDiffs +WHERE + exp_id = 3; Index: trunk/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 18663) @@ -0,0 +1,7 @@ +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + CONCAT(tess_id, '.', skycell_id) AS node +FROM magicBestDiffs Index: trunk/ippTools/share/magictool_definebyquery_select_part1.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_select_part1.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_select_part1.sql (revision 18663) @@ -0,0 +1,22 @@ +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- INSERT HERE magictool_definebyquery_select_part2.sql Index: trunk/ippTools/share/magictool_definebyquery_select_part2.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_select_part2.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_select_part2.sql (revision 18663) @@ -0,0 +1,18 @@ +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: trunk/ippTools/share/magictool_definebyquery_select_test.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_select_test.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_select_test.sql (revision 18663) @@ -0,0 +1,46 @@ +-- This is the combination of the two parts of the query to get a list +-- of exposures to be magic-ed + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO magictool_definebyquery_select_part1.sql +-- AND magictool_definebyquery_select_part2.sql +-------------------------------------------------------------------------------- + +-- magictool_definebyquery_select_part1.sql +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- magictool_definebyquery_select_part2.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: trunk/ippTools/share/magictool_definebyquery_temp_create.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_temp_create.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_temp_create.sql (revision 18663) @@ -0,0 +1,7 @@ +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; Index: trunk/ippTools/share/magictool_definebyquery_temp_insert.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 18663) @@ -0,0 +1,22 @@ +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id, tess_id) +JOIN camRun USING(chip_id, tess_id) +JOIN fakeRun USING(cam_id, tess_id) +JOIN warpRun USING(fake_id, tess_id) +JOIN warpSkyCellMap USING(warp_id, tess_id) +JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 Index: trunk/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql =================================================================== --- trunk/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 18663) +++ trunk/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 18663) @@ -0,0 +1,3 @@ +GROUP BY + exp_id, + warpSkyfile.skycell_id Index: trunk/ippTools/share/magictool_find_complete_diffed_exposures.sql =================================================================== --- trunk/ippTools/share/magictool_find_complete_diffed_exposures.sql (revision 18662) +++ (revision ) @@ -1,29 +1,0 @@ -SELECT - warp_id, - diff_id, - skycell_id, - tess_id -FROM diffSkyfile -JOIN - (SELECT - warp_id, - diff_id, - skycell_id, - tess_id, - diffSkyfile.fault - FROM warpComplete - LEFT JOIN diffInputSkyfile - USING(warp_id, skycell_id, tess_id) - LEFT JOIN diffSkyfile - USING(diff_id) - WHERE - (diffInputSkyfile.template = 1 || diffInputSkyfile.template IS NULL) - GROUP BY - warpComplete.warp_id - HAVING - SUM(diffSkyfile.fault) = 0 - AND COUNT(warpComplete.warp_id) = COUNT(diffInputSkyfile.warp_id) - AND COUNT(warpComplete.skycell_id) = COUNT(diffInputSkyfile.skycell_id) - AND COUNT(warpComplete.tess_id) = COUNT(diffInputSkyfile.tess_id) - ) as Foo - USING(diff_id) Index: trunk/ippTools/share/magictool_find_complete_warpruns.sql =================================================================== --- trunk/ippTools/share/magictool_find_complete_warpruns.sql (revision 18662) +++ (revision ) @@ -1,37 +1,0 @@ --- find stopped warp runsthat have had all of their skycell's warped without --- any faults -INSERT INTO warpComplete -SELECT - warp_id, - skycell_id, - tess_id -FROM warpSkyCellMap -JOIN (SELECT warp_id, ignored - FROM (SELECT - warpSkyCellMap.warp_id, - warpSkyCellMap.skycell_id, - warpSkyCellMap.tess_id, - warpSkyfile.warp_id as w2, - warpSkyfile.skycell_id as sc2, - warpSkyfile.tess_id as t2, - warpSkyfile.ignored, - warpSkyfile.fault - FROM warpRun - JOIN warpSkyCellMap - USING(warp_id, tess_id) - LEFT JOIN warpSkyfile - USING(warp_id, skycell_id, tess_id) - WHERE - warpRun.state = 'stop' - GROUP BY - warpRun.warp_id - HAVING - SUM(warpSkyfile.fault) = 0 - AND COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id) - AND COUNT(warpSkyCellMap.skycell_id) = COUNT(warpSkyfile.skycell_id) - AND COUNT(warpSkyCellMap.tess_id) = COUNT(warpSkyfile.tess_id) - ) as Foo - WHERE - ignored = 0 - ) as Bar - USING (warp_id) Index: trunk/ippTools/share/magictool_find_unmagiced.sql =================================================================== --- trunk/ippTools/share/magictool_find_unmagiced.sql (revision 18662) +++ (revision ) @@ -1,47 +1,0 @@ --- find warp_id that have had all of their skycell's differenced but have not --- yet been magic'd -SELECT - warp_id -FROM - (SELECT - warpRun.warp_id, - warpSkyfile.fault as f1, - diffSkyfile.fault as f2, - warpSkyCellMap.skycell_id as wscm_s, - warpSkyfile.skycell_id as wsf_s, - diffInputSkyfile.skycell_id as s2 - FROM warpRun - -- find all of the skycells that came out of a single exposure - JOIN warpSkyCellMap - USING(warp_id, tess_id) - LEFT JOIN warpSkyfile - USING(warp_id, skycell_id, tess_id) - -- find out which skycells have been diff'd - LEFT JOIN diffInputSkyfile - USING(warp_id, skycell_id, tess_id) - LEFT JOIN diffRun - USING(diff_id) - LEFT JOIN diffSkyfile - USING(diff_id) - -- find out which diff'd skycells have been added to a magic run - LEFT JOIN magicInputSkyfile - USING(diff_id) - WHERE - warpRun.state = 'stop' - AND (diffRun.state = 'stop' OR diffRun.state IS NULL) - -- AND (warpSkyfile.fault = 0 || warpSkyfile.fault IS NULL) - AND (diffInputSkyfile.template = 1 || diffInputSkyfile.template IS NULL) - AND (diffInputSkyfile.kind = 'warped' || diffInputSkyfile.kind IS NULL) - -- AND (diffSkyfile.fault = 0 || diffSkyfile.fault IS NULL) - AND magicInputSkyfile.diff_id IS NULL - AND warpSkyfile.warp_id IS NOT NULL - AND diffInputSkyfile.warp_id IS NOT NULL - GROUP BY - warpRun.warp_id - -- warpRun.warp_id, warpSkyCellMap.skycell_id, warpSkyCellMap.tess_id - HAVING - COUNT(warpSkyCellMap.skycell_id) = COUNT(warpSkyfile.skycell_id) - AND COUNT(warpSkyCellMap.skycell_id) = COUNT(diffInputSkyfile.skycell_id) - AND SUM(warpSkyfile.fault) = 0 - AND SUM(diffSkyfile.fault) = 0 -) as Foo Index: trunk/ippTools/src/magictool.c =================================================================== --- trunk/ippTools/src/magictool.c (revision 18662) +++ trunk/ippTools/src/magictool.c (revision 18663) @@ -110,12 +110,16 @@ PS_ASSERT_PTR_NON_NULL(config, false); + // Required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + + // Optional PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - // create warped skycells temp table + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // Create temporary table of the best diffs { - psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -127,13 +131,36 @@ return false; } - } - - // find warped skycells + psFree(query); + } + + // Insert list of best diffs into temporary table { - psString query = pxDataGet("magictool_find_complete_warpruns.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "diffSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(query); + return false; + } + + psStringAppend(&query, " %s", groupby); + psFree(groupby); if (!p_psDBRunQuery(config->dbh, query)) { @@ -145,11 +172,33 @@ } - // find the diff_id's of the warped skycells + // Get list of exposures ready to magic { - psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql"); + psString query = pxDataGet("magictool_definebyquery_select_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + } + + psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, " %s", part2); + psFree(part2); if (!p_psDBRunQuery(config->dbh, query)) { @@ -180,91 +229,71 @@ } - // pre-allocate enough hash buckets for 1 unique warp_id per row - psHash *groups = psHashAlloc(psArrayLength(output)); - // iterate over result set, and group by warp_id + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + + // iterate over array of grouped warp_ids for (long i = 0; i < psArrayLength(output); i++) { - // lookup the key we're grouping by - bool status = false; - psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id"); - if (!status) { - psAbort("failed to lookup value for warp_id"); - } - - psString key = psDBIntToString(warp_id); - // look to see if there is a bin for this key - psArray *groupBin = psHashLookup(groups, key); - if (!groupBin) { - // if not, create one - groupBin = psArrayAlloc(0); - psHashAdd(groups, key, groupBin); - } - psFree(key); - - // add this row to the bin - psArrayAdd(groupBin, 0, output->data[i]); - psFree(groupBin); - } - psFree(output); - - // create a magic run - psArray *grouped = psHashToArray(groups); - psFree(groups); - - // iterate over array of grouped warp_ids - for (long i = 0; i < psArrayLength(grouped); i++) { - psArray *group = grouped->data[i]; + psMetadata *row = output->data[i]; // Row of interest + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc( - 0, // ID - "reg", // state - workdir, // workdir - "dirty", // workdir_state - label, // label - dvodb, // dvodb - registered, // registered - 0 // fault - ); + magicRunRow *run = magicRunRowAlloc(0, exp_id, "reg", workdir, "dirty", label, dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); } + if (!magicRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - psFree(grouped); - return false; - } - - // get the assigned warp_id - psS64 magic_id = psDBLastInsertID(config->dbh); - - // insert all rows in this bin into the new magicRun - for (long j = 0; j < psArrayLength(group); j++) { - psMetadata *row = group->data[j]; - - bool status = false; - psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - psString node = psMetadataLookupStr(&status, row, "skycell_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + return false; + } + if (!magicRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - - if (!magicInputSkyfileInsert(config->dbh, - magic_id, - diff_id, - node)) { + return false; + } + psFree(run); + + psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, magic_id); + psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); + psFree(idString); + + if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(grouped); - return false; } - } - - // set magicRun.state to 'run' - return setmagicRunState(config, magic_id, "run"); - } - psFree(grouped); + return false; + } + psFree(thisInsert); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); return true; @@ -277,4 +306,5 @@ // required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // optional @@ -286,4 +316,5 @@ magicRunRow *run = magicRunRowAlloc( 0, // ID + exp_id, "reg", // state workdir, Index: trunk/ippTools/src/magictoolConfig.c =================================================================== --- trunk/ippTools/src/magictoolConfig.c (revision 18662) +++ trunk/ippTools/src/magictoolConfig.c (revision 18663) @@ -48,13 +48,17 @@ // -definebyquery - psMetadata *queueArgs = psMetadataAlloc(); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id (required)", 0); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); @@ -152,5 +156,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs);