Changeset 24174 for trunk/ippTools

Timestamp:

May 13, 2009, 12:37:33 PM (17 years ago)

Author:

Paul Price

Message:

Merging branches/pap_magic@24173 into trunk. One conflict in dbconfig/changes.txt resolved easily.

Location:

trunk

Files:

• . (modified) (1 prop)
• ippTools/configure.ac (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/chiptool_export_imfile.sql (modified) (1 prop)
• ippTools/share/chiptool_export_processed_imfile.sql (modified) (1 prop)
• ippTools/share/difftool_definebyquery.sql (deleted)
• ippTools/share/difftool_definebyquery_part1.sql (deleted)
• ippTools/share/difftool_definebyquery_part2.sql (deleted)
• ippTools/share/difftool_definebyquery_temp_create.sql (deleted)
• ippTools/share/difftool_definewarpstack.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack.sql )
• ippTools/share/difftool_definewarpstack_part1.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_part1.sql )
• ippTools/share/difftool_definewarpstack_part2.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_part2.sql )
• ippTools/share/difftool_definewarpstack_temp_create.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_temp_create.sql )
• ippTools/share/difftool_definewarpwarp_select.sql (modified) (2 diffs)
• ippTools/share/difftool_definewarpwarp_temp_create.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_temp_create.sql )
• ippTools/share/difftool_definewarpwarp_temp_insert.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_temp_insert.sql )
• ippTools/share/difftool_definewarpwarp_test.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_test.sql )
• ippTools/share/difftool_skyfile.sql (modified) (1 diff)
• ippTools/share/difftool_todiffskyfile.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak.sql (modified) (6 diffs)
• ippTools/share/magictool_definebyquery_insert.sql (modified) (1 diff)
• ippTools/share/magictool_definebyquery_select.sql (modified) (1 diff)
• ippTools/share/magictool_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_inputskyfile.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_tree.sql (modified) (1 diff)
• ippTools/share/magictool_totree.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (6 diffs)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/src/difftool.c (modified) (21 diffs)
• ippTools/src/difftool.h (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (5 diffs)
• ippTools/src/magictool.c (modified) (17 diffs)
• ippTools/src/magictoolConfig.c (modified) (5 diffs)

trunk/ippTools/configure.ac

@@ -18 +18 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.52]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.53]) 
 
 PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"

trunk/ippTools/share/Makefile.am

@@ -81 +81 @@
      dettool_tostacked.sql \
      difftool_completed_runs.sql \
-     difftool_definebyquery_part1.sql \
-     difftool_definebyquery_part2.sql \
-     difftool_definebyquery_temp_create.sql \
+     difftool_definewarpstack_part1.sql \
+     difftool_definewarpstack_part2.sql \
+     difftool_definewarpstack_temp_create.sql \
+     difftool_definewarpwarp_temp_create.sql \
+     difftool_definewarpwarp_temp_insert.sql \
      difftool_definewarpwarp_select.sql \
      difftool_definewarpwarp_insert.sql \

trunk/ippTools/share/difftool_definewarpwarp_select.sql

@@ -1 +1 @@
 SELECT
     inputWarpRun.warp_id AS input_warp_id,
+    inputWarpRun.tess_id AS tess_id,
     inputRawExp.exp_id AS input_exp_id,
-    inputWarpRun.tess_id AS tess_id,
+    -- The following trick pulls out the warp_id that has the smallest distance
     SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id
 FROM warpRun AS inputWarpRun
@@ -9 +10 @@
 JOIN chipRun AS inputChipRun USING(chip_id)
 JOIN rawExp AS inputRawExp USING(exp_id)
--- To find exposures that haven't been diffed:%s LEFT JOIN diffRun ON diffRun.exp_id = inputRawExp.exp_id
+-- To find exposures that haven't been diffed, insert newline here:%s LEFT JOIN diffs USING(exp_id)
 JOIN warpRun AS templateWarpRun
     ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template!

trunk/ippTools/share/difftool_skyfile.sql

@@ -8 +8 @@
     stack1,
     warp2,
-    stack2,
-    exp_id
+    stack2
 FROM diffRun
-JOIN diffSkyfile
-    USING(diff_id)
-JOIN diffInputSkyfile
-    USING (diff_id, skycell_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id, skycell_id)
 JOIN warpRun
     ON warpRun.warp_id = diffInputSkyfile.warp1
-JOIN rawExp
-    USING(exp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)

trunk/ippTools/share/difftool_todiffskyfile.sql

@@ -9 +9 @@
     diffRun.tess_id,
     diffRun.label,
-    diffRun.state
+    diffRun.state,
+    diffRun.bothways
 FROM diffRun
 JOIN diffInputSkyfile USING(diff_id)

trunk/ippTools/share/magicdstool_todestreak.sql

@@ -13 +13 @@
     rawImfile.uri as uri,
     NULL as path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -44 +45 @@
     chipProcessedImfile.uri,
     chipProcessedImfile.path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -51 +53 @@
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
+JOIN magicRun USING(magic_id)
 JOIN camProcessedExp USING(cam_id)
 JOIN chipRun ON chip_id = stage_id
@@ -79 +82 @@
     warpSkyfile.uri,
     warpSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -113 +117 @@
     magicRun.diff_id as stage_id,
     diffSkyfile.skycell_id as component,
-    diffSkyfile.uri,
+    NULL AS uri,
     diffSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -126 +131 @@
 JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON  magicInputSkyfile.diff_id = diffSkyfile.diff_id
+    ON  magicRun.diff_id = diffSkyfile.diff_id
     AND magicInputSkyfile.node = diffSkyfile.skycell_id
 LEFT JOIN magicDSFile

trunk/ippTools/share/magictool_definebyquery_insert.sql

@@ -3 +3 @@
 SELECT
     @MAGIC_ID@, -- Update this with the appropriate magic_id
-    diff_id,
     skycell_id
 FROM diffSkyfile

trunk/ippTools/share/magictool_definebyquery_select.sql

@@ -1 +1 @@
 -- Get a list of exposures on which magic may be performed
-SELECT 
+SELECT
     exp_id,
-    MAX(diffRun.diff_id) AS diff_id
-FROM diffRun
-JOIN rawExp USING(exp_id)
+    MAX(diffWarps.diff_id) AS diff_id,
+    -- The following trick pulls out the 'inverse' value for the maximum diff_id
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id,
+        0 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.exposure = 1
+    -- WHERE hook %s
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id,
+        1 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+    -- WHERE hook %s
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
 LEFT JOIN magicRun USING(exp_id)
 -- WHERE hook %s

trunk/ippTools/share/magictool_inputs.sql

@@ -1 @@
-SELECT *
-FROM (
--- Single skycells
+-- Get input details for a magic node
 SELECT
     magicRun.magic_id,
-    magicRun.state,
-    magicInputSkyfile.node,
-    diffSkyfile.diff_id,
-    diffSkyfile.uri,
-    diffSkyfile.path_base,
-    diffSkyfile.fault,
-    diffSkyfile.quality
+    magicRun.inverse, -- Using the inverse subtraction?
+    magicRun.diff_id,
+    magicInputs.node,
+    -- Only one of diff_path_base and magic_path_base should be non-NULL
+    -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL
+    magicInputs.diff_path_base, -- path_base for the diff (if any)
+    magicInputs.warp_path_base, -- path_base for the template warp (if any)
+    magicInputs.stack_path_base, -- path_base for the template stack (if any)
+    magicInputs.magic_path_base -- path_base for child nodes (if any)
 FROM magicRun
-JOIN magicInputSkyfile
-USING(magic_id)
-JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-UNION
--- Merged skycells
-SELECT
-    magicRun.magic_id,
-    magicRun.state,
-    magicTree.node,
-    0,   -- no diff_id
-    magicNodeResult.uri,
-    NULL, -- magicNodeResult doesn't have a path_base
-    magicNodeResult.fault,
-    0 as quality
-FROM magicTree
-JOIN magicRun
-    USING(magic_id)
-JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.dep = magicNodeResult.node
-) as Foo
-WHERE
-    fault = 0
-    AND quality = 0
+JOIN (
+    -- Single skycells: have uri=NULL
+    SELECT
+        magic_id,
+        magicRun.diff_id,
+        skycell_id AS node,
+        diffSkyfile.path_base AS diff_path_base,
+        warpSkyfile.path_base AS warp_path_base,
+        stackSumSkyfile.path_base AS stack_path_base,
+        NULL AS magic_path_base
+    FROM magicRun
+    JOIN magicTree USING(magic_id)
+    JOIN magicInputSkyfile USING(magic_id, node)
+    JOIN diffRun USING(diff_id)
+    JOIN diffSkyfile
+        ON diffSkyfile.diff_id = magicRun.diff_id
+        AND diffSkyfile.skycell_id = magicInputSkyfile.node
+    JOIN (
+        -- Template for non-inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp1 AS warp_id,
+            stack1 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 0
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        UNION
+        -- Template for inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp2 AS warp_id,
+            stack2 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 1
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        ) AS diffTemplates USING(magic_id, skycell_id)
+    LEFT JOIN warpSkyfile USING(warp_id, skycell_id)
+    LEFT JOIN stackSumSkyfile USING(stack_id)
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    UNION
+    -- Merged skycells: have diff_id=0, various_path_base=NULL
+    SELECT
+        magicTree.magic_id,
+        0 AS diff_id,
+        magicTree.dep,
+        NULL AS diff_path_base,
+        NULL AS warp_path_base,
+        NULL AS stack_path_base,
+        magicNodeResult.path_base AS magic_path_base
+    FROM magicTree
+    JOIN magicRun USING(magic_id)
+    JOIN magicNodeResult
+        ON magicTree.magic_id = magicNodeResult.magic_id
+        AND magicTree.dep = magicNodeResult.node
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    ) AS magicInputs USING(magic_id)

trunk/ippTools/share/magictool_inputskyfile.sql

@@ -1 +1 @@
 SELECT
     magicInputSkyfile.*,
-    diffSkyfile.uri
+    magicRun.inverse,
+    diffSkyfile.path_base
 FROM magicRun
-JOIN magicInputSkyfile
-    USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
 WHERE
     magicRun.state = 'new'

trunk/ippTools/share/magictool_toprocess_inputs.sql

@@ -1 @@
-SELECT * FROM
-(SELECT
+SELECT
     magicTree.*,
+    magicRun.workdir,
     rawExp.exp_id,
     rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    warpSkyfile.uri as template_uri
-FROM magicTree
-JOIN magicRun USING(magic_id)
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
+FROM magicRun
+JOIN magicTree USING(magic_id)
 JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN warpSkyfile
-    ON diffInputSkyfile.warp2 = warpSkyfile.warp_id
-    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+JOIN diffSkyfile -- only get nodes that match a skycell
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
     AND magicNodeResult.magic_id IS NULL
     AND magicNodeResult.node IS NULL
-    -- where hook 1 %s
-UNION
-SELECT
-    magicTree.*,
-    rawExp.exp_id,
-    rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
-    -- convert magic_id into a boolean value (1 or 0)
-    -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    stackSumSkyfile.uri as template_uri
-FROM magicTree
-JOIN magicRun USING(magic_id)
-JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN stackSumSkyfile
-    ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id
-JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
-WHERE
-    magicRun.state = 'new'
-    AND magicNodeResult.magic_id IS NULL
-    AND magicNodeResult.node IS NULL
-    -- where hook 2 %s
-) as foo
+-- WHERE hook %s

trunk/ippTools/share/magictool_toprocess_tree.sql

@@ -1 +1 @@
 SELECT
     magicTree.*,
-    exp_id,
-    camera,
     magicRun.workdir,
+    rawExp.exp_id,
+    rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    'notused' as template_uri
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
 FROM magicTree
 JOIN magicRun USING(magic_id)
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
-    -- where hook %s
+-- WHERE hook %s
 ORDER BY
     magicRun.magic_id

trunk/ippTools/share/magictool_totree.sql

@@ -6 +6 @@
     ra,
     decl,
-    tess_id
+    diffRun.tess_id
 FROM magicRun
+JOIN diffRun USING(diff_id)
 JOIN rawExp USING(exp_id)
 LEFT JOIN magicTree

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -877 +877 @@
         KEY(state),
         KEY(skycell_id),
-        KEY(tess_id)
+        KEY(tess_id),
+        KEY(label)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -933 +934 @@
         registered DATETIME,
         tess_id VARCHAR(64),
-        exp_id  BIGINT,
+        bothways TINYINT DEFAULT 0,
+        exposure TINYINT DEFAULT 0,
         magicked TINYINT,
         PRIMARY KEY(diff_id),
@@ -939 +941 @@
         KEY(state),
         KEY(tess_id),
+        KEY(label),
         FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -969 +972 @@
         diff_id BIGINT,
         skycell_id VARCHAR(64),
-        uri VARCHAR(255),
         path_base VARCHAR(255),
         data_state VARCHAR(64),
@@ -1006 +1008 @@
         exp_id BIGINT,
         diff_id BIGINT,
+        inverse TINYINT NOT NULL DEFAULT 0,
         state VARCHAR(64),
         workdir VARCHAR(255),
@@ -1046 +1049 @@
         magic_id BIGINT,
         node VARCHAR(64),
-        uri VARCHAR(255),
+        path_base VARCHAR(255),
         fault SMALLINT,
         PRIMARY KEY(magic_id, node),

trunk/ippTools/share/pxadmin_drop_tables.sql

@@ -48 +48 @@
 DROP TABLE IF EXISTS magicNodeResult;
 DROP TABLE IF EXISTS magicMask;
-DROP TABLE IF EXISTS magicSkyfileMask;
 DROP TABLE IF EXISTS magicDSRun;
 DROP TABLE IF EXISTS magicDSFile;

trunk/ippTools/src/difftool.c

@@ -40 +40 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definebyqueryMode(pxConfig *config);
+static bool definewarpstackMode(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -79 +79 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -116 +116 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -133 +134 @@
             registered,
             tess_id,
-            exp_id,
+            bothways,
+            exposure,
             false
     );
@@ -487 +489 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
@@ -521 +522 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
                            "full",
@@ -771 +771 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
-                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -802 +801 @@
             registered,
             tess_id,
-            exp_id,
+            false,
+            false,
             false       // magicked
     );
@@ -882 +882 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -902 +901 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
-        return false;
-    }
-    if (input_warp_id && !exp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -917 +911 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
-                      exp_id ? exp_id : PS_MAX_S64,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -935 +928 @@
 
 
-static bool definebyqueryMode(pxConfig *config)
+static bool definewarpstackMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -960 +953 @@
     PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery_part1.sql");
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1057 +1051 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
     // create temporary table
-    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1070 +1075 @@
     query = NULL;
 
-    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
@@ -1183 +1188 @@
                 registered,
                 tess_id,
-                exp_id,
+                true,
+                false,
                 false       // magicked
         );
@@ -1284 +1290 @@
     // Haversine formula for great circle distance
     PXOPT_COPY_F32(config->args, selectWhere, "-distance",
-                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
+                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
+                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
+                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
 
     PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);
@@ -1308 +1316 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        return false;
+    }
+
+    if (!rerun) {
+        // Need to build table of exposures with diffs
+        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
+        if (!tempCreate) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
+            psFree(tempCreate);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(tempCreate);
+
+        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
+        if (!tempInsert) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString where = psStringCopy(""); // WHERE for insertion
+        if (label) {
+            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
+            psFree(tempInsert);
+            psFree(where);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(where);
+        psFree(tempInsert);
+    }
+
+    // Get list of warps to diff
     psString select = pxDataGet("difftool_definewarpwarp_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1323 +1400 @@
     psFree(selectWhere);
 
-    if (!rerun) {
-        psStringAppend(&whereClause, "\n%s (diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
-        if (label) {
-            psStringAppend(&whereClause, " OR diffRun.label != '%s'", label);
-        }
-        psStringAppend(&whereClause, ")");
-    }
-
     if (!available) {
         psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
     }
 
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(select);
-        psFree(whereClause);
-        psFree(insertWhere);
-        return false;
+    if (!rerun) {
+        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
     }
 
     if (!p_psDBRunQueryF(config->dbh, select,
-                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                          whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
@@ -1414 +1479 @@
         psMetadata *row = results->data[i]; // Result row from query
 
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
         psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
         const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
-        if (!exp_id || !input_id || !template || !tess_id) {
+        if (!input_id || !template || !tess_id) {
             psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
             psFree(list);
@@ -1430 +1494 @@
 
         diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
-                                          tess_id, exp_id, false); // Run to insert
+                                          tess_id, true, true, false); // Run to insert
         if (!diffRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");

trunk/ippTools/src/difftool.h

@@ -34 +34 @@
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,
     DIFFTOOL_MODE_DEFINEPOPRUN,
-    DIFFTOOL_MODE_DEFINEBYQUERY,
+    DIFFTOOL_MODE_DEFINEWARPSTACK,
     DIFFTOOL_MODE_DEFINEWARPWARP,
     DIFFTOOL_MODE_PENDINGCLEANUPRUN,

trunk/ippTools/src/difftoolConfig.c

@@ -49 +49 @@
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID (required)", NULL);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways",  0,            "do the subtraction both ways?", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure",  0,            "subtraction for entire exposure?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label",  0,            "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
@@ -90 +92 @@
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
-    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", 0);
     psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
@@ -142 +143 @@
     psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
     psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-exp_id", 0,              "define exposure ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0,            "define warp ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0,            "define stack ID for template", 0);
@@ -148 +148 @@
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0,            "define stack ID for input", 0);
 
-    // -definebyquery
-    psMetadata *definebyqueryArgs = psMetadataAlloc();
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
-    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
-    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    // -definewarpstack
+    psMetadata *definewarpstackArgs = psMetadataAlloc();
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
+    psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
+    psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false);
 
     // -definewarpwarp
@@ -239 +240 @@
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
-    PXOPT_ADD_MODE("-definebyquery",    "", DIFFTOOL_MODE_DEFINEBYQUERY,     definebyqueryArgs);
+    PXOPT_ADD_MODE("-definewarpstack",  "", DIFFTOOL_MODE_DEFINEWARPSTACK,   definewarpstackArgs);
     PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);

trunk/ippTools/src/magictool.c

@@ -127 +127 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
+    PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
+    PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "==");
+
+    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
 
     // Get list of exposures ready to magic
@@ -146 +149 @@
         PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
 
-        psString queryWhere = NULL;     // WHERE conditions for entire query
+        psString diffWhereStr = NULL;   // WHERE conditions for diffRuns
+        psString queryWhereStr = NULL;  // WHERE conditions for entire query
         if (!available) {
-            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
-        } else {
-            // what if no skycells for the diff run completed?
-        }
+            psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'");
+        }
+        // what if no skycells for the diff run completed?
+
         if (!rerun) {
-            const char *newWhere = " magic_id IS NULL"; // String to add
-            if (queryWhere) {
-                psStringAppend(&queryWhere, " AND %s", newWhere);
-            } else {
-                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
-            }
-        }
+            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        }
+
         // now add the user specified qualifiers
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
+        if (psListLength(diffWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL);
+            psStringAppend(&diffWhereStr, "\nAND %s", whereClause);
             psFree(whereClause);
         }
-        psFree(where);
-        if (!queryWhere) {
-            psStringAppend(&queryWhere, " ");
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) {
+        if (psListLength(queryWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL);
+            psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause);
+            psFree(whereClause);
+        }
+        psFree(diffWhere);
+        psFree(queryWhere);
+
+        // Ensure the WHERE strings have something
+        if (!diffWhereStr) {
+            diffWhereStr = psStringCopy("");
+        }
+        if (!queryWhereStr) {
+            queryWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(queryWhere);
+            psFree(diffWhereStr);
+            psFree(queryWhereStr);
             psFree(query);
             return false;
         }
-        psFree(queryWhere);
+        psFree(diffWhereStr);
+        psFree(queryWhereStr);
         psFree(query);
     }
@@ -201 +214 @@
     }
 
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+
     // Parse the list of exposures ready to magic
-
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -215 +240 @@
         psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
+        bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types!
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label,
+                                            dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -296 +323 @@
 
     // optional
+    PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
@@ -305 +333 @@
             exp_id,
             diff_id ? diff_id : PS_MAX_S64,
+            inverse,
             "reg",      // state
             workdir,
@@ -363 +392 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
     PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
 
@@ -369 +397 @@
             config->dbh,
             magic_id,
-            diff_id,
             node
     );
@@ -563 +590 @@
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
-    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0");
+    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
 
     if (psListLength(where->list)) {
@@ -584 +611 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
+    // Regrettably, there are multiple WHERE hooks which call the same things different names
+    psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template
+    psMetadata *magicWhere = psMetadataAlloc();     // WHERE for selecting magic runs
+
+    PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "==");
+
+    PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -597 +630 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template
+    psString magicWhereStr = psStringCopy("");     // WHERE for selecting magic runs
+
+    if (psListLength(templatesWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL);
+        psStringAppend(&templatesWhereStr, "\nAND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
+    psFree(templatesWhere);
+
+    if (psListLength(magicWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL);
+        psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(magicWhere);
 
     // treat limit == 0 as "no limit"
@@ -611 +654 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query,
+                         templatesWhereStr, templatesWhereStr,
+                         magicWhereStr, magicWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(templatesWhereStr);
+        psFree(magicWhereStr);
         psFree(query);
         return false;
     }
+    psFree(templatesWhereStr);
+    psFree(magicWhereStr);
     psFree(query);
 
@@ -704 +753 @@
 
             bool status = false;
-            psS32 done = psMetadataLookupS32(&status, data, "done");
+            psS64 done = psMetadataLookupS64(&status, data, "done");
             if (!status) {
                 psAbort("failed to lookup value for done column");
             }
-            psS16 bad = psMetadataLookupS16(&status, data, "bad");
+            psS64 bad = psMetadataLookupS64(&status, data, "bad");
             if (!status) {
                 psAbort("failed to lookup value for bad column");
@@ -738 +787 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
     pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
@@ -794 +843 @@
     if (limit) {
         if (psArrayLength(output) >= limit) {
-            // we've found enough (not that limit was applied to the query so '> limit' won't happen)
+            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
             // negative simple so the default is true
             if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
@@ -936 +985 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
 
     // default values
@@ -944 +993 @@
                                magic_id,
                                node,
-                               uri,
+                               path_base,
                                fault
         )) {

trunk/ippTools/src/magictoolConfig.c

@@ -53 +53 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
     psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
@@ -58 +59 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);
 
     // -definerun
@@ -65 +67 @@
     // following argument is for compatability
     psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff_id", 0);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-inverse", 0, "use the inverse subtraction?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL);
@@ -78 +81 @@
     psMetadata *addinputskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0);
     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
 
@@ -125 +127 @@
     psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
-    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
+    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL);
     psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);