Index: trunk/ippTools/src/magictool.c
===================================================================
--- trunk/ippTools/src/magictool.c	(revision 23970)
+++ trunk/ippTools/src/magictool.c	(revision 24174)
@@ -127,9 +127,12 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
+    PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
+    PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "==");
+
+    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
 
     // Get list of exposures ready to magic
@@ -146,36 +149,46 @@
         PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
 
-        psString queryWhere = NULL;     // WHERE conditions for entire query
+        psString diffWhereStr = NULL;   // WHERE conditions for diffRuns
+        psString queryWhereStr = NULL;  // WHERE conditions for entire query
         if (!available) {
-            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
-        } else {
-            // what if no skycells for the diff run completed?
-        }
+            psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'");
+        }
+        // what if no skycells for the diff run completed?
+
         if (!rerun) {
-            const char *newWhere = " magic_id IS NULL"; // String to add
-            if (queryWhere) {
-                psStringAppend(&queryWhere, " AND %s", newWhere);
-            } else {
-                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
-            }
-        }
+            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        }
+
         // now add the user specified qualifiers
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
+        if (psListLength(diffWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL);
+            psStringAppend(&diffWhereStr, "\nAND %s", whereClause);
             psFree(whereClause);
         }
-        psFree(where);
-        if (!queryWhere) {
-            psStringAppend(&queryWhere, " ");
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) {
+        if (psListLength(queryWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL);
+            psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause);
+            psFree(whereClause);
+        }
+        psFree(diffWhere);
+        psFree(queryWhere);
+
+        // Ensure the WHERE strings have something
+        if (!diffWhereStr) {
+            diffWhereStr = psStringCopy("");
+        }
+        if (!queryWhereStr) {
+            queryWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(queryWhere);
+            psFree(diffWhereStr);
+            psFree(queryWhereStr);
             psFree(query);
             return false;
         }
-        psFree(queryWhere);
+        psFree(diffWhereStr);
+        psFree(queryWhereStr);
         psFree(query);
     }
@@ -201,6 +214,18 @@
     }
 
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+
     // Parse the list of exposures ready to magic
-
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -215,7 +240,9 @@
         psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
+        bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types!
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label,
+                                            dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -296,4 +323,5 @@
 
     // optional
+    PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
@@ -305,4 +333,5 @@
             exp_id,
             diff_id ? diff_id : PS_MAX_S64,
+            inverse,
             "reg",      // state
             workdir,
@@ -363,5 +392,4 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
     PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
 
@@ -369,5 +397,4 @@
             config->dbh,
             magic_id,
-            diff_id,
             node
     );
@@ -563,5 +590,5 @@
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
-    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0");
+    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
 
     if (psListLength(where->list)) {
@@ -584,7 +611,13 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
+    // Regrettably, there are multiple WHERE hooks which call the same things different names
+    psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template
+    psMetadata *magicWhere = psMetadataAlloc();     // WHERE for selecting magic runs
+
+    PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "==");
+
+    PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -597,10 +630,20 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template
+    psString magicWhereStr = psStringCopy("");     // WHERE for selecting magic runs
+
+    if (psListLength(templatesWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL);
+        psStringAppend(&templatesWhereStr, "\nAND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
+    psFree(templatesWhere);
+
+    if (psListLength(magicWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL);
+        psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(magicWhere);
 
     // treat limit == 0 as "no limit"
@@ -611,9 +654,15 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query,
+                         templatesWhereStr, templatesWhereStr,
+                         magicWhereStr, magicWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(templatesWhereStr);
+        psFree(magicWhereStr);
         psFree(query);
         return false;
     }
+    psFree(templatesWhereStr);
+    psFree(magicWhereStr);
     psFree(query);
 
@@ -704,9 +753,9 @@
 
             bool status = false;
-            psS32 done = psMetadataLookupS32(&status, data, "done");
+            psS64 done = psMetadataLookupS64(&status, data, "done");
             if (!status) {
                 psAbort("failed to lookup value for done column");
             }
-            psS16 bad = psMetadataLookupS16(&status, data, "bad");
+            psS64 bad = psMetadataLookupS64(&status, data, "bad");
             if (!status) {
                 psAbort("failed to lookup value for bad column");
@@ -738,5 +787,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
     pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
@@ -794,5 +843,5 @@
     if (limit) {
         if (psArrayLength(output) >= limit) {
-            // we've found enough (not that limit was applied to the query so '> limit' won't happen)
+            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
             // negative simple so the default is true
             if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
@@ -936,5 +985,5 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
 
     // default values
@@ -944,5 +993,5 @@
                                magic_id,
                                node,
-                               uri,
+                               path_base,
                                fault
         )) {