Index: trunk/ippTools/src/magictool.c
===================================================================
--- trunk/ippTools/src/magictool.c	(revision 20783)
+++ trunk/ippTools/src/magictool.c	(revision 20973)
@@ -128,48 +128,8 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // Create temporary table of the best diffs
-    {
-        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            return false;
-        }
-        psFree(query);
-    }
-
-    // Insert list of best diffs into temporary table
-    {
-        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psMetadata *where = psMetadataAlloc();
-        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-        PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-        PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
-
-        psString whereClause = NULL;    // WHERE conditions
-        if (psListLength(where->list)) {
-            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringPrepend(&whereClause, "\n AND ");
-        }
-        psFree(where);
-
-        if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(whereClause);
-            psFree(query);
-            return false;
-        }
-        psFree(whereClause);
-        psFree(query);
-    }
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
+
 
     // Get list of exposures ready to magic
@@ -181,49 +141,41 @@
         }
 
-        psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
-        {
-            psMetadata *where = psMetadataAlloc();
-            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-            PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-            PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
-
-            if (psListLength(where->list)) {
-                magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
-                psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
-            }
-            psFree(where);
-        }
-
-        // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
-        // told to only take what's available.
-        // "new" means we want a new run even if there's already a magic run defined
+        // "available" means queue magic run even though the diffRun has skycells that did not complete
+        // "rerun" means we want a new run even if there's already a magic run defined for the exposure
         PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
-        PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
+        PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
 
         psString queryWhere = NULL;     // WHERE conditions for entire query
-        if (available) {
-            psStringAppend(&queryWhere, " WHERE num_done = num_todo");
-        }
-        if (new) {
+        if (!available) {
+            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
+        } else {
+            // what if no skycells for the diff run completed?
+        }
+        if (!rerun) {
             const char *newWhere = " magic_id IS NULL"; // String to add
             if (queryWhere) {
                 psStringAppend(&queryWhere, " AND %s", newWhere);
             } else {
-                psStringAppend(&queryWhere, " WHERE %s", newWhere);
+                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
             }
         }
-        if (queryWhere) {
-            psStringAppend(&query, " %s", queryWhere);
+        // now add the user specified qualifiers
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
+            psFree(whereClause);
+        }
+        psFree(where);
+        if (!queryWhere) {
+            psStringAppend(&queryWhere, " ");
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query, queryWhere)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(queryWhere);
-        }
-
-
-        if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(magicSkyCellNumsWhere);
             psFree(query);
             return false;
         }
-        psFree(magicSkyCellNumsWhere);
+        psFree(queryWhere);
         psFree(query);
     }
@@ -262,7 +214,8 @@
         psMetadata *row = output->data[i]; // Row of interest
         psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
+        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "run", workdir, "dirty", label, dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -297,6 +250,6 @@
         {
             psString idString = NULL;
-            psStringAppend(&idString, "%" PRId64, exp_id);
-            psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
+            psStringAppend(&idString, "%" PRId64, diff_id);
+            psStringSubstitute(&thisInsert, idString, "@DIFF_ID@");
             psFree(idString);
         }
@@ -340,4 +293,5 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
+    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
 
     // optional
@@ -350,4 +304,5 @@
             0,          // ID
             exp_id,
+            diff_id,
             "reg",      // state
             workdir,