Index: trunk/ippTools/share/Makefile.am
===================================================================
--- trunk/ippTools/share/Makefile.am	(revision 20723)
+++ trunk/ippTools/share/Makefile.am	(revision 20734)
@@ -101,10 +101,8 @@
      magictool_definebyquery.sql \
      magictool_definebyquery_insert.sql \
-     magictool_definebyquery_select_part1.sql \
-     magictool_definebyquery_select_part2.sql \
+     magictool_definebyquery_select.sql \
      magictool_definebyquery_select_test.sql \
      magictool_definebyquery_temp_create.sql \
      magictool_definebyquery_temp_insert.sql \
-     magictool_definebyquery_temp_insert_groupby.sql \
      magictool_inputs.sql \
      magictool_inputskyfile.sql \
Index: trunk/ippTools/share/magictool_definebyquery_select.sql
===================================================================
--- trunk/ippTools/share/magictool_definebyquery_select.sql	(revision 20734)
+++ trunk/ippTools/share/magictool_definebyquery_select.sql	(revision 20734)
@@ -0,0 +1,38 @@
+-- Get a list of exposures on which magic may be performed
+SELECT
+    exp_id,
+    filter,
+    num_todo,
+    num_done
+FROM (
+    -- Number of skycells as a function of exposure
+    SELECT
+        exp_id,
+        filter,
+        COUNT(DISTINCT skycell_id) AS num_todo
+    FROM rawExp
+    JOIN chipRun USING(exp_id)
+    JOIN camRun USING(chip_id)
+    JOIN fakeRun USING(cam_id)
+    JOIN warpRun USING(fake_id)
+    JOIN warpSkyCellMap USING(warp_id)
+    JOIN warpSkyfile USING(warp_id, skycell_id)
+    WHERE
+        warpSkyfile.ignored = 0
+        AND warpRun.state = 'full'
+        --- magicSkycellNums WHERE hook %s
+    GROUP BY
+        exp_id
+    ) AS magicSkycellNums
+JOIN (
+    -- Number of completed diffs for an exposure
+    SELECT
+        exp_id,
+        COUNT(diff_id) AS num_done
+    FROM magicBestDiffs
+    GROUP BY
+        exp_id
+    ) AS magicDiffNums USING(exp_id)
+LEFT JOIN magicRun USING(exp_id)
+WHERE
+    magicRun.magic_id IS NULL
Index: trunk/ippTools/share/magictool_definebyquery_select_part1.sql
===================================================================
--- trunk/ippTools/share/magictool_definebyquery_select_part1.sql	(revision 20723)
+++ 	(revision )
@@ -1,25 +1,0 @@
--- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed
--- After this follows magictool_definebyquery_select_part2.sql
-SELECT
-    exp_id,
-    filter,
-    num_todo,
-    num_done
-FROM (
-    -- Number of skycells as a function of exposure
-    SELECT
-        exp_id,
-        filter,
-        COUNT(DISTINCT skycell_id) AS num_todo
-    FROM rawExp
-    JOIN chipRun USING(exp_id)
-    JOIN camRun USING(chip_id)
-    JOIN fakeRun USING(cam_id)
-    JOIN warpRun USING(fake_id)
-    JOIN warpSkyCellMap USING(warp_id)
-    JOIN warpSkyfile USING(warp_id, skycell_id)
-    WHERE
-        warpSkyfile.ignored = 0
-    AND warpRun.state = 'full'
--- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac)
--- INSERT HERE magictool_definebyquery_select_part2.sql
Index: trunk/ippTools/share/magictool_definebyquery_select_part2.sql
===================================================================
--- trunk/ippTools/share/magictool_definebyquery_select_part2.sql	(revision 20723)
+++ 	(revision )
@@ -1,18 +1,0 @@
--- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed
--- This follows magictool_definebyquery_select_part1.sql
-    GROUP BY
-        exp_id
-    ) AS magicSkycellNums
-JOIN (
-    -- Number of completed diffs for an exposure
-    SELECT
-        exp_id,
-        COUNT(diff_id) AS num_done
-    FROM magicBestDiffs
-    GROUP BY
-        exp_id
-    ) AS magicDiffNums USING(exp_id)
-LEFT JOIN magicRun USING(exp_id)
-WHERE
-    num_done = num_todo
-    AND magicRun.magic_id IS NULL
Index: trunk/ippTools/share/magictool_definebyquery_temp_insert.sql
===================================================================
--- trunk/ippTools/share/magictool_definebyquery_temp_insert.sql	(revision 20723)
+++ trunk/ippTools/share/magictool_definebyquery_temp_insert.sql	(revision 20734)
@@ -21,2 +21,6 @@
 WHERE
     diffSkyfile.fault = 0
+---- WHERE hook %s
+GROUP BY
+    exp_id,
+    warpSkyfile.skycell_id
Index: trunk/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql
===================================================================
--- trunk/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql	(revision 20723)
+++ 	(revision )
@@ -1,3 +1,0 @@
-GROUP BY
-    exp_id,
-    warpSkyfile.skycell_id
Index: trunk/ippTools/src/magictool.c
===================================================================
--- trunk/ippTools/src/magictool.c	(revision 20723)
+++ trunk/ippTools/src/magictool.c	(revision 20734)
@@ -155,26 +155,18 @@
         PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
 
+        psString whereClause = NULL;    // WHERE conditions
         if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
+            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringPrepend(&whereClause, "\n AND ");
+        }
+        psFree(where);
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(whereClause);
-        }
-        psFree(where);
-
-        psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql");
-        if (!groupby) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             psFree(query);
             return false;
         }
-
-        psStringAppend(&query, " %s", groupby);
-        psFree(groupby);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            return false;
-        }
+        psFree(whereClause);
         psFree(query);
     }
@@ -182,5 +174,5 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select_part1.sql");
+        psString query = pxDataGet("magictool_definebyquery_select.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -188,4 +180,5 @@
         }
 
+        psString whereClause = NULL;    // WHERE conditions
         {
             psMetadata *where = psMetadataAlloc();
@@ -194,27 +187,27 @@
 
             if (psListLength(where->list)) {
-                psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-                psStringAppend(&query, " AND %s", whereClause);
-                psFree(whereClause);
+                whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringPrepend(&whereClause, "\n AND ");
             }
             psFree(where);
         }
 
-        psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql");
-        if (!part2) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psStringAppend(&query, " %s", part2);
-        psFree(part2);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
+        // Only concern ourselves with exposures that have all diffs completed, unless we're told to only take
+        // what's available.
+        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+        if (!available) {
+            psStringAppend(&query, " AND num_done = num_todo");
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause ? whereClause : "")) {
             psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(whereClause);
             psFree(query);
             return false;
         }
-        psFree(query);
-    }
+        psFree(whereClause);
+        psFree(query);
+    }
+
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
@@ -236,4 +229,6 @@
         return true;
     }
+
+    // Parse the list of exposures ready to magic
 
     if (!psDBTransaction(config->dbh)) {
Index: trunk/ippTools/src/magictoolConfig.c
===================================================================
--- trunk/ippTools/src/magictoolConfig.c	(revision 20723)
+++ trunk/ippTools/src/magictoolConfig.c	(revision 20734)
@@ -55,4 +55,5 @@
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);