Index: branches/pap/ippTools/src/difftool.c
===================================================================
--- branches/pap/ippTools/src/difftool.c	(revision 23948)
+++ branches/pap/ippTools/src/difftool.c	(revision 25027)
@@ -40,5 +40,5 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definebyqueryMode(pxConfig *config);
+static bool definewarpstackMode(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -49,5 +49,5 @@
 static bool importrunMode(pxConfig *config);
 
-static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked);
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked);
 static bool diffRunComplete(pxConfig *config);
 
@@ -79,5 +79,5 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -116,7 +116,8 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -133,15 +134,16 @@
             registered,
             tess_id,
-            exp_id,
+            bothways,
+            exposure,
             false
     );
     if (!run) {
         psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
-        return true;
+        return false;
     }
     if (!diffRunInsertObject(config->dbh, run)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(run);
-        return true;
+        return false;
     }
 
@@ -150,7 +152,7 @@
 
     if (!diffRunPrintObject(stdout, run, !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print object");
-            psFree(run);
-            return false;
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(run);
+        return false;
     }
 
@@ -487,5 +489,4 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
@@ -511,5 +512,5 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
+    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -521,5 +522,4 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
                            "full",
@@ -582,5 +582,5 @@
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
     PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
@@ -662,4 +662,10 @@
     PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");
 
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -695,5 +701,5 @@
         psFree(query);
 
-        psLogMsg("diftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
+        psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
     }
 
@@ -739,5 +745,5 @@
 
 
-static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked)
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -749,5 +755,5 @@
     }
 
-    char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64;
+    char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) {
@@ -771,5 +777,4 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
-                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -802,6 +807,7 @@
             registered,
             tess_id,
-            exp_id,
-            false       // magicked
+            false,
+            false,
+            0       // magicked
     );
 
@@ -882,5 +888,4 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -902,9 +907,4 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
-        return false;
-    }
-    if (input_warp_id && !exp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -917,5 +917,4 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
-                      exp_id ? exp_id : PS_MAX_S64,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -935,5 +934,5 @@
 
 
-static bool definebyqueryMode(pxConfig *config)
+static bool definewarpstackMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -944,8 +943,8 @@
 
     PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "==");
     PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "==");
-    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
     PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
@@ -960,7 +959,8 @@
     PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery_part1.sql");
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1057,6 +1057,17 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
     // create temporary table
-    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1070,5 +1081,5 @@
     query = NULL;
 
-    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
@@ -1183,6 +1194,7 @@
                 registered,
                 tess_id,
-                exp_id,
-                false       // magicked
+                false,                  // bothways
+                true,                   // exposure
+                0       // magicked
         );
 
@@ -1282,7 +1294,40 @@
                    "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");
 
+    // other where restrictions:
+    PXOPT_COPY_TIME(config->args,  selectWhere, "-dateobs_begin",      "inputRawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args,  selectWhere, "-dateobs_end",        "inputRawExp.dateobs",        "<=");
+    PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_min",             "inputRawExp.ra",             ">=");
+    PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_max",             "inputRawExp.ra",             "<");
+    PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_min",           "inputRawExp.decl",           ">=");
+    PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_max",           "inputRawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-sat_pixel_frac_min", "inputRawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-sat_pixel_frac_max", "inputRawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-airmass_min",        "inputRawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-airmass_max",        "inputRawExp.airmass",        "<");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-exp_time_min",       "inputRawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-exp_time_max",       "inputRawExp.exp_time",       "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_min",             "inputRawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_max",             "inputRawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_stdev_min",       "inputRawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_stdev_max",       "inputRawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_mean_stdev_min",  "inputRawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-bg_mean_stdev_max",  "inputRawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-alt_min",            "inputRawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-alt_max",            "inputRawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-az_min",             "inputRawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-az_max",             "inputRawExp.az",             "<");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-ccd_temp_min",       "inputRawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-ccd_temp_max",       "inputRawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-posang_min",         "inputRawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args,   selectWhere, "-posang_max",         "inputRawExp.posang",         "<");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_min",      "inputRawExp.sun_angle",      ">=");
+    PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_max",      "inputRawExp.sun_angle",      "<");
+    PXOPT_COPY_STR(config->args,   selectWhere, "-comment",            "inputRawExp.comment",        "LIKE");
+
     // Haversine formula for great circle distance
     PXOPT_COPY_F32(config->args, selectWhere, "-distance",
-                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
+                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
+                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
+                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
 
     PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);
@@ -1308,7 +1353,76 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        return false;
+    }
+
+    if (!rerun) {
+        // Need to build table of exposures with diffs
+        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
+        if (!tempCreate) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
+            psFree(tempCreate);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(tempCreate);
+
+        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
+        if (!tempInsert) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString where = psStringCopy(""); // WHERE for insertion
+        if (label) {
+            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
+            psFree(tempInsert);
+            psFree(where);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(where);
+        psFree(tempInsert);
+    }
+
+    // Get list of warps to diff
     psString select = pxDataGet("difftool_definewarpwarp_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1323,26 +1437,17 @@
     psFree(selectWhere);
 
+    if (!available) {
+        psStringAppend(&whereClause,
+                       "\n%s inputWarpRun.state = 'full'"
+                       "AND templateWarpRun.state = 'full'",
+                       whereClause ? "AND" : "WHERE");
+    }
+
     if (!rerun) {
-        psStringAppend(&whereClause, "\n%s (diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
-        if (label) {
-            psStringAppend(&whereClause, " OR diffRun.label != %s", label);
-        }
-        psStringAppend(&whereClause, ")");
-    }
-
-    if (!available) {
-        psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
-    }
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(select);
-        psFree(whereClause);
-        psFree(insertWhere);
-        return false;
+        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
     }
 
     if (!p_psDBRunQueryF(config->dbh, select,
-                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                          whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
@@ -1414,9 +1519,8 @@
         psMetadata *row = results->data[i]; // Result row from query
 
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
         psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
         const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
-        if (!exp_id || !input_id || !template || !tess_id) {
+        if (!input_id || !template || !tess_id) {
             psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
             psFree(list);
@@ -1430,5 +1534,5 @@
 
         diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
-                                          tess_id, exp_id, false); // Run to insert
+                                          tess_id, true, true, false); // Run to insert
         if (!diffRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1511,5 +1615,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanuprun.sql");
@@ -1576,5 +1680,5 @@
         PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
     }
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
@@ -1738,5 +1842,5 @@
 
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
-        bool magicked = psMetadataLookupBool(NULL, row, "magicked");
+        psS64 magicked = psMetadataLookupS64(NULL, row, "magicked");
 
         // set diffRun.state to 'stop'