Index: trunk/ippTools/src/difftool.c
===================================================================
--- trunk/ippTools/src/difftool.c	(revision 20719)
+++ trunk/ippTools/src/difftool.c	(revision 20973)
@@ -47,4 +47,5 @@
 
 static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
+static bool diffRunComplete(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -108,8 +109,8 @@
     // required options
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
-    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -120,11 +121,11 @@
             0,          // ID
             "reg",      // state
+            workdir,
             label,
             reduction,
-            workdir,
             NULL,       // dvodb
             registered,
-            skycell_id,
-            tess_id
+            tess_id,
+            exp_id
     );
     if (!run) {
@@ -178,26 +179,32 @@
 
     // optional
-    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
-    PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false);
-
-    // defaults to false
-    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
-
-    // must provide either stack_id or warp_id but not BOTH
-    if (!(stack_id || warp_id)) {
-        psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
-        return false;
-    }
-    if (stack_id && warp_id) {
-        psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
-        return false;
-    }
-
-    // if a warp_id was provided we need to lookup the skycell_id and tess_id
-    // from the warpRun
-    psString skycell_id = NULL;
+    PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false);
+    PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false);
+    PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false);
+    PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false);
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
+
+    // must provide either stack1 or warp1 but not BOTH
+    if (!(stack1 || warp1)) {
+        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
+        return false;
+    }
+    if (stack1 && warp1) {
+        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
+        return false;
+    }
+    // must provide either stack2 or warp2 but not BOTH
+    if (!(stack2 || warp2)) {
+        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
+        return false;
+    }
+    if (stack2 && warp2) {
+        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
+        return false;
+    }
+
+    // if a warp1 was provided we need to lookup the and tess_id from the diffRun
     psString tess_id = NULL;
-    if (warp_id) {
+    if (warp1) {
         if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -217,5 +224,4 @@
 
         diffRunRow *run = diffRunObjectFromMetadata(output->data[0]);
-        skycell_id = run->skycell_id;
         tess_id = run->tess_id;
     }
@@ -228,10 +234,10 @@
     if (!diffInputSkyfileInsert(config->dbh,
             diff_id,
-            template,
-            stack_id ? stack_id : PS_MAX_S64, // defined or NULL
-            warp_id ? warp_id : PS_MAX_S64, // defined or NULL
             skycell_id,
-            tess_id,
-            kind
+            warp1 ? warp1 : PS_MAX_S64, // defined or NULL
+            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
+            warp2 ? warp2 : PS_MAX_S64, // defined or NULL
+            stack2 ? stack2 : PS_MAX_S64, // defined or NULL
+            tess_id
         )) {
         if (!psDBRollback(config->dbh)) {
@@ -298,7 +304,6 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diff_id", "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
     PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");
-    PXOPT_COPY_STR(config->args, where,  "-kind", "kind", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -349,5 +354,5 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) {
+    if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(whereClause);
@@ -471,4 +476,5 @@
 
     PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
     PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
     PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
@@ -503,4 +509,5 @@
     if (!diffSkyfileInsert(config->dbh,
                            diff_id,
+                           skycell_id,
                            uri,
                            path_base,
@@ -524,5 +531,7 @@
                            hostname,
                            good_frac,
-                           code
+                           code,
+                           0,       // diff_image_id
+                           0        // magic_ds_id
           )) {
         if (!psDBRollback(config->dbh)) {
@@ -533,5 +542,5 @@
     }
 
-    if (!setdiffRunState(config, diff_id, "full")) {
+    if (!diffRunComplete(config)) {
         if (!psDBRollback(config->dbh)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -546,4 +555,5 @@
         return false;
     }
+
 
     return true;
@@ -762,4 +772,5 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
+                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -786,11 +797,11 @@
             0,          // ID
             "reg",      // state
+            workdir,
             label,
             reduction,
-            workdir,
             NULL,       // dvodb
             registered,
-            skycell_id,
-            tess_id
+            tess_id,
+            exp_id
     );
 
@@ -817,27 +828,10 @@
     if (!diffInputSkyfileInsert(config->dbh,
             run->diff_id,
-            true,
+            skycell_id,
+            input_warp_id,
+            input_stack_id,
+            template_warp_id,
             template_stack_id,
-            template_warp_id,
-            skycell_id,
-            tess_id,
-            NULL    // kind
-        )) {
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    // Input
-    if (!diffInputSkyfileInsert(config->dbh,
-            run->diff_id,
-            false,
-            input_stack_id,
-            input_warp_id,
-            skycell_id,
-            tess_id,
-            NULL    // kind
+            tess_id
         )) {
         if (!psDBRollback(config->dbh)) {
@@ -887,4 +881,5 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -906,4 +901,9 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
+        return false;
+    }
+    if (input_warp_id && !exp_id) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -916,4 +916,5 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
+                      exp_id,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -937,7 +938,9 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    psMetadata *expWhere = psMetadataAlloc();
     psMetadata *warpWhere = psMetadataAlloc();
     psMetadata *stackWhere = psMetadataAlloc();
 
+    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
     PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "==");
@@ -945,5 +948,4 @@
     PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
-    PXOPT_COPY_STR(config->args, warpWhere,  "-kind", "warpsToDiff.kind", "==");
     PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
@@ -955,7 +957,9 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
+    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery.sql");
+    psString query = pxDataGet("difftool_definebyquery_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -963,15 +967,16 @@
     }
 
-    if (newTemplates) {
-        // Warps that haven't been diffed OR warps that can take a newer template in the diff
-        psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)");
-    } else {
-        // Only warps that haven't been diffed
-        psStringAppend(&query, " WHERE diff_id IS NULL");
-    }
-
     psString warpQuery = NULL;
     psString stackQuery = NULL;
-
+    psString expQuery = NULL;
+
+    if (psListLength(expWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
+        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        expQuery = psStringCopy("\n");
+    }
+    psFree(expWhere);
     if (psListLength(warpWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
@@ -979,11 +984,24 @@
         psFree(whereClause);
     } else {
-        warpQuery = psStringCopy("");
+        warpQuery = psStringCopy("\n");
     }
     psFree(warpWhere);
+
+    if (!available) {
+        // diff what's available, even if warp run has some faults and is incomplete
+        psStringAppend(&warpQuery, " AND warpRun.state = 'full'");
+    }
+
+    // don't queue for exposures that have already been diff'd unless requested
+    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
+    } else {
+        diffQuery = psStringCopy("\n");
+    }
 
     if (psListLength(stackWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
-        psStringAppend(&stackQuery, "\n AND %s", whereClause);
+        psStringAppend(&stackQuery, "\nAND %s", whereClause);
         psFree(whereClause);
     } else {
@@ -992,7 +1010,8 @@
     psFree(stackWhere);
 
-    psTrace("difftool", 1, query, warpQuery, stackQuery);
-
-    if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) {
+    psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);
+
+
+    if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -1021,43 +1040,156 @@
     }
 
+    // create temporary table
+    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    query = NULL;
+
+    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     long numGood = 0;                   // Number of good rows added
+    psS64 last_exp_id = 0;
     for (long i = 0; i < output->n; i++) {
         psMetadata *row = output->data[i]; // Output row from query
-
-        // Selected parameters
         bool mdok;                      // Status of MD lookup
-        const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
-        if (!mdok || !skycell_id) {
-            psWarning("skycell_id not found --- ignoring row %ld", i);
+
+        // Take the first warp for each exposure.
+        // The list is sorted by exposure id and warp_id and the warps are in descending
+        // order. 
+        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+            return false;
+        }
+        if (exp_id == last_exp_id) {
             continue;
         }
-        const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
-        if (!mdok || !tess_id) {
-            psWarning("tess_id not found --- ignoring row %ld", i);
-            continue;
-        }
-        psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id");
-        if (!mdok) {
-            psWarning("stack_id not found --- ignoring row %ld", i);
-            continue;
+        last_exp_id = exp_id;
+
+        // clear temporary table
+        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
         }
         psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
         if (!mdok) {
-            psWarning("warp_id not found --- ignoring row %ld", i);
+            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psString filter = psMetadataLookupStr(&mdok, row, "filter");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "filter not found");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, filter, warpQuery, stackQuery)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psS64 num = psDBAffectedRows(config->dbh);
+
+        if (num == 0) {
+            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id);
             continue;
         }
 
-        if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id,
-                          PS_MAX_S64, PS_MAX_S64, stack_id, config)) {
-            psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
-                      warp_id, stack_id);
-            psErrorClear();
+        if (!available && (num != skycell_count)) {
+            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" 
+                PRId64 " need %" PRId64, num, skycell_count, warp_id);
             continue;
         }
 
+        // ok we've got one create the diffRun
+        diffRunRow *run = diffRunRowAlloc(
+                0,          // ID
+                "reg",      // state
+                workdir,
+                label,
+                reduction,
+                NULL,       // dvodb
+                registered,
+                tess_id,
+                exp_id
+        );
+
+        if (!diffRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            return true;
+        }
+        run->diff_id = psDBLastInsertID(config->dbh);
+
+        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+        psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff");
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+
+        if (!setdiffRunState(config, run->diff_id, "new")) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                run->diff_id);
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+
+        psArrayAdd(list, list->n, run);
         numGood++;
     }
     psFree(output);
+    psFree(warpQuery);
+    psFree(stackQuery);
+    psFree(skycell_query);
 
     if (!diffRunPrintObjects(stdout, list, !simple)) {
@@ -1274,2 +1406,47 @@
 }
 
+static bool diffRunComplete(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // look for completed diffRuns
+    psString query = pxDataGet("difftool_completed_runs.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
+
+        // set diffRun.state to 'stop'
+        if (!setdiffRunState(config, diff_id, "full")) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                diff_id);
+            psFree(output);
+            return false;
+        }
+    }
+
+    return true;
+}
+