Index: branches/simmosaic_branches/ippTools/src/stacktool.c
===================================================================
--- branches/simmosaic_branches/ippTools/src/stacktool.c	(revision 24860)
+++ branches/simmosaic_branches/ippTools/src/stacktool.c	(revision 27839)
@@ -109,8 +109,14 @@
 
     // required options
-    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
-    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
 
     psMetadata *where = psMetadataAlloc();
@@ -118,4 +124,5 @@
 
     // select based on properties of the raw exposures
+    PXOPT_COPY_STR(config->args,  where, "-select_comment",            "rawExp.comment", "LIKE");
     PXOPT_COPY_STR(config->args,  where, "-select_inst",               "rawExp.camera", "==");
     PXOPT_COPY_STR(config->args,  where, "-select_telescope",          "rawExp.telescope", "==");
@@ -133,17 +140,32 @@
     PXOPT_COPY_F32(config->args,  where, "-select_posang_min",         "rawExp.posang", ">=");
     PXOPT_COPY_F32(config->args,  where, "-select_posang_max",         "rawExp.posang", "<=");
-    PXOPT_COPY_F32(config->args,  where, "-select_solang_min",         "rawExp.solang", ">=");
-    PXOPT_COPY_F32(config->args,  where, "-select_solang_max",         "rawExp.solang", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_sun_angle_min",      "rawExp.sun_angle", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-select_sun_angle_max",      "rawExp.sun_angle", "<=");
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_major_min",     "camProcessedExp.fwhm_major", ">=");
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_major_max",     "camProcessedExp.fwhm_major", "<=");
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_minor_min",     "camProcessedExp.fwhm_minor", ">=");
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_minor_max",     "camProcessedExp.fwhm_minor", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m2_max",          "camProcessedExp.iq_m2", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m2_min",          "camProcessedExp.iq_m2", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m3_max",          "camProcessedExp.iq_m3", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m4_min",          "camProcessedExp.iq_m4", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m4_max",          "camProcessedExp.iq_m4", "<=");
+
     PXOPT_COPY_STR(config->args,  where, "-select_exp_type",           "rawExp.exp_type", "==");
     PXOPT_COPY_F32(config->args,  where, "-select_good_frac_min",      "warpSkyfile.good_frac", ">=");
     PXOPT_COPY_STR(config->args,  where, "-select_skycell_id",         "warpSkyfile.skycell_id", "==");
-    pxAddLabelSearchArgs (config, where, "-select_label",              "warpRun.label", "LIKE");
+    PXOPT_COPY_STR(config->args,  where, "-select_data_group",         "warpRun.data_group", "==");
+    pxAddLabelSearchArgs (config, where, "-select_label",              "warpRun.label", "LIKE"); // define using warp label
 
     // these are used to build the HAVING restriction
     PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">=");
+    PXOPT_LOOKUP_S32(min_num,     config->args, "-min_num",  false, false);
+    if (min_num < 2) {
+        psError(PXTOOLS_ERR_CONFIG, true, "Require at least two inputs for a stack, but min_num = %d",
+                min_num);
+        psFree(where);
+        psFree(having);
+        return false;
+    }
 
     // other options applied outside of the WHERE
@@ -154,33 +176,31 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
-    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
-
-    if (!psListLength(where->list) &&
-        !psMetadataLookupBool(NULL, config->args, "-all")) {
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
         psFree(where);
-        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
-        return false;
-    }
-
-    psString select = pxDataGet("stacktool_definebyquery_part1.sql");
+        psFree(having);
+        return false;
+    }
+
+    psString select = pxDataGet("stacktool_definebyquery_select.sql");
     if (!select) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        psFree(having);
+        return false;
+    }
+
+    psString where1 = psStringCopy("");
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&select, " AND %s", whereClause);
+        psStringAppend(&where1, "\nAND %s", whereClause);
         psFree(whereClause);
     }
 
-    psString groupby = pxDataGet("stacktool_definebyquery_part2.sql");
-    if (!groupby) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        psFree(where);
-        return false;
-    }
-    psStringAppend(&select, " %s", groupby);
-    psFree(groupby);
+    psString where2 = psStringCopy("");
+    if (label) {
+        psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label);
+    }
 
     // Restriction on aggregated quantities using HAVING
@@ -214,11 +234,14 @@
     psFree(having);
 
-    if (!p_psDBRunQuery(config->dbh, select)) {
+    if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(select);
-        psFree(where);
+        psFree(where1);
+        psFree(where2);
         return false;
     }
     psFree(select);
+    psFree(where1);
+    psFree(where2);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -262,5 +285,5 @@
     }
     if (!insert) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
         return false;
@@ -277,5 +300,5 @@
         psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql");
         if (!part2) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
             psFree(insert);
             return false;
@@ -339,4 +362,6 @@
             workdir,
             label,
+            data_group ? data_group : label,
+            dist_group,
             reduction,
             dvodb,
@@ -344,5 +369,6 @@
             skycell_id,
             tess_id,
-            filter);
+            filter,
+            note);
 
         if (!stackRunInsertObject(config->dbh, run)) {
@@ -462,5 +488,5 @@
 
     // required options
-    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false);
     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
@@ -469,10 +495,13 @@
     // default
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
 
     // options
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
-    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false);
 
     // we have to support multipe exp_ids
@@ -489,4 +518,6 @@
         workdir,
         label,
+        data_group ? data_group : label,
+        dist_group,
         reduction,
         dvodb,
@@ -494,5 +525,6 @@
         skycell_id,
         tess_id,
-        filter);
+        filter,
+        note);
 
     if (!run) {
@@ -563,13 +595,44 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false);
+#ifdef notdef
+    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
-
-    if (state) {
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+#endif
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-stack_id",  "stack_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "state",     "==");
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE stackRun");
+
+    // pxUpdateRun gets parameters from config->args and updates
+    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true);
+
+    psFree(query);
+    psFree(where);
+
+    return result;
+
+#ifdef notdef
+    // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully).
+
+    if ((state)&&(stack_id)) {
         // set detRun.state to state
         return setstackRunState(config, stack_id, state);
     }
 
-    return true;
+    if ((state)&&(label)) {
+      return setstackRunStateByLabel(config, label, state);
+    }
+
+    psError(PS_ERR_UNKNOWN, false, "Required options not found.");
+    return false;
+#endif
 }
 
@@ -613,5 +676,5 @@
     psString query = pxDataGet("stacktool_inputskyfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -619,5 +682,5 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile");
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
     }
@@ -687,12 +750,13 @@
     psString query = pxDataGet("stacktool_tosum.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereClause = psStringCopy(""); // WHERE conditions to add
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
+        psString new = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&whereClause, "\nAND %s", new);
+        psFree(new);
     }
     psFree(where);
@@ -705,5 +769,5 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -853,8 +917,18 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
+    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+
+//  The following three selectors are incompatible with the sql so omit them
+//    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
+//     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+//    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -863,5 +937,5 @@
     psString query = pxDataGet("stacktool_sumskyfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -871,5 +945,9 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    }
+    } else if (!all) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+
     psFree(where);
 
@@ -928,97 +1006,36 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
 
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
-        return false;
-    }
-
-    int numStacks;                      // Number of stacks affected
-
-    // Update state to 'new'
-    {
-        psString update = pxDataGet("stacktool_revertsumskyfile_update.sql");
-        if (!update) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&update, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-
-        if (!p_psDBRunQuery(config->dbh, update)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(update);
-            psFree(where);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(update);
-
-        numStacks = psDBAffectedRows(config->dbh);
-
-        if (numStacks < 1) {
-            psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
     }
 
     // Delete product
-    {
-        psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql");
-        if (!delete) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&delete, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-
-        if (!p_psDBRunQuery(config->dbh, delete)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(delete);
-            psFree(where);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
+    psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql");
+    if (!delete) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&delete, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, delete)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(delete);
-
-        if (psDBAffectedRows(config->dbh) != numStacks) {
-            psError(PS_ERR_UNKNOWN, true, "Updated and deleted different number of entries!");
-            psFree(where);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-    }
-
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
+        psFree(where);
+        return false;
+    }
+    psFree(delete);
+
+    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
+    psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows);
 
     psFree(where);
@@ -1048,4 +1065,26 @@
 }
 
+#ifdef notdef
+static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    // check that state is a valid string value
+    if (!pxIsValidState(state)) {
+        psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state);
+        return false;
+    }
+
+    char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'";
+    if (!p_psDBRunQueryF(config->dbh, query, state, label)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for label %s", label);
+        return false;
+    }
+
+    return true;
+}
+#endif
+
 static bool pendingcleanuprunMode(pxConfig *config)
 {
@@ -1060,5 +1099,5 @@
     psString query = pxDataGet("stacktool_pendingcleanuprun.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1124,5 +1163,5 @@
     psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1185,5 +1224,5 @@
     psString query = pxDataGet("stacktool_donecleanup.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1273,4 +1312,8 @@
   }
 
+  if (!pxExportVersion(config, f)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+    return false;
+  }
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
@@ -1285,5 +1328,5 @@
     psString query = pxDataGet(tables[i].sqlFilename);
     if (!query) {
-      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
       return false;
     }
@@ -1359,7 +1402,13 @@
   psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
 
-  fprintf (stdout, "---- input ----\n");
+#ifdef notdef
+  fprintf (stderr, "---- input ----\n");
   psMetadataPrint (stderr, input, 1);
-
+#endif
+
+  if (!pxCheckImportVersion(config, input)) {
+      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+      return false;
+  }
   psMetadataItem *item = psMetadataLookup (input, "stackRun");
   psAssert (item, "entry not in input?");