Index: trunk/ippTools/src/regtool.c
===================================================================
--- trunk/ippTools/src/regtool.c	(revision 11037)
+++ trunk/ippTools/src/regtool.c	(revision 11047)
@@ -27,16 +27,20 @@
 #include "p0tool.h"
 
+static bool pendingimfileMode(pxConfig *config);
+static bool addprocessedimfileMode(pxConfig *config);
+static bool processedimfileMode(pxConfig *config);
+static bool updateprocessedimfileMode(pxConfig *config);
+
 static bool pendingexpMode(pxConfig *config);
-static bool pendingimfileMode(pxConfig *config);
-static bool updateexpMode(pxConfig *config);
-static bool updateimfileMode(pxConfig *config);
-static bool faultexpMode(pxConfig *config);
-static bool faultimfileMode(pxConfig *config);
-static bool rawimfileMode(pxConfig *config);
+static bool addprocessedexpMode(pxConfig *config);
+static bool processedexpMode(pxConfig *config);
+static bool updateprocessedexpMode(pxConfig *config);
+
+
 // static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp);
 static p2PendingExpRow *newToP2PendingExp(pxConfig *config, newExpRow *newExp);
 static p2PendingImfileRow *rawImfileToP2PendingImfile(pxConfig *config, rawImfileRow *rawImfile);
-static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp);
-static rawDetrendExpRow *newToRawDetrendExp(pxConfig *config, newExpRow *exp);
+
+static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp);
 static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *exp);
 //static psU32 mapCodeStrToInt(const char *codeStr);
@@ -59,11 +63,12 @@
 
     switch (config->mode) {
-        MODECASE(P0TOOL_MODE_PENDINGEXP,        pendingexpMode);
-        MODECASE(P0TOOL_MODE_PENDINGIMFILE,     pendingimfileMode);
-        MODECASE(P0TOOL_MODE_UPDATEEXP,         updateexpMode);
-        MODECASE(P0TOOL_MODE_FAULTEXP,          faultexpMode);
-        MODECASE(P0TOOL_MODE_UPDATEIMFILE,      updateimfileMode);
-        MODECASE(P0TOOL_MODE_FAULTIMFILE,       faultimfileMode);
-        MODECASE(P0TOOL_MODE_RAWIMFILE,         rawimfileMode);
+        MODECASE(P0TOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
+        MODECASE(P0TOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
+        MODECASE(P0TOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
+        MODECASE(P0TOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode);
+        MODECASE(P0TOOL_MODE_PENDINGEXP,            pendingexpMode);
+        MODECASE(P0TOOL_MODE_ADDPROCESSEDEXP,       addprocessedexpMode);
+        MODECASE(P0TOOL_MODE_PROCESSEDEXP,          processedexpMode);
+        MODECASE(P0TOOL_MODE_UPDATEPROCESSEDEXP,    updateprocessedexpMode);
         default:
             psAbort(argv[0], "invalid option (this should not happen)");
@@ -88,5 +93,6 @@
 }
 
-static bool pendingexpMode(pxConfig *config)
+
+static bool pendingimfileMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -99,4 +105,238 @@
     }
 
+    // select newImfiles that:
+    // exp_tag is in newExp
+    // don't have their exp_tag in rawExp 
+    // XXX having the same exp_tag in newExp and raw*Exp is probably an error
+    // that should be checked for
+
+    psString query = psStringCopy(
+        "SELECT\n"
+        "   newImfile.*\n"
+        " FROM newImfile\n"
+        " LEFT JOIN newExp\n"
+        "   USING(exp_tag)\n"
+        " LEFT JOIN rawExp\n"
+        "   USING(exp_tag)\n"
+        " WHERE\n"
+        "   newExp.exp_tag is NOT NULL\n"
+        "   AND rawExp.exp_tag IS NULL\n"
+    );
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
+        psError(PXTOOLS_ERR_PROG, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
+        psError(PXTOOLS_ERR_PROG, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psFree(output);
+        return true;
+    }
+
+    bool simple = false;
+    simple = psMetadataLookupBool(&status, config->args, "-simple");
+    if (!status) {
+	psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple");
+	return false;
+    }
+
+    // negate simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "p0PendingImfile", !simple)) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool addprocessedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // XXX search by the whole frame some imfiles without a newExp don't get
+    // processed -- this may not be the correct thing to do
+    psString query = psStringCopy(
+        "SELECT\n"
+        "   *\n"
+        " FROM\n"
+        "   (SELECT newImfile.* FROM newImfile\n"
+        "       LEFT JOIN newExp USING(exp_tag)\n"
+        "       LEFT JOIN rawExp USING(exp_tag)\n"
+        "       WHERE newExp.exp_tag IS NOT NULL\n"
+        "       AND rawExp.exp_tag IS NULL) as Foo\n"
+        ); // WHERE class is generated from exp_tag, class, & class_id
+
+    {
+        // build a query to search by exp_tag, class, class_id
+        psMetadata *where = psMetadataAlloc();
+        bool status = false;
+        psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
+            psFree(query);
+            return false;
+        }
+        if (exp_tag) {
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag");
+                psFree(where);
+                psFree(query);
+                return false;
+            }
+        }
+        psString class = psMetadataLookupStr(&status, config->args, "-class");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class");
+            psFree(query);
+            return false;
+        }
+        if (class) {
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class", 0, "==", class)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to add item class");
+                psFree(where);
+                psFree(query);
+                return false;
+            }
+        }
+        psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
+            psFree(query);
+            return false;
+        }
+        if (class_id) {
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to add item class_id");
+                psFree(where);
+                psFree(query);
+                return false;
+            }
+        }
+
+        // there's not
+        psString whereClaus = psDBGenerateWhereSQL(where, NULL);
+        psFree(where);
+        if (whereClaus) {
+            psStringAppend(&query, " %s", whereClaus);
+            psFree(whereClaus);
+        }
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        // XXX check psError here
+        psError(PS_ERR_UNKNOWN, false, "no pending newImfile rows found");
+        psFree(output);
+        return false;
+    }
+
+    // insert 'newImfile's into rawImfile
+    if (psArrayLength(output) > 0) {
+        // start a transaction so we don't end up half of the imfiles we were
+        // trying to update uninserted
+        if (!psDBTransaction(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            return false;
+        }
+
+        for (long i = 0; i < psArrayLength(output); i++) {
+            // convert newImfile metadata -> newImfile object
+            newImfileRow *object = newImfileObjectFromMetadata(output->data[i]);
+            // convert newImfile object -> rawImfile object
+            rawImfileRow *imfile = newToRawImfile(config, object); 
+            if (!imfile) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "failed to create a new rawImfile row");
+                psFree(object);
+                psFree(output);
+                return false;
+            }
+            // insert the rawImfile object into the database
+            if (!rawImfileInsertObject(config->dbh, imfile)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "failed to insert row into the database");
+                psFree(imfile);
+                psFree(object);
+                psFree(output);
+                return false;
+            }
+            psFree(imfile);
+            // remove the neImfile object from the database
+            if (!newImfileDeleteObject(config->dbh, object)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "failed to delete row from the database");
+                psFree(object);
+                psFree(output);
+                return false;
+            }
+            psFree(object);
+        }
+
+        // point of no return for rawImfile
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool processedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
+        return false;
+    }
+
     bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
     if (!status) {
@@ -105,7 +345,109 @@
     }
 
+    // find all rawImfiles matching the default query
+    psString query = psStringCopy( 
+        "SELECT\n"
+        "   *\n"
+        " FROM rawImfile\n"
+        " WHERE rawImfile.exp_tag is NOT NULL\n" //bogus conditional so there is a where clause to append to
+    );
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", "AND rawImfile.fault != 0");
+    } else {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psError(PS_ERR_UNKNOWN, false, "no pending rawImfile rows found");
+        psFree(output);
+        return true;
+    }
+
+    bool simple = false;
+    {
+        bool status = false;
+        simple = psMetadataLookupBool(&status, config->args, "-simple");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
+            return false;
+        }
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool updateprocessedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
+
+    if (!pxSetFaultCode(config->dbh, "rawImfile", config->where, code)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool pendingexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
+        return false;
+    }
+
     // return only exps that:
-    // are not in rawScienceExp 
-    // are not in rawDetrendExp 
+    // are not in rawExp 
     // have ALL of their imfiles in rawImfile (by count)
     // and have no associated imfiles left in newImfile
@@ -117,24 +459,13 @@
         " LEFT JOIN newImfile"
         "   USING(exp_tag)"
-        " LEFT JOIN rawScienceExp"
-        "   USING(exp_tag)"
-        " LEFT JOIN rawDetrendExp"
+        " LEFT JOIN rawExp"
         "   USING(exp_tag)"
         " WHERE"
         "   newImfile.exp_tag IS NULL"
-        "   AND rawScienceExp.exp_tag IS NULL"
-        "   AND rawDetrendExp.exp_tag IS NULL"
+        "   AND rawExp.exp_tag IS NULL"
         "   AND newExp.imfiles ="
         "   (SELECT COUNT(exp_tag) FROM rawImfile"
         "       WHERE rawImfile.exp_tag = newExp.exp_tag)"
     );
-
-    if (faulted) {
-        // list only faulted rows
-        psStringAppend(&query, " %s", "AND newExp.fault != 0");
-    } else {
-        // don't list faulted rows
-        psStringAppend(&query, " %s", "AND newExp.fault = 0");
-    }
 
     // treat limit == 0 as "no limit"
@@ -183,135 +514,11 @@
 }
 
-static bool pendingimfileMode(pxConfig *config)
+
+static bool addprocessedexpMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    bool status = false;
-    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
-    if (!status) {
-        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
-        return false;
-    }
-
-    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
-    if (!status) {
-        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
-        return false;
-    }
-
-    // select newImfiles that:
-    // exp_tag is in newExp
-    // don't have their exp_tag in rawScienceExp 
-    // don't have their exp_tag in rawDetrendExp
-    // XXX having the same exp_tag in newExp and raw*Exp is probably an error
-    // that should be checked for
-
-    psString query = psStringCopy(
-        "SELECT newImfile.* FROM newImfile"
-        " LEFT JOIN newExp USING(exp_tag)"
-        " LEFT JOIN rawScienceExp USING(exp_tag)"
-        " LEFT JOIN rawDetrendExp USING (exp_tag)"
-        " WHERE newExp.exp_tag is NOT NULL"
-        " AND rawScienceExp.exp_tag IS NULL"
-        " AND rawDetrendExp.exp_tag IS NULL"
-    );
-
-    if (faulted) {
-        // list only faulted rows
-        psStringAppend(&query, " %s", "AND newImfile.fault != 0");
-    } else {
-        // don't list faulted rows
-        psStringAppend(&query, " %s", "AND newImfile.fault = 0");
-    }
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
-        psError(PXTOOLS_ERR_PROG, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
-        psError(PXTOOLS_ERR_PROG, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psFree(output);
-        return true;
-    }
-
-    bool simple = false;
-    simple = psMetadataLookupBool(&status, config->args, "-simple");
-    if (!status) {
-	psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple");
-	return false;
-    }
-
-    // negate simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "newImfile", !simple)) {
-        psError(PXTOOLS_ERR_PROG, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool faultexpMode(pxConfig *config)
-{
-    bool status = false;
-    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
-        return false;
-    }
-
-    if (!pxSetFaultCode(config->dbh, "newExp", config->where, code)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-        return false;
-    }
-
-    return true;
-}
-
-static bool faultimfileMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    bool status = false;
-    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
-        return false;
-    }
-
-    if (!pxSetFaultCode(config->dbh, "newImfile", config->where, code)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool updateexpMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
     // make sure that the exp_tag(s) are ready to be updated based on:
-    // exp_tag is not in rawScienceExp
-    // exp_tag is not in rawDetrendExp
+    // exp_tag is not in rawExp
     // exp_tag is not in newImfile
     // that the correct count of imfiles is in rawImfile
@@ -340,12 +547,10 @@
         " LEFT JOIN newImfile"
         "   USING(exp_tag)"
-        " LEFT JOIN rawScienceExp"
-        "   USING(exp_tag)"
-        " LEFT JOIN rawDetrendExp"
+        " LEFT JOIN rawExp"
         "   USING(exp_tag)"
         " WHERE"
         "   newExp.exp_tag IS NOT NULL"
         "   AND newImfile.exp_tag IS NULL"
-        "   AND rawScienceExp.exp_tag IS NULL"
+        "   AND rawExp.exp_tag IS NULL"
         "   AND newExp.imfiles ="
         "   (SELECT COUNT(exp_tag) FROM rawImfile"
@@ -369,7 +574,6 @@
     }
 
-
-    // start a transaction so we don't end up with an exp in
-    // rawScience/DetrendExp and in newExp
+    // start a transaction so we don't end up with an exp in both rawExp &
+    // newExp
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -378,67 +582,11 @@
     }
 
-    // if it's a detrend exp
-    if (detrend) {        
-        for (long i = 0; psArrayLength(output) > i; i++) {
-            psMetadata *row = output->data[i];
-            // convert metadata into a newExp object
-            newExpRow *newExp = newExpObjectFromMetadata(row);
-            // convert newExp object into a rawDetrendExp object
-            rawDetrendExpRow *rawExp = newToRawDetrendExp(config, newExp);
-            if (!rawExp) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawDetrendExp");
-                psFree(newExp);
-                psFree(output);
-                return false;
-            }
-            // insert the rawDetrendExp object into the database
-            if (!rawDetrendExpInsertObject(config->dbh, rawExp)) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "database error");
-                psFree(rawExp);
-                psFree(newExp);
-                psFree(output);
-                return false;
-            }
-            psFree(rawExp);
-            // delete the newExp object from the database
-            if (!newExpDeleteObject(config->dbh, newExp)) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "database error");
-                psFree(newExp);
-                psFree(output);
-                return false;
-            }
-            psFree(newExp);
-        }
-
-        psFree(output);
-
-        if (!psDBCommit(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            return false;
-        }
-
-        return true;
-    } 
-
-    // else
-    // it's a science exp
+    // insert the exp into rawExp
     for (long i = 0; psArrayLength(output) > i; i++) {
         psMetadata *row = output->data[i];
         // convert metadata into a newExp object
         newExpRow *newExp = newExpObjectFromMetadata(row);
-        // convert newExp object into a rawDetrendExp object
-        rawScienceExpRow *rawExp = newToRawScienceExp(config, newExp);
+        // convert newExp object into a rawExp object
+        rawExpRow *rawExp = newToRawExp(config, newExp);
         if (!rawExp) {
             // rollback
@@ -446,11 +594,12 @@
                 psError(PS_ERR_UNKNOWN, false, "database error");
             }
-            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawScienceExp");
+            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawExp");
             psFree(newExp);
             psFree(output);
             return false;
         }
-        // insert the rawDetrendExp object into the database
-        if (!rawScienceExpInsertObject(config->dbh, rawExp)) {
+
+        // insert the rawExp object into the database
+        if (!rawExpInsertObject(config->dbh, rawExp)) {
             // rollback
             if (!psDBRollback(config->dbh)) {
@@ -464,4 +613,5 @@
         }
         psFree(rawExp);
+
         // delete the newExp object from the database
         if (!newExpDeleteObject(config->dbh, newExp)) {
@@ -475,4 +625,12 @@
             return false;
         }
+
+        // if this is a detrend image don't put it in the p2 queue (and we're
+        // done)
+        if (detrend) {
+            psFree(newExp);
+            continue;
+        }
+
         // insert an entry into the p2PendingExp table
         p2PendingExpRow *p2PendingExp = newToP2PendingExp(config, newExp);
@@ -483,4 +641,5 @@
             return false;
         }
+
         // insert the p2PendingExp object into the database
         if (!p2PendingExpInsertObject(config->dbh, p2PendingExp)) {
@@ -496,4 +655,5 @@
         }
         psFree(p2PendingExp);
+
         // find all of the rawImfiles associated with the p2PendingExp object
         psArray *rawImfiles = NULL;
@@ -570,80 +730,54 @@
 }
 
-static bool updateimfileMode(pxConfig *config)
+
+static bool processedexpMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // XXX search by the whole frame some imfiles without a newExp don't get
-    // processed -- this may not be the correct thing to do
-    psString query = psStringCopy(
-        "SELECT * FROM"
-            " (SELECT newImfile.* FROM newImfile"
-                " LEFT JOIN newExp USING(exp_tag)"
-                " LEFT JOIN rawScienceExp USING(exp_tag)"
-                " LEFT JOIN rawDetrendExp USING (exp_tag)"
-                " WHERE newExp.exp_tag IS NOT NULL"
-                " AND rawScienceExp.exp_tag IS NULL"
-                " AND rawDetrendExp.exp_tag IS NULL) AS foo"
-        ); // WHERE class is generated from exp_tag, class, & class_id
-
-    {
-        // build a query to search by exp_tag, class, class_id
-        psMetadata *where = psMetadataAlloc();
-        bool status = false;
-        psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
-            psFree(query);
-            return false;
-        }
-        if (exp_tag) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-        psString class = psMetadataLookupStr(&status, config->args, "-class");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class");
-            psFree(query);
-            return false;
-        }
-        if (class) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class", 0, "==", class)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item class");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-        psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
-            psFree(query);
-            return false;
-        }
-        if (class_id) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item class_id");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-
-        // there's not
-        psString whereClaus = psDBGenerateWhereSQL(where, NULL);
-        psFree(where);
-        if (whereClaus) {
-            psStringAppend(&query, " %s", whereClaus);
-            psFree(whereClaus);
-        }
+    bool status = false;
+    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
+        return false;
+    }
+
+    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
+        return false;
+    }
+
+    // find all rawImfiles matching the default query
+    psString query = psStringCopy( 
+        "SELECT\n"
+        "   *\n"
+        " FROM rawExp\n"
+        " WHERE\n"
+        "   rawExp.exp_tag IS NOT NULL\n" // bogus where clause
+    );
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", "AND rawExp.fault != 0");
+    } else {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", "AND rawExp.fault = 0");
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
     }
 
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
         return false;
     }
@@ -656,85 +790,9 @@
     }
     if (!psArrayLength(output)) {
-        // XXX check psError here
-        psError(PS_ERR_UNKNOWN, false, "no pending newImfile rows found");
+        psError(PS_ERR_UNKNOWN, false, "no pending rawExp rows found");
         psFree(output);
-        return false;
-    }
-
-    // insert 'newImfile's into rawImfile
-    if (psArrayLength(output) > 0) {
-        // start a transaction so we don't end up half of the imfiles we were
-        // trying to update uninserted
-        if (!psDBTransaction(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(output);
-            return false;
-        }
-
-        for (long i = 0; i < psArrayLength(output); i++) {
-            // convert newImfile metadata -> newImfile object
-            newImfileRow *object = newImfileObjectFromMetadata(output->data[i]);
-            // convert newImfile object -> rawImfile object
-            rawImfileRow *imfile = newToRawImfile(config, object); 
-            if (!imfile) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "failed to create a new rawImfile row");
-                psFree(object);
-                psFree(output);
-                return false;
-            }
-            // insert the rawImfile object into the database
-            if (!rawImfileInsertObject(config->dbh, imfile)) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "failed to insert row into the database");
-                psFree(imfile);
-                psFree(object);
-                psFree(output);
-                return false;
-            }
-            psFree(imfile);
-            // remove the neImfile object from the database
-            if (!newImfileDeleteObject(config->dbh, object)) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "failed to delete row from the database");
-                psFree(object);
-                psFree(output);
-                return false;
-            }
-            psFree(object);
-        }
-
-        // point of no return for rawImfile
-        if (!psDBCommit(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool rawimfileMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // find all rawImfiles matching the default query
-    psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, config->where, 0);
-    if (!rawImfiles) {
-        psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found");
-        return false;
-    }
+        return true;
+    }
+
     bool simple = false;
     {
@@ -747,17 +805,37 @@
     }
 
-    if (psArrayLength(rawImfiles)) {
+    if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!rawImfilePrintObjects(stdout, rawImfiles, !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(rawImfiles);
-            return false;
-        }
-    }
-
-    psFree(rawImfiles);
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
 
     return true;
 }
+
+
+static bool updateprocessedexpMode(pxConfig *config)
+{
+    bool status = false;
+    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
+
+    if (!pxSetFaultCode(config->dbh, "rawExp", config->where, code)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        return false;
+    }
+
+    return true;
+}
+
+
 # if 0
 static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp)
@@ -781,4 +859,5 @@
 #endif
 
+
 static p2PendingExpRow *newToP2PendingExp(pxConfig *config, newExpRow *exp)
 {
@@ -804,4 +883,5 @@
     return p2Exp;
 }
+
 
 static p2PendingImfileRow *rawImfileToP2PendingImfile(pxConfig *config, rawImfileRow *rawImfile)
@@ -817,15 +897,6 @@
 }
 
-static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-    PS_ASSERT_PTR_NON_NULL(exp, NULL);
-
-    // XXX this is dangerous but we should be able to get away with this as
-    // long rawScienceExp & rawDetrendExp are idetnical
-    return (rawScienceExpRow *)newToRawDetrendExp(config, exp);
-}
-
-static rawDetrendExpRow *newToRawDetrendExp(pxConfig *config, newExpRow *exp)
+
+static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -932,5 +1003,12 @@
     }
 
-    rawDetrendExpRow *raw = rawDetrendExpRowAlloc(
+    // default
+    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
+
+    rawExpRow *raw = rawExpRowAlloc(
         exp->exp_tag,
         exp->camera,
@@ -951,5 +1029,6 @@
         posang,
         object,
-        dateobs
+        dateobs,
+        code
     );
 
@@ -960,4 +1039,5 @@
     return raw;
 }
+
 
 static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *imfile)
@@ -1065,4 +1145,10 @@
             dateobs = NULL;
         }
+    }
+
+    psS8 code = psMetadataLookupS8(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
     }
 
@@ -1086,5 +1172,6 @@
         posang,
         object,
-        dateobs
+        dateobs,
+        code
     );