Index: /branches/end_stage/ippTools/src/regtool.c
===================================================================
--- /branches/end_stage/ippTools/src/regtool.c	(revision 16049)
+++ /branches/end_stage/ippTools/src/regtool.c	(revision 16050)
@@ -43,13 +43,4 @@
 static bool revertprocessedexpMode(pxConfig *config);
 static bool updateprocessedexpMode(pxConfig *config);
-
-
-// static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp);
-//static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *newExp);
-//static chipInputImfileRow *rawImfileTochipInputImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile);
-
-static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp);
-//static psU32 mapCodeStrToInt(const char *codeStr);
-
 
 # define MODECASE(caseName, func) \
@@ -822,4 +813,321 @@
 
     bool status = false;
+    // required
+    psString exp_name = psMetadataLookupStr(&status, config->args, "-exp_name");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_name");
+        return false;
+    }
+    if (!exp_name) {
+        psError(PS_ERR_UNKNOWN, true, "-exp_name is required");
+        return false;
+    }
+
+    psString camera = psMetadataLookupStr(&status, config->args, "-inst");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst");
+        return false;
+    }
+    if (!camera) {
+        psError(PS_ERR_UNKNOWN, true, "-inst is required");
+        return false;
+    }
+
+    psString telescope = psMetadataLookupStr(&status, config->args, "-telescope");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope");
+        return false;
+    }
+    if (!telescope) {
+        psError(PS_ERR_UNKNOWN, true, "-telescope is required");
+        return false;
+    }
+
+    psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
+        return false;
+    }
+    if (!exp_tag) {
+        psError(PS_ERR_UNKNOWN, true, "-exp_tag is required");
+        return false;
+    }
+
+    // optional
+    psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type");
+        return false;
+    }
+
+    psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel");
+        return false;
+    }
+    if (!filelevel) {
+        psError(PS_ERR_UNKNOWN, true, "-filelevel is required");
+        return false;
+    }
+
+    psString workdir = psMetadataLookupStr(&status, config->args, "-workdir");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir");
+        return false;
+    }
+
+    psString reduction = psMetadataLookupStr(&status, config->args, "-reduction");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction");
+        return false;
+    }
+
+    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
+        return false;
+    }
+
+    psString comment = psMetadataLookupStr(&status, config->args, "-comment");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -comment");
+        return false;
+    }
+
+    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
+        return false;
+    }
+
+    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
+        return false;
+    }
+
+    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
+        return false;
+    }
+
+    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
+        return false;
+    }
+
+    psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac");
+        return false;
+    }
+
+    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
+        return false;
+    }
+
+    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
+        return false;
+    }
+
+    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
+        return false;
+    }
+
+    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
+        return false;
+    }
+
+    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
+        return false;
+    }
+
+    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
+        return false;
+    }
+
+    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");
+        return false;
+    }
+
+    psF32 m1_x = psMetadataLookupF32(&status, config->args, "-m1_x");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_x");
+        return false;
+    }
+    psF32 m1_y = psMetadataLookupF32(&status, config->args, "-m1_y");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_y");
+        return false;
+    }
+    psF32 m1_z = psMetadataLookupF32(&status, config->args, "-m1_z");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_z");
+        return false;
+    }
+    psF32 m1_tip = psMetadataLookupF32(&status, config->args, "-m1_tip");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_tip");
+        return false;
+    }
+    psF32 m1_tilt = psMetadataLookupF32(&status, config->args, "-m1_tilt");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_tilt");
+        return false;
+    }
+
+    psF32 m2_x = psMetadataLookupF32(&status, config->args, "-m2_x");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_x");
+        return false;
+    }
+    psF32 m2_y = psMetadataLookupF32(&status, config->args, "-m2_y");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_y");
+        return false;
+    }
+    psF32 m2_z = psMetadataLookupF32(&status, config->args, "-m2_z");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_z");
+        return false;
+    }
+    psF32 m2_tip = psMetadataLookupF32(&status, config->args, "-m2_tip");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_tip");
+        return false;
+    }
+    psF32 m2_tilt = psMetadataLookupF32(&status, config->args, "-m2_tilt");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_tilt");
+        return false;
+    }
+
+    psF32 env_temp = psMetadataLookupF32(&status, config->args, "-env_temperature");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_temperature");
+        return false;
+    }
+    psF32 env_humid = psMetadataLookupF32(&status, config->args, "-env_humidity");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_humidity");
+        return false;
+    }
+    psF32 env_wind = psMetadataLookupF32(&status, config->args, "-env_wind_speed");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_wind_speed");
+        return false;
+    }
+    psF32 env_dir = psMetadataLookupF32(&status, config->args, "-env_wind_dir");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_wind_dir");
+        return false;
+    }
+
+    psF32 teltemp_m1 = psMetadataLookupF32(&status, config->args, "-teltemp_m1");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m1");
+        return false;
+    }
+    psF32 teltemp_m1cell = psMetadataLookupF32(&status, config->args, "-teltemp_m1cell");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m1cell");
+        return false;
+    }
+    psF32 teltemp_m2 = psMetadataLookupF32(&status, config->args, "-teltemp_m2");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m2");
+        return false;
+    }
+    psF32 teltemp_spider = psMetadataLookupF32(&status, config->args, "-teltemp_spider");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_spider");
+        return false;
+    }
+    psF32 teltemp_truss = psMetadataLookupF32(&status, config->args, "-teltemp_truss");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_truss");
+        return false;
+    }
+    psF32 teltemp_extra = psMetadataLookupF32(&status, config->args, "-teltemp_extra");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_extra");
+        return false;
+    }
+
+    psF32 pon_time = psMetadataLookupF32(&status, config->args, "-pon_time");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pon_time");
+        return false;
+    }
+
+    psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1");
+        return false;
+    }
+    psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2");
+        return false;
+    }
+    psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3");
+        return false;
+    }
+    psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4");
+        return false;
+    }
+    psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5");
+        return false;
+    }
+
+    psString object = psMetadataLookupStr(&status, config->args, "-object");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object");
+        return false;
+    }
+
+    psTime *dateobs = NULL;
+    {
+        psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs");
+            return false;
+        }
+        // pass through NULL as this is an optional field
+        if (dateobsStr) {
+            dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC);
+        } else {
+            dateobs = NULL;
+        }
+    }
+
+    // default
+    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
     psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id");
     if (!status) {
@@ -865,13 +1173,10 @@
         // build a query to search by exp_id
         psMetadata *where = psMetadataAlloc();
-        if (exp_id) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-
+        if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
+            psFree(where);
+            psFree(query);
+            return false;
+        }
 
         psString whereClause = psDBGenerateWhereSQL(where, NULL);
@@ -909,4 +1214,14 @@
         return false;
     }
+    // sanity check that we only got one row
+    if (psArrayLength(output) != 1) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output));
+        psFree(output);
+        return NULL;
+    }
 
     if (!psDBTransaction(config->dbh)) {
@@ -917,634 +1232,12 @@
 
     // insert the exp into rawExp
-    for (long i = 0; psArrayLength(output) > i; i++) {
-        psMetadata *row = output->data[i];
-        // convert metadata into a newExp object
-        newExpRow *newExp = newExpObjectFromMetadata(row);
-        // convert newExp object into a rawExp object
-        rawExpRow *rawExp = newToRawExp(config, newExp);
-        if (!rawExp) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawExp");
-            psFree(newExp);
-            psFree(output);
-            return false;
-        }
-        psFree(newExp);
-
-        // insert the rawExp object into the database
-        if (!rawExpInsertObject(config->dbh, rawExp)) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(rawExp);
-            psFree(output);
-            return false;
-        }
-
-        // if this is a detrend image don't put it in the chip queue (and we're
-        // done)
-        if (detrend) {
-            // set the state for the newExp to stop
-            if (!pxnewExpSetState(config, exp_id, "stop")) {
-                // rollback
-                if (!psDBRollback(config->dbh)) {
-                    psError(PS_ERR_UNKNOWN, false, "database error");
-                }
-                psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %s", exp_id);
-                psFree(rawExp);
-                psFree(output);
-                return false;
-            }
-            psFree(rawExp);
-            continue;
-        }
-
-
-        // set the state for the newExp to stop
-        if (!pxnewExpSetState(config, exp_id, "stop")) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %s", exp_id);
-            psFree(rawExp);
-            psFree(output);
-            return false;
-        }
-
-        // insert an entry into the chipPendingExp table
-        // this can only be run as the newExp's state has been set to stop
-        if (!pxchipQueueByExpTag(config,
-                    rawExp->exp_id,
-                    rawExp->workdir,
-                    NULL, // label
-                    rawExp->reduction,
-                    NULL, // expgroup
-                    dvodb,
-                    tess_id,
-                    end_stage
-        )) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp");
-            psFree(rawExp);
-            psFree(output);
-            return false;
-        }
-        psFree(rawExp);
-    }
+    psMetadata *row = output->data[0];
+    // convert metadata into a newExp object
+    newExpRow *newExp = newExpObjectFromMetadata(row);
     psFree(output);
 
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool processedexpMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    bool status = false;
-    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
-    if (!status) {
-        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
-        return false;
-    }
-
-    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
-    if (!status) {
-        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
-        return false;
-    }
-
-    psString query = pxDataGet("regtool_processedexp.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (config->where) {
-        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-
-    if (faulted) {
-        // list only faulted rows
-        psStringAppend(&query, " %s", "AND rawExp.fault != 0");
-    } else {
-        // don't list faulted rows
-        psStringAppend(&query, " %s", "AND rawExp.fault = 0");
-    }
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("regtool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    bool simple = false;
-    {
-        bool status = false;
-        simple = psMetadataLookupBool(&status, config->args, "-simple");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
-            return false;
-        }
-    }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool revertprocessedexpMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psString query = pxDataGet("regtool_revertprocessedexp.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (config->where) {
-        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool updateprocessedexpMode(pxConfig *config)
-{
-    bool status = false;
-    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
-        return false;
-    }
-    if (code == INT16_MAX) {
-        psError(PS_ERR_UNKNOWN, true, "-code is required");
-        return false;
-    }
-
-    if (!pxSetFaultCode(config->dbh, "rawExp", config->where, code)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-        return false;
-    }
-
-    return true;
-}
-
-
-# if 0
-static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp)
-{
-    return guidePendingExpRowAlloc(
-        newExp->exp_id,
-        newExp->camera,
-        newExp->telescope,
-        newExp->exp_type,
-        newExp->imfiles,
-        "my filter",
-        0.1, // airmass
-        0.2, // ra
-        0.3, // dec
-        0.4, // exp time
-        0.5, // background
-        "my recipe",
-        0xff // XXX calc version number
-    );
-}
-#endif
-
-
-#if 0
-static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *exp)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-    PS_ASSERT_PTR_NON_NULL(exp, NULL);
-
-    // optional
-    bool status = false;
-    psString label = psMetadataLookupStr(&status, config->args, "-label");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
-        return false;
-    }
-
-    chipPendingExpRow *chipExp = chipPendingExpRowAlloc(
-            0,              // chip ID, assigned by db
-            exp->exp_id,
-            0xdeadbeef,     // guide version
-            label,
-            "my recipe",    // recipe
-            NULL,           // expgroup
-            NULL            // dvodb
-    );
-
-    return chipExp;
-}
-
-
-static chipInputImfileRow *rawImfileTochipInputImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile)
-{
-    return chipInputImfileRowAlloc(
-            chip_id,
-            rawImfile->class_id,
-            rawImfile->uri
-    );
-}
-#endif
-
-
-static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-    PS_ASSERT_PTR_NON_NULL(exp, NULL);
-
-    bool status = false;
-    // required
-    psString exp_name = psMetadataLookupStr(&status, config->args, "-exp_name");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_name");
-        return false;
-    }
-    if (!exp_name) {
-        psError(PS_ERR_UNKNOWN, true, "-exp_name is required");
-        return false;
-    }
-
-    psString camera = psMetadataLookupStr(&status, config->args, "-inst");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst");
-        return false;
-    }
-    if (!camera) {
-        psError(PS_ERR_UNKNOWN, true, "-inst is required");
-        return false;
-    }
-
-    psString telescope = psMetadataLookupStr(&status, config->args, "-telescope");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope");
-        return false;
-    }
-    if (!telescope) {
-        psError(PS_ERR_UNKNOWN, true, "-telescope is required");
-        return false;
-    }
-
-    psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
-        return false;
-    }
-    if (!exp_tag) {
-        psError(PS_ERR_UNKNOWN, true, "-exp_tag is required");
-        return false;
-    }
-
-    // optional
-    psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type");
-        return false;
-    }
-
-    psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel");
-        return false;
-    }
-    if (!filelevel) {
-        psError(PS_ERR_UNKNOWN, true, "-filelevel is required");
-        return false;
-    }
-
-    psString workdir = psMetadataLookupStr(&status, config->args, "-workdir");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir");
-        return false;
-    }
-
-    psString reduction = psMetadataLookupStr(&status, config->args, "-reduction");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction");
-        return false;
-    }
-
-    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
-        return false;
-    }
-
-    psString comment = psMetadataLookupStr(&status, config->args, "-comment");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -comment");
-        return false;
-    }
-
-    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
-        return false;
-    }
-
-    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
-        return false;
-    }
-
-    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
-        return false;
-    }
-
-    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
-        return false;
-    }
-
-    psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac");
-        return false;
-    }
-
-    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
-        return false;
-    }
-
-    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
-        return false;
-    }
-
-    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
-        return false;
-    }
-
-    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
-        return false;
-    }
-
-    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
-        return false;
-    }
-
-    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
-        return false;
-    }
-
-    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");
-        return false;
-    }
-
-    psF32 m1_x = psMetadataLookupF32(&status, config->args, "-m1_x");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_x");
-        return false;
-    }
-    psF32 m1_y = psMetadataLookupF32(&status, config->args, "-m1_y");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_y");
-        return false;
-    }
-    psF32 m1_z = psMetadataLookupF32(&status, config->args, "-m1_z");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_z");
-        return false;
-    }
-    psF32 m1_tip = psMetadataLookupF32(&status, config->args, "-m1_tip");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_tip");
-        return false;
-    }
-    psF32 m1_tilt = psMetadataLookupF32(&status, config->args, "-m1_tilt");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m1_tilt");
-        return false;
-    }
-
-    psF32 m2_x = psMetadataLookupF32(&status, config->args, "-m2_x");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_x");
-        return false;
-    }
-    psF32 m2_y = psMetadataLookupF32(&status, config->args, "-m2_y");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_y");
-        return false;
-    }
-    psF32 m2_z = psMetadataLookupF32(&status, config->args, "-m2_z");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_z");
-        return false;
-    }
-    psF32 m2_tip = psMetadataLookupF32(&status, config->args, "-m2_tip");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_tip");
-        return false;
-    }
-    psF32 m2_tilt = psMetadataLookupF32(&status, config->args, "-m2_tilt");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -m2_tilt");
-        return false;
-    }
-
-    psF32 env_temp = psMetadataLookupF32(&status, config->args, "-env_temperature");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_temperature");
-        return false;
-    }
-    psF32 env_humid = psMetadataLookupF32(&status, config->args, "-env_humidity");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_humidity");
-        return false;
-    }
-    psF32 env_wind = psMetadataLookupF32(&status, config->args, "-env_wind_speed");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_wind_speed");
-        return false;
-    }
-    psF32 env_dir = psMetadataLookupF32(&status, config->args, "-env_wind_dir");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -env_wind_dir");
-        return false;
-    }
-
-    psF32 teltemp_m1 = psMetadataLookupF32(&status, config->args, "-teltemp_m1");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m1");
-        return false;
-    }
-    psF32 teltemp_m1cell = psMetadataLookupF32(&status, config->args, "-teltemp_m1cell");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m1cell");
-        return false;
-    }
-    psF32 teltemp_m2 = psMetadataLookupF32(&status, config->args, "-teltemp_m2");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_m2");
-        return false;
-    }
-    psF32 teltemp_spider = psMetadataLookupF32(&status, config->args, "-teltemp_spider");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_spider");
-        return false;
-    }
-    psF32 teltemp_truss = psMetadataLookupF32(&status, config->args, "-teltemp_truss");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_truss");
-        return false;
-    }
-    psF32 teltemp_extra = psMetadataLookupF32(&status, config->args, "-teltemp_extra");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -teltemp_extra");
-        return false;
-    }
-
-    psF32 pon_time = psMetadataLookupF32(&status, config->args, "-pon_time");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pon_time");
-        return false;
-    }
-
-    psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1");
-        return false;
-    }
-    psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2");
-        return false;
-    }
-    psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3");
-        return false;
-    }
-    psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4");
-        return false;
-    }
-    psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5");
-        return false;
-    }
-
-    psString object = psMetadataLookupStr(&status, config->args, "-object");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object");
-        return false;
-    }
-
-    psTime *dateobs = NULL;
-    {
-        psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs");
-            return false;
-        }
-        // pass through NULL as this is an optional field
-        if (dateobsStr) {
-            dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC);
-        } else {
-            dateobs = NULL;
-        }
-    }
-
-    // default
-    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
-        return false;
-    }
-
-    rawExpRow *raw = rawExpRowAlloc(
-        exp->exp_id,
+    if (!rawExpInsert(
+        config->dbh,
+        (psS64)atoll(exp_id),
         exp_name,
         camera,
@@ -1554,8 +1247,8 @@
         exp_type,
         filelevel,
-        workdir ? workdir : exp->workdir,
-        reduction ? reduction : exp->reduction,
+        workdir ? workdir : newExp->workdir,
+        reduction ? reduction : newExp->reduction,
         filter,
-	comment,
+        comment,
         airmass,
         ra,
@@ -1570,25 +1263,25 @@
         ccd_temp,
         posang,
-	m1_x,
-	m1_y,
-	m1_z,
-	m1_tip,
-	m1_tilt,
-	m2_x,
-	m2_y,
-	m2_z,
-	m2_tip,
-	m2_tilt,
-	env_temp,
-	env_humid,
-	env_wind,
-	env_dir,
-	teltemp_m1,
-	teltemp_m1cell,
-	teltemp_m2,
-	teltemp_spider,
-	teltemp_truss,
-	teltemp_extra,
-	pon_time,
+        m1_x,
+        m1_y,
+        m1_z,
+        m1_tip,
+        m1_tilt,
+        m2_x,
+        m2_y,
+        m2_z,
+        m2_tip,
+        m2_tilt,
+        env_temp,
+        env_humid,
+        env_wind,
+        env_dir,
+        teltemp_m1,
+        teltemp_m1cell,
+        teltemp_m2,
+        teltemp_spider,
+        teltemp_truss,
+        teltemp_extra,
+        pon_time,
         user_1,
         user_2,
@@ -1599,10 +1292,219 @@
         0.0,
         code
-    );
-
-    if (dateobs) {
-        psFree(dateobs);
-    }
-
-    return raw;
+    )) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(newExp);
+        return false;
+    }
+
+    // if this is a detrend image don't put it in the chip queue (and we're
+    // done)
+    if (detrend) {
+        psFree(newExp);
+
+        // set the state for the newExp to stop
+        if (!pxnewExpSetState(config, exp_id, "stop")) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %s", exp_id);
+            return false;
+        }
+
+        return true;
+    }
+
+
+    // set the state for the newExp to stop
+    if (!pxnewExpSetState(config, exp_id, "stop")) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %s", exp_id);
+        psFree(newExp);
+        return false;
+    }
+
+    // insert an entry into the chipPendingExp table
+    // this can only be run as the newExp's state has been set to stop
+    if (!pxchipQueueByExpTag(config,
+                newExp->exp_id,
+                newExp->workdir,
+                NULL, // label
+                newExp->reduction,
+                NULL, // expgroup
+                dvodb,
+                tess_id,
+                end_stage
+    )) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp");
+        psFree(newExp);
+        return false;
+    }
+    psFree(newExp);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
 }
+
+
+static bool processedexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
+        return false;
+    }
+
+    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
+    if (!status) {
+        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
+        return false;
+    }
+
+    psString query = pxDataGet("regtool_processedexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", "AND rawExp.fault != 0");
+    } else {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", "AND rawExp.fault = 0");
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("regtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    bool simple = false;
+    {
+        bool status = false;
+        simple = psMetadataLookupBool(&status, config->args, "-simple");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
+            return false;
+        }
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool revertprocessedexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psString query = pxDataGet("regtool_revertprocessedexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool updateprocessedexpMode(pxConfig *config)
+{
+    bool status = false;
+    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
+    if (code == INT16_MAX) {
+        psError(PS_ERR_UNKNOWN, true, "-code is required");
+        return false;
+    }
+
+    if (!pxSetFaultCode(config->dbh, "rawExp", config->where, code)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        return false;
+    }
+
+    return true;
+}