Index: /branches/backtrack/ippTools/scripts/chiptest.sh
===================================================================
--- /branches/backtrack/ippTools/scripts/chiptest.sh	(revision 13989)
+++ /branches/backtrack/ippTools/scripts/chiptest.sh	(revision 13990)
@@ -8,9 +8,9 @@
 
 for ID in `seq 0 3`; do
-    chiptool -addprocessedimfile -chip_id 1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
+    chiptool -addprocessedimfile -chip_id 1 -exp_tag t10.1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
 done;
 
 for ID in `seq 0 3`; do
-    chiptool -addprocessedimfile -chip_id 2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
+    chiptool -addprocessedimfile -chip_id 2 -exp_tag t11.2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
 done;
 
Index: /branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql
===================================================================
--- /branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql	(revision 13989)
+++ /branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql	(revision 13990)
@@ -1,26 +1,32 @@
--- select * from chipPendingExp
--- where exp_tag is not in chipProcessedExp
--- where exp_tag is not in chipPendingImfile
--- where the number of entries in chipProccessedImfile matches the .imfiles
--- entry in rawExp
+-- the output of this query must match the format of chipRun row
 SELECT DISTINCT
-    chipPendingExp.*,
-    rawExp.imfiles,
-    chipProcessedImfile.class_id
-FROM chipPendingExp
-JOIN rawExp
-    ON chipPendingExp.exp_tag = rawExp.exp_tag
-LEFT JOIN chipProcessedExp
-    ON chipPendingExp.chip_id = chipProcessedExp.chip_id
-LEFT JOIN chipPendingImfile
-    ON chipPendingExp.chip_id = chipPendingImfile.chip_id
-LEFT JOIN chipProcessedImfile
-    ON chipPendingExp.chip_id = chipProcessedImfile.chip_id
-WHERE
-    chipProcessedExp.chip_id IS NULL
-    AND chipPendingImfile.chip_id IS NULL
-    AND chipProcessedImfile.chip_id IS NOT NULL
-    AND chipProcessedImfile.fault = 0
-GROUP BY
-    chipPendingExp.chip_id
-HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id)
+    chip_id,
+    state,
+    workdir,
+    workdir_state,
+    label,
+    reduction,
+    expgroup,
+    dvodb
+FROM
+    (SELECT DISTINCT
+        chipRun.*,
+        rawExp.imfiles,
+        chipProcessedImfile.exp_tag,
+        chipProcessedImfile.class_id
+    FROM chipRun
+    JOIN chipPendingImfile
+        USING(chip_id)
+    JOIN rawExp
+        ON chipPendingImfile.exp_tag = rawExp.exp_tag
+    LEFT JOIN chipProcessedImfile
+        USING(chip_id)
+    WHERE
+        chipProcessedImfile.chip_id IS NOT NULL
+        AND chipProcessedImfile.exp_tag IS NOT NULL
+        AND chipProcessedImfile.class_id IS NOT NULL
+        AND chipProcessedImfile.fault = 0
+    GROUP BY
+        chipProcessedImfile.class_id,
+        chipProcessedImfile.exp_tag
+    HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id)) as Foo
Index: /branches/backtrack/ippTools/share/chiptool_pendingimfile.sql
===================================================================
--- /branches/backtrack/ippTools/share/chiptool_pendingimfile.sql	(revision 13989)
+++ /branches/backtrack/ippTools/share/chiptool_pendingimfile.sql	(revision 13990)
@@ -1,17 +1,29 @@
 SELECT
-    chipPendingExp.*,
+    chipRun.*,
+    chipPendingImfile.exp_tag,
     chipPendingImfile.class_id,
-    chipPendingImfile.uri,
+    rawImfile.uri,
     rawExp.exp_id,
     rawExp.camera,
     rawExp.telescope,
     rawExp.filelevel
-FROM chipPendingImfile
-JOIN chipPendingExp
+FROM chipRun
+JOIN chipPendingImfile
     USING(chip_id)
+LEFT JOIN chipProcessedImfile
+    ON chipPendingImfile.chip_id = chipProcessedImfile.chip_id
+    AND chipPendingImfile.exp_tag = chipProcessedImfile.exp_tag
+    AND chipPendingImfile.class_id = chipProcessedImfile.class_id
+JOIN rawImfile
+    ON chipPendingImfile.exp_tag = rawImfile.exp_tag
+    AND chipPendingImfile.class_id = rawImfile.class_id
 JOIN rawExp
-    ON chipPendingExp.exp_tag = rawExp.exp_tag
+    ON rawImfile.exp_tag = rawExp.exp_tag
 LEFT JOIN chipMask
-    ON chipPendingExp.label = chipMask.label
+    ON chipRun.label = chipMask.label
 WHERE
-    chipMask.label IS NULL
+    chipRun.state = 'run'
+    AND chipProcessedImfile.chip_id IS NULL
+    AND chipProcessedImfile.exp_tag IS NULL
+    AND chipProcessedImfile.class_id IS NULL
+    AND chipMask.label IS NULL
Index: anches/backtrack/ippTools/share/chiptool_queuerawexp.sql
===================================================================
--- /branches/backtrack/ippTools/share/chiptool_queuerawexp.sql	(revision 13989)
+++ 	(revision )
@@ -1,16 +1,0 @@
--- we can only enqueue one exp_tag at a time or the generated chip_id of all
--- put the last one inserted will be lost
-INSERT INTO chipPendingExp
-    SElECT
-        0,              -- chip_id
-        exp_tag,        -- exp_tag
-        0xdeadbeef,     -- guide_id
-        '%s',           -- workdir
-        '%s',           -- label
-        '%s',           -- recipe
-        '%s',           -- expgroup
-        '%s'            -- dvodb 
-   FROM rawExp
-   WHERE
-        rawExp.fault = 0
-        AND rawExp.exp_tag = '%s'
Index: /branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql
===================================================================
--- /branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql	(revision 13989)
+++ /branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql	(revision 13990)
@@ -1,11 +1,13 @@
 INSERT INTO chipPendingImfile
     SELECT
-        chipPendingExp.chip_id,
-        rawImfile.class_id,
-        rawImfile.uri
-    FROM chipPendingExp
-    JOIN rawImfile
+        LAST_INSERT_ID(),
+        rawImfile.exp_tag,
+        rawImfile.class_id
+    FROM rawImfile
+    JOIN newExp
         USING(exp_tag)
     WHERE
         rawImfile.fault = 0
-        AND chipPendingExp.chip_id = LAST_INSERT_ID()
+        AND newExp.state = 'stop'
+        AND rawImfile.exp_tag = '%s'
+
Index: /branches/backtrack/ippTools/src/chipqueue.c
===================================================================
--- /branches/backtrack/ippTools/src/chipqueue.c	(revision 13989)
+++ /branches/backtrack/ippTools/src/chipqueue.c	(revision 13990)
@@ -23,5 +23,4 @@
 
 #include "pxtools.h"
-#include "pxdata.h"
 #include "chiptool.h"
 
@@ -30,5 +29,5 @@
                      psString workdir,
                      psString label,
-                     psString recipe,
+                     psString reduction,
                      psString expgroup,
                      psString dvodb)
@@ -36,34 +35,44 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // load the SQL to enqueue our exp_tags from disk once
-    static psString queuerawexp_query = NULL;
-    if (!queuerawexp_query) {
-        queuerawexp_query = pxDataGet("chiptool_queuerawexp.sql");
-        psMemSetPersistent(queuerawexp_query, true);
-        if (!queuerawexp_query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
+    // start a transaction
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    
+    // create a chipRun
+    if (!chipRunInsert(config->dbh,
+            0x0,
+            "run",      // state                
+            workdir,
+            "dirty",    // workdir_state
+            label,
+            reduction,
+            expgroup,
+            dvodb)
+    ) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
         }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
     }
 
-    static psString queuerawimfile_query = NULL;
-    if (!queuerawimfile_query) {
-        queuerawimfile_query = pxDataGet("chiptool_queuerawimfile.sql");
-        psMemSetPersistent(queuerawimfile_query, true);
-        if (!queuerawexp_query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
+#if 0
+    // get the chip_id
+    long chip_id = psDBLastInsertID(config->dbh);
+#endif
+
+    // populate the chipPendingImfile table from rawImfile based on exp_tag,
+    // where the coresponding newExp.state entry is set to stop
+    psString query = pxDataGetCached("chiptool_queuerawimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
     }
 
-    // queue the exp
-    if (!p_psDBRunQuery(config->dbh, queuerawexp_query,
-                workdir  ? workdir  : "NULL",
-                label    ? label    : "NULL",
-                recipe   ? recipe   : "NULL",
-                expgroup ? expgroup : "NULL",
-                dvodb    ? dvodb    : "NULL",
-                exp_tag
-    )) {
+    // queue the imfiles for the exp_tag
+    if (!p_psDBRunQuery(config->dbh, query, exp_tag)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -77,14 +86,7 @@
     }
 
-    // queue the imfiles for the exp we just queued
-    if (!p_psDBRunQuery(config->dbh, queuerawimfile_query)) {
+    // point of no return
+    if (!psDBCommit(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    // just to be safe, we should have changed at least one row
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false,
-                "no rows affected - should have changed at least one row");
         return false;
     }
Index: /branches/backtrack/ippTools/src/chiptool.c
===================================================================
--- /branches/backtrack/ippTools/src/chiptool.c	(revision 13989)
+++ /branches/backtrack/ippTools/src/chiptool.c	(revision 13990)
@@ -42,6 +42,4 @@
 static bool unblockMode(pxConfig *config);
 
-static chipProcessedImfileRow *chipPendingToProcessedImfile(pxConfig *config, chipPendingImfileRow *imfile);
-static chipProcessedExpRow *chipPendingToProcessedExp(pxConfig *config, chipPendingExpRow *pendingExp);
 static bool chipProcessedCompleteExp(pxConfig *config);
 
@@ -346,491 +344,36 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // select * from chipProcessedImfiles
-    // where
-    // exp_tag & class_id are not in chipPendingImfile
-
-    // add the completed imfile to
-    // the chipProcessedeImfile tables
-    // remove corresponding entries from the
-    // chipPendingImfile table
-    // check to see if any chipPendingExps have no
-    // associated chipPendingImfiles
-    // if so move the chipPendingExp(s) to chipProcessedExp
-
-    psString query = pxDataGet("chiptool_find_unprocessed_imfile.pl");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    {
-        psMetadata *where = psMetadataAlloc();
-        bool status = false;
-        psString chip_id = psMetadataLookupStr(&status, config->args, "-chip_id");        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -chip_id");
-            psFree(query);
-            return false;
-        }
-        if (chip_id) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "chip_id", 0, "==", chip_id)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-
-        psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
-            psFree(query);
-            return false;
-        }
-        if (class_id) {
-            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to add item class_id");
-                psFree(where);
-                psFree(query);
-                return false;
-            }
-        }
-
-        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "chipPendingImfile");
-        psFree(where);
-        if (whereClause) {
-            psStringAppend(&query, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        // XXX check psError here
-        psError(PS_ERR_UNKNOWN, false, "no chipPendingImfile rows found");
-        psFree(output);
-        return false;
-    }
-
-    // start a transaction so we don't end up with an incremented iteration
-    // count but no detInputExps
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(output);
-        return false;
-    }
-
-    // insert into chipProcessedImfile
-    // remove chipPendingImfile entry
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *row = output->data[i];
-        // convert metadata into a chipPendingImfile object
-        chipPendingImfileRow *object = chipPendingImfileObjectFromMetadata(row);
-        // convert chipPendingImfile object into a chipProcessedImfile object
-        chipProcessedImfileRow *imfile = chipPendingToProcessedImfile(config, object);
-        if (!imfile) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to convert chipPendingImfile to chipProcessedImfile");
-            psFree(object);
-            psFree(output);
-            return false;
-        }
-        // insert chipProccessedImfile object into the database
-        if (!chipProcessedImfileInsertObject(config->dbh, imfile)) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(imfile);
-            psFree(object);
-            psFree(output);
-            return false;
-        }
-        psFree(imfile);
-        // delete the chipPendingImfile object from the database
-        if (!chipPendingImfileDeleteObject(config->dbh, object)) {
-            // there must be atleast 1 Imfile to get this far
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(object);
-            psFree(output);
-            return false;
-        }
-        psFree(object);
-    }
-
-    psFree(output);
-
-    // XXX I've decided to make the transaction cover the Exp migration as
-    // well.  Otherwise, if the last imfile in an exp is moved and the exp
-    // migration fails then the data base is left in a satuation where the exp
-    // migration can't happen.
-
-    if (!chipProcessedCompleteExp(config)) {
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    // point of no return for chipPendingImfile -> chipProcessedImfile
-    // point of no return for chipPendingExp -> chipProcessedExp
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool processedimfileMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
     bool status = false;
-    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit");
-        return false;
-    }
-
-    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted");
-        return false;
-    }
-
-    // XXX does this need to be constrained so that it won't return any results
-    // if a match chipPendingExp hasn't been registered?
-    psString query = pxDataGet("chiptool_processedimfile.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (config->where) {
-        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "chipProcessedImfile");
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-
-    if (faulted) {
-        // list only faulted rows
-        psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0");
-    } else {
-        // don't list faulted rows
-        psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
-    }
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("chiptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    bool simple = false;
-    {
-        bool status = false;
-        simple = psMetadataLookupBool(&status, config->args, "-simple");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
-            return false;
-        }
-    }
-
-    if (!convertIdToStr(output)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
-        psFree(output);
-        return false;
-    }
-
-    // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-static bool updateprocessedimfileMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    bool status = false;
-    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
-        return false;
-    }
-    if (code == INT16_MAX) {
-        psError(PS_ERR_UNKNOWN, true, "-code is required");
-        return false;
-    }
-
-    if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", config->where, code)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool blockMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    bool status = false;
-    psString label = psMetadataLookupStr(&status, config->args, "-label");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
-        return false;
-    }
-    if (!label) {
-        psError(PS_ERR_UNKNOWN, true, "-label is required");
-        return false;
-    }
-
-    if (!chipMaskInsert(config->dbh, label)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool maskedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psString query = psStringCopy("SELECT * FROM chipMask");
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("chiptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    bool simple = false;
-    {
-        bool status = false;
-        simple = psMetadataLookupBool(&status, config->args, "-simple");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
-            return false;
-        }
-    }
-
-    if (!convertIdToStr(output)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
-        psFree(output);
-        return false;
-    }
-
-    // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-static bool unblockMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    bool status = false;
-    psString label = psMetadataLookupStr(&status, config->args, "-label");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
-        return false;
-    }
-    if (!label) {
-        psError(PS_ERR_UNKNOWN, true, "-label is required");
-        return false;
-    }
-
-    char *query = "DELETE FROM chipMask WHERE label = '%s'";
-
-    if (!p_psDBRunQuery(config->dbh, query, label)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool chipProcessedCompleteExp(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // look for completed chipPendingExp
-    // migrate them to chipProccessedExp & camPendingExp
-    psString query = pxDataGet("chiptool_completely_processed_exp.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("chiptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    // insert into chipProcessedExp
-    // insert into camPendingExp
-    // remove chipPendingExp entry
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *row = output->data[i];
-
-        // convert metadata into a chipPendingExp object
-        chipPendingExpRow *pendingExp = chipPendingExpObjectFromMetadata(row);
-        if (!pendingExp) {
-            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipPendingExp");
-            psFree(output);
-            return false;
-        }
-
-        // convert chipPendingExp object into a chipProcesseExp object
-        chipProcessedExpRow *processedExp = chipPendingToProcessedExp(config, pendingExp);
-        if (!processedExp) {
-            psError(PS_ERR_UNKNOWN, false, "failed to convert chipPendingExp to chipProcessedExp");
-            psFree(pendingExp);
-            psFree(output);
-            return false;
-        }
-
-        // delete the chipPendingExp object from the database
-        if (!chipPendingExpDeleteObject(config->dbh, pendingExp)) {
-            // there must be atleast 1 Imfile to get this far
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(processedExp);
-            psFree(pendingExp);
-            psFree(output);
-            return false;
-        }
-        psFree(pendingExp);
-
-        // insert chipProccessedExp object into the database
-        if (!chipProcessedExpInsertObject(config->dbh, processedExp)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(processedExp);
-            psFree(output);
-            return false;
-        }
-
-        // camQueueChipID() can only be run after the chipProcessedExp entry
-        // has been inserted.
-        // queue the chip_id in the camPendingExp table
-        if (!camQueueChipID(config,
-                    processedExp->chip_id,
-                    processedExp->workdir,
-                    processedExp->label,
-                    processedExp->reduction,
-                    processedExp->expgroup,
-                    processedExp->dvodb
-        )) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp");
-            psFree(processedExp);
-            psFree(output);
-            return false;
-        }
-
-        psFree(processedExp);
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-static chipProcessedImfileRow *chipPendingToProcessedImfile(pxConfig *config, chipPendingImfileRow *imfile)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-    PS_ASSERT_PTR_NON_NULL(imfile, NULL);
-
-    bool status = false;
+    // chip_id, ext_tag, class_id are required
+    psString chip_id = psMetadataLookupStr(&status, config->args, "-chip_id");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -chip_id");
+        return false;
+    }
+    if (!chip_id) {
+        psError(PS_ERR_UNKNOWN, true, "-chip_id is required");
+        return false;
+    }
+
+    psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
+        return false;
+    }
+    if (!exp_tag) {
+        psError(PS_ERR_UNKNOWN, true, "-exp_tag is required");
+        return false;
+    }
+
+    psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
+        return false;
+    }
+    if (!class_id) {
+        psError(PS_ERR_UNKNOWN, true, "-class_id is required");
+        return false;
+    }
+
     psString uri = psMetadataLookupStr(&status, config->args, "-uri");
     if (!status) {
@@ -961,7 +504,13 @@
     }
 
-    return chipProcessedImfileRowAlloc(
-        imfile->chip_id,
-        imfile->class_id,
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    chipProcessedImfileRow *imfile = chipProcessedImfileRowAlloc(
+        (psU64)atoll(chip_id),
+        exp_tag,
+        class_id,
         uri,
         bg,
@@ -985,21 +534,328 @@
         path_base,
         code
-        );
-}
-
-
-static chipProcessedExpRow *chipPendingToProcessedExp(pxConfig *config, chipPendingExpRow *pendingExp)
+    );
+    if (!imfile) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "failed to alloc chipProcessedImfileRow object");
+        return false;
+    }
+    if (!chipProcessedImfileInsertObject(config->dbh, imfile)) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(imfile);
+        return false;
+    }
+
+    // XXX I've decided to make the transaction cover the Exp migration as
+    // well.  Otherwise, if the last imfile in an exp is moved and the exp
+    // migration fails then the data base is left in a satuation where the exp
+    // migration can't happen.
+
+    if (!chipProcessedCompleteExp(config)) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool processedimfileMode(pxConfig *config)
 {
-    PS_ASSERT_PTR_NON_NULL(pendingExp, NULL);
-
-    return chipProcessedExpRowAlloc(
-        pendingExp->chip_id,
-        pendingExp->exp_tag,
-        pendingExp->guide_id,
-        pendingExp->workdir,
-        pendingExp->label,
-        pendingExp->reduction,
-        pendingExp->expgroup,
-        pendingExp->dvodb
-    );
-}
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    bool status = false;
+    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit");
+        return false;
+    }
+
+    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted");
+        return false;
+    }
+
+    // XXX does this need to be constrained so that it won't return any results
+    // if a match chipPendingExp hasn't been registered?
+    psString query = pxDataGet("chiptool_processedimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "chipProcessedImfile");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0");
+    } else {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    bool simple = false;
+    {
+        bool status = false;
+        simple = psMetadataLookupBool(&status, config->args, "-simple");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
+            return false;
+        }
+    }
+
+    if (!convertIdToStr(output)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
+        psFree(output);
+        return false;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool updateprocessedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
+        return false;
+    }
+    if (code == INT16_MAX) {
+        psError(PS_ERR_UNKNOWN, true, "-code is required");
+        return false;
+    }
+
+    if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", config->where, code)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool blockMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psString label = psMetadataLookupStr(&status, config->args, "-label");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
+        return false;
+    }
+    if (!label) {
+        psError(PS_ERR_UNKNOWN, true, "-label is required");
+        return false;
+    }
+
+    if (!chipMaskInsert(config->dbh, label)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool maskedMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psString query = psStringCopy("SELECT * FROM chipMask");
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    bool simple = false;
+    {
+        bool status = false;
+        simple = psMetadataLookupBool(&status, config->args, "-simple");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
+            return false;
+        }
+    }
+
+    if (!convertIdToStr(output)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings");
+        psFree(output);
+        return false;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool unblockMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    bool status = false;
+    psString label = psMetadataLookupStr(&status, config->args, "-label");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
+        return false;
+    }
+    if (!label) {
+        psError(PS_ERR_UNKNOWN, true, "-label is required");
+        return false;
+    }
+
+    char *query = "DELETE FROM chipMask WHERE label = '%s'";
+
+    if (!p_psDBRunQuery(config->dbh, query, label)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool chipProcessedCompleteExp(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // look for completed chipPendingExp
+    // migrate them to chipProccessedExp & camPendingExp
+    psString query = pxDataGet("chiptool_completely_processed_exp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // insert into camPendingExp
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        chipRunRow *chipRun = chipRunObjectFromMetadata(row);
+
+        // camQueueChipID() can only be run after the chipProcessedExp entry
+        // has been inserted.
+        // queue the chip_id in the camPendingExp table
+        if (!camQueueChipID(config,
+                    chipRun->chip_id,
+                    chipRun->workdir,
+                    chipRun->label,
+                    chipRun->reduction,
+                    chipRun->expgroup,
+                    chipRun->dvodb
+        )) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp");
+            psFree(chipRun);
+            psFree(output);
+            return false;
+        }
+        psFree(chipRun);
+    }
+
+    psFree(output);
+
+    return true;
+}
Index: /branches/backtrack/ippTools/src/chiptoolConfig.c
===================================================================
--- /branches/backtrack/ippTools/src/chiptoolConfig.c	(revision 13989)
+++ /branches/backtrack/ippTools/src/chiptoolConfig.c	(revision 13990)
@@ -127,4 +127,6 @@
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id",  0,
             "define chip ID (required)", NULL);
+    psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag",  0,
+            "define exposure ID", NULL);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,
             "define class ID (requied)", NULL);
Index: /branches/backtrack/ippTools/src/pxdata.h
===================================================================
--- /branches/backtrack/ippTools/src/pxdata.h	(revision 13989)
+++ /branches/backtrack/ippTools/src/pxdata.h	(revision 13990)
@@ -24,4 +24,5 @@
 
 psString pxDataGet(const char *filename);
+psString pxDataGetCached(const char *filename);
 
 #endif // PXDATA_H
Index: /branches/backtrack/ippTools/src/pxtables.c
===================================================================
--- /branches/backtrack/ippTools/src/pxtables.c	(revision 13989)
+++ /branches/backtrack/ippTools/src/pxtables.c	(revision 13990)
@@ -60,7 +60,6 @@
     CREATE_TABLE(rawImfileCreateTable);
     CREATE_TABLE(guidePendingExpCreateTable);
-    CREATE_TABLE(chipPendingExpCreateTable);
+    CREATE_TABLE(chipRunCreateTable);
     CREATE_TABLE(chipPendingImfileCreateTable);
-    CREATE_TABLE(chipProcessedExpCreateTable);
     CREATE_TABLE(chipProcessedImfileCreateTable);
     CREATE_TABLE(chipMaskCreateTable);
@@ -142,7 +141,6 @@
     DROP_TABLE(rawImfileDropTable);
     DROP_TABLE(guidePendingExpDropTable);
-    DROP_TABLE(chipPendingExpDropTable);
+    DROP_TABLE(chipRunDropTable);
     DROP_TABLE(chipPendingImfileDropTable);
-    DROP_TABLE(chipProcessedExpDropTable);
     DROP_TABLE(chipProcessedImfileDropTable);
     DROP_TABLE(chipMaskDropTable);
Index: /branches/backtrack/ippTools/src/regtool.c
===================================================================
--- /branches/backtrack/ippTools/src/regtool.c	(revision 13989)
+++ /branches/backtrack/ippTools/src/regtool.c	(revision 13990)
@@ -620,14 +620,4 @@
         }
 
-        // insert an entry into the chipPendingExp table
-        if (!chipQueueExpTag(config, exp_tag, NULL, NULL, NULL, NULL, NULL)) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp");
-            psFree(output);
-            return false;
-        }
 
         // set the state for the newExp to stop
@@ -638,4 +628,16 @@
             }
             psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_tag: %s", exp_tag);
+            psFree(output);
+            return false;
+        }
+
+        // insert an entry into the chipPendingExp table
+        // this can only be run as the newExp's state has been set to stop
+        if (!chipQueueExpTag(config, exp_tag, NULL, NULL, NULL, NULL, NULL)) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp");
             psFree(output);
             return false;