Index: branches/simmosaic_branches/ippTools/src/magictool.c
===================================================================
--- branches/simmosaic_branches/ippTools/src/magictool.c	(revision 24860)
+++ branches/simmosaic_branches/ippTools/src/magictool.c	(revision 27839)
@@ -47,4 +47,5 @@
 static bool revertmaskMode(pxConfig *config);
 static bool maskMode(pxConfig *config);
+static bool censorrunMode(pxConfig *config);
 
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
@@ -85,4 +86,5 @@
         MODECASE(MAGICTOOL_MODE_REVERTMASK,          revertmaskMode);
         MODECASE(MAGICTOOL_MODE_MASK,                maskMode);
+        MODECASE(MAGICTOOL_MODE_CENSORRUN,           censorrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -112,7 +114,9 @@
     // Required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);
 
     // Optional
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-note", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
@@ -131,5 +135,5 @@
         psString query = pxDataGet("magictool_definebyquery_select.sql");
         if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
             return false;
         }
@@ -147,6 +151,8 @@
         // what if no skycells for the diff run completed?
 
-        if (!rerun) {
-            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        psString rerunWhereStr = NULL;
+        if (rerun) {
+            psStringAppend(&rerunWhereStr, "\n WHERE magicRun.label = '%s'", label);
         }
 
@@ -172,9 +178,13 @@
             queryWhereStr = psStringCopy("");
         }
-
-        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
+        if (!rerunWhereStr) {
+            rerunWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, rerunWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(diffWhereStr);
             psFree(queryWhereStr);
+            psFree(rerunWhereStr);
             psFree(query);
             return false;
@@ -182,4 +192,5 @@
         psFree(diffWhereStr);
         psFree(queryWhereStr);
+        psFree(rerunWhereStr);
         psFree(query);
     }
@@ -234,6 +245,17 @@
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label,
-                                            dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0,
+                                            exp_id,
+                                            diff_id,
+                                            inverse,
+                                            "new",      // state
+                                            workdir,
+                                            "dirty",    // workdir_state
+                                            label,
+                                            data_group ? data_group : label,
+                                            dvodb,
+                                            registered,
+                                            0,          // fault
+                                            note);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -329,7 +351,9 @@
             "dirty",    // workdir_state
             label,
+            NULL,       // data_group
             dvodb,
             registered,
-            0
+            0,          // fault
+            NULL
     );
 
@@ -410,5 +434,5 @@
     psString query = pxDataGet("magictool_inputskyfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -483,5 +507,5 @@
     psString query = pxDataGet("magictool_totree.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -556,5 +580,5 @@
 
     if (fault > 0) {
-        char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64;
+        char *query = "UPDATE magicRun SET fault = %d WHERE magic_id = %" PRId64;
         if (!p_psDBRunQueryF(config->dbh, query, fault, magic_id)) {
             psError(PS_ERR_UNKNOWN, false,
@@ -580,4 +604,5 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 
     psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
@@ -594,4 +619,7 @@
         return false;
     }
+    psS32 numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic runs", numUpdated);
+
     return true;
 }
@@ -617,5 +645,5 @@
     psString query = pxDataGet("magictool_inputs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -787,5 +815,5 @@
     psString query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -801,5 +829,7 @@
     // treat limit == 0 as "no limit"
     if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
+        // cut limit in half
+        // hack to prevent pending leaf nodes from blocking branch nodes
+        psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
         psStringAppend(&query, " %s", limitString);
         psFree(limitString);
@@ -847,12 +877,24 @@
 
     // look for tree nodes that need to be processed
-    // XXX: This gets all nodes from all magicRuns that are in 'new'state
-    // That doens't seem particularly efficient
-    query = pxDataGet("magictool_toprocess_tree.sql");
+
+    // first find incomplete magicRuns
+    query = pxDataGet("magictool_toprocess_runs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // we limit the query even though it is cheap (only magic_id is selected)
+
+    // XXX: if the first 1000 unfinished magicRuns have no ready nodes
+    // that haven't faulted, later runs won't get returned even though
+    // they have work to do. When we used a limit of 100 we actually ran
+    // into this problem. Since we're using labels a limit of 1000 will
+    // probably be ok.
+    {
+        psString limitString = psDBGenerateLimitSQL( 1000 );
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
 
     if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString :  "")) {
@@ -865,6 +907,6 @@
     psFree(query);
 
-    psArray *magicTree = p_psDBFetchResult(config->dbh);
-    if (!magicTree) {
+    psArray *magicRuns = p_psDBFetchResult(config->dbh);
+    if (!magicRuns) {
         psErrorCode err = psErrorCodeLast();
         switch (err) {
@@ -879,42 +921,60 @@
         return false;
     }
-    if (!psArrayLength(magicTree)) {
+    if (!psArrayLength(magicRuns)) {
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicTree);
+        psFree(magicRuns);
         return true;
     }
 
-    // entries are ordered by magic_id
-    long index = 0;
-    while (index <  psArrayLength(magicTree)) {
-        bool status;
+    query = pxDataGet("magictool_toprocess_tree.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
         if (limit && (psArrayLength(output) >= limit)) {
             break;
         }
-        psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id");
+        bool status;
+        psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
         if (!status) {
             psAbort("failed to lookup value for magic_id column");
         }
 
-        // find the end of this block
-        long first = index;
-        long last = index;
-        for (long i = index + 1; i < psArrayLength(magicTree); i++) {
-            psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id");
-            if (!status) {
-                psAbort("failed to lookup value for magic_id column");
+        whereString = NULL;
+        psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
+        if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(whereString);
+            psFree(query);
+            return false;
+        }
+        psFree(whereString);
+        psArray *magicTree = p_psDBFetchResult(config->dbh);
+        if (!magicTree) {
+            psErrorCode err = psErrorCodeLast();
+            switch (err) {
+                case PS_ERR_DB_CLIENT:
+                    psError(PXTOOLS_ERR_SYS, false, "database error");
+                case PS_ERR_DB_SERVER:
+                    psError(PXTOOLS_ERR_PROG, false, "database error");
+                default:
+                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
             }
-            if (magic_id != current_magic_id) {
-                break;
-            }
-            last = i;
-        }
-
-        index = last + 1;
-
-        psHash *forest = psHashAlloc(last - first + 1);
+
+            return false;
+        }
+        psS64 length = psArrayLength(magicTree);
+        if (!length) {
+            psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
+            psFree(magicTree);
+            continue;
+        }
+
+        psHash *forest = psHashAlloc(length);
 
         // convert the array of metadata into a pxTree structure
-        for (long i = first; i <= last; i++) {
+        for (long i = 0; i < length; i++) {
             bool status;
             psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
@@ -941,7 +1001,6 @@
         pxTreeCrawl(root, findReadyNodes, output);
         psFree(root);
-
-    }
-    psFree(magicTree);
+        psFree(magicTree);
+    }
 
     if (psArrayLength(output)) {
@@ -1001,10 +1060,10 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     psString query = pxDataGet("magictool_revertnode.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1027,5 +1086,5 @@
 
     psS32 numUpdated = psDBAffectedRows(config->dbh);
-    psLogMsg("magictool", PS_LOG_INFO, "Updated %d magic nodes", numUpdated);
+    psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic nodes", numUpdated);
 
     return true;
@@ -1043,5 +1102,5 @@
     psString query = pxDataGet("magictool_tomask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1131,5 +1190,5 @@
     psString query = pxDataGet("magictool_addmask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         if (!psDBRollback(config->dbh)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1238,5 +1297,5 @@
     psString query = pxDataGet("magictool_mask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1367,2 +1426,90 @@
     return true;
 }
+
+static bool censorrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+
+    if (!magic_id) {
+        if (!exp_id && !label) {
+            psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required");
+            return false;
+        }
+    }
+
+    // at least one of these required
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+    if (!psListLength(where->list)) {
+        psError(PS_ERR_UNKNOWN, true, "either -exp_id or -magic_id is required");
+        psFree(where);
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE magicRun SET state = 'censored'");
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, "magicRun");
+    psFree(where);
+    psStringAppend(&query, " WHERE %s", whereClause);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereClause);
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psS32 numUpdated = psDBAffectedRows(config->dbh);
+    if (numUpdated == 0) {
+        psError(PS_ERR_UNKNOWN, false, "failed to censor magicRun");
+        psFree(whereClause);
+        return false;
+    }
+
+    // Now queue any destreaked files to be re-verted
+
+    // note: on failure pxmagicRestoreStage issues the rollback
+    if (!pxmagicRestoreStage(config, "raw", whereClause, "goto_censored")) {
+        psFree(whereClause);
+        return false;
+    }
+    if (!pxmagicRestoreStage(config, "chip", whereClause, "goto_censored")) {
+        psFree(whereClause);
+        return false;
+    }
+    if (!pxmagicRestoreStage(config, "camera", whereClause, "goto_censored")) {
+        psFree(whereClause);
+        return false;
+    }
+    if (!pxmagicRestoreStage(config, "warp", whereClause, "goto_censored")) {
+        psFree(whereClause);
+        return false;
+    }
+    if (!pxmagicRestoreStage(config, "diff", whereClause, "goto_censored")) {
+        psFree(whereClause);
+        return false;
+    }
+
+    psFree(whereClause);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}