Index: trunk/ippTools/src/magictool.c
===================================================================
--- trunk/ippTools/src/magictool.c	(revision 30374)
+++ trunk/ippTools/src/magictool.c	(revision 30388)
@@ -835,11 +835,4 @@
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("magictool_toprocess_inputs.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
     psString whereString = NULL;
     if (psListLength(where->list)) {
@@ -850,59 +843,11 @@
     psFree(where);
 
-    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        // cut limit in half
-        // hack to prevent pending leaf nodes from blocking branch nodes
-        psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(whereString);
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("magictool", PS_LOG_INFO, "no rows found");
-    }
-
-    if (limit) {
-        if (psArrayLength(output) >= limit) {
-            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
-            // negative simple so the default is true
-            if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to print array");
-                psFree(output);
-                return false;
-            }
-            psFree(output);
-            return true;
-        }
-    }
-
-    // look for tree nodes that need to be processed
+    // First look for branch nodes that need to be processed.
+    // These get priority over skycells because they are from runs
+    // that are already in progress and there are fewer of them.
+    // When we looked for skycells first we got starved.
 
     // first find incomplete magicRuns
-    query = pxDataGet("magictool_toprocess_runs.sql");
+    psString query = pxDataGet("magictool_toprocess_runs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -910,11 +855,10 @@
     }
 
-    // we limit the query even though it is cheap (only magic_id is selected)
-
-    // XXX: if the first 1000 unfinished magicRuns have no ready nodes
-    // that haven't faulted, later runs won't get returned even though
-    // they have work to do. When we used a limit of 100 we actually ran
-    // into this problem. Since we're using labels a limit of 1000 will
-    // probably be ok.
+    // Find outstanding magicRuns in new state.
+    // We limit the query, but this is problematic. In practice do
+    // we need to?
+    // XXX: If the first 1000 magicRuns have no branch nodes ready
+    // but higher runs do we they won't be noticed.
+    // Perhaps have this limit be an argument.
     {
         psString limitString = psDBGenerateLimitSQL( 1000 );
@@ -946,11 +890,111 @@
         return false;
     }
-    if (!psArrayLength(magicRuns)) {
-        psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicRuns);
-        return true;
-    }
-
-    query = pxDataGet("magictool_toprocess_tree.sql");
+    psArray *output = NULL;
+    if (psArrayLength(magicRuns)) {
+        output = psArrayAllocEmpty(100);
+
+        query = pxDataGet("magictool_toprocess_tree.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
+            if (limit && (psArrayLength(output) >= limit)) {
+                break;
+            }
+            bool status;
+            psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
+            if (!status) {
+                psAbort("failed to lookup value for magic_id column");
+            }
+
+            whereString = NULL;
+            psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
+            if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(whereString);
+                psFree(query);
+                return false;
+            }
+            psFree(whereString);
+            psArray *magicTree = p_psDBFetchResult(config->dbh);
+            if (!magicTree) {
+                psErrorCode err = psErrorCodeLast();
+                switch (err) {
+                    case PS_ERR_DB_CLIENT:
+                        psError(PXTOOLS_ERR_SYS, false, "database error");
+                    case PS_ERR_DB_SERVER:
+                        psError(PXTOOLS_ERR_PROG, false, "database error");
+                    default:
+                        psError(PXTOOLS_ERR_PROG, false, "unknown error");
+                }
+
+                return false;
+            }
+            psS64 length = psArrayLength(magicTree);
+            if (!length) {
+                psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
+                psFree(magicTree);
+                continue;
+            }
+
+            psHash *forest = psHashAlloc(length);
+
+            // convert the array of metadata into a pxTree structure
+            for (long i = 0; i < length; i++) {
+                bool status;
+                psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
+                if (!status) {
+                    psAbort("failed to lookup value for node column");
+                }
+
+                psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
+                if (!status) {
+                    psAbort("failed to lookup value for dep column");
+                }
+
+                pxTreeBuilder(forest, node, dep, magicTree->data[i]);
+
+            }
+
+            // find the root of the tree
+            pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
+            psFree(forest);
+            //    pxTreePrint(stdout, root);
+
+            // crawl through the tree and looking for nodes with children that are all
+            // "done"
+            pxTreeCrawl(root, findReadyNodes, output);
+            psFree(root);
+            psFree(magicTree);
+        }
+
+        int len = psArrayLength(output);
+        if (len) {
+            if (limit) {
+                if (limit < psArrayLength(output)) {
+                    // truncate the array
+                    long arrayLength = psArrayLength(output);
+                    for (long i = arrayLength - 1; i >= limit; i--) {
+                        psArrayRemoveIndex(output, i);
+                    }
+                    // negative simple so the default is true
+                    if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
+                        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+                        psFree(output);
+                        return false;
+                    }
+                    psFree(output);
+                    return true;
+                } else {
+                    limit -= len;
+                }
+            }
+        }
+    }
+
+    // look for "inputs" (skycells) that need to processed
+    query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -958,84 +1002,44 @@
     }
 
-    for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
-        if (limit && (psArrayLength(output) >= limit)) {
-            break;
-        }
-        bool status;
-        psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
-        if (!status) {
-            psAbort("failed to lookup value for magic_id column");
-        }
-
-        whereString = NULL;
-        psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
-        if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(whereString);
-            psFree(query);
-            return false;
-        }
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(whereString);
-        psArray *magicTree = p_psDBFetchResult(config->dbh);
-        if (!magicTree) {
-            psErrorCode err = psErrorCodeLast();
-            switch (err) {
-                case PS_ERR_DB_CLIENT:
-                    psError(PXTOOLS_ERR_SYS, false, "database error");
-                case PS_ERR_DB_SERVER:
-                    psError(PXTOOLS_ERR_PROG, false, "database error");
-                default:
-                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
-            }
-
-            return false;
-        }
-        psS64 length = psArrayLength(magicTree);
-        if (!length) {
-            psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
-            psFree(magicTree);
-            continue;
-        }
-
-        psHash *forest = psHashAlloc(length);
-
-        // convert the array of metadata into a pxTree structure
-        for (long i = 0; i < length; i++) {
-            bool status;
-            psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
-            if (!status) {
-                psAbort("failed to lookup value for node column");
-            }
-
-            psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
-            if (!status) {
-                psAbort("failed to lookup value for dep column");
-            }
-
-            pxTreeBuilder(forest, node, dep, magicTree->data[i]);
-
-        }
-
-        // find the root of the tree
-        pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
-        psFree(forest);
-        //    pxTreePrint(stdout, root);
-
-        // crawl through the tree and looking for nodes with children that are all
-        // "done"
-        pxTreeCrawl(root, findReadyNodes, output);
-        psFree(root);
-        psFree(magicTree);
-    }
-
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *skycellOutput = p_psDBFetchResult(config->dbh);
+    if (!skycellOutput) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    // merge the arrays so that we can print them all at once
+    if (psArrayLength(skycellOutput)) {
+        int len = psArrayLength(skycellOutput);
+        for (int i=0; i < len; i++) {
+            psArrayAdd(output, 0, skycellOutput->data[i]);
+        }
+    }
+    psFree(skycellOutput);
     if (psArrayLength(output)) {
-        if (limit && (limit < psArrayLength(output))) {
-            // truncate the array
-            long arrayLength = psArrayLength(output);
-            for (long i = arrayLength - 1; i >= limit; i--) {
-                psArrayRemoveIndex(output, i);
-            }
-        }
-        // negative simple so the default is true
         if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
@@ -1043,8 +1047,8 @@
             return false;
         }
-    }
-
+    } else {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+    }
     psFree(output);
-
     return true;
 }