Changeset 30586 for branches/czw_branch/20101203/ippTools/src/magictool.c

Timestamp:

Feb 11, 2011, 4:29:48 PM (15 years ago)

Author:

watersc1

Message:

Trying to merge back into trunk.

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippTools/src/magictool.c (modified) (10 diffs, 1 prop)

branches/czw_branch/20101203/ippTools/src/magictool.c

@@ -49 +49 @@
 static bool censorrunMode(pxConfig *config);
 static bool exposureMode(pxConfig *config);
+static bool setgotocleanedMode(pxConfig *config);
+static bool tocleanupMode(pxConfig *config);
+static bool setworkdirstateMode(pxConfig *config);
 
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
@@ -89 +92 @@
         MODECASE(MAGICTOOL_MODE_CENSORRUN,           censorrunMode);
         MODECASE(MAGICTOOL_MODE_EXPOSURE,            exposureMode);
+        MODECASE(MAGICTOOL_MODE_SETGOTOCLEANED,      setgotocleanedMode);
+        MODECASE(MAGICTOOL_MODE_TOCLEANUP,           tocleanupMode);
+        MODECASE(MAGICTOOL_MODE_SETWORKDIRSTATE,     setworkdirstateMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -621 +627 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0");
@@ -829 +835 @@
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("magictool_toprocess_inputs.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
     psString whereString = NULL;
     if (psListLength(where->list)) {
@@ -844 +843 @@
     psFree(where);
 
-    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        // cut limit in half
-        // hack to prevent pending leaf nodes from blocking branch nodes
-        psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(whereString);
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("magictool", PS_LOG_INFO, "no rows found");
-    }
-
-    if (limit) {
-        if (psArrayLength(output) >= limit) {
-            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
-            // negative simple so the default is true
-            if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to print array");
-                psFree(output);
-                return false;
-            }
-            psFree(output);
-            return true;
-        }
-    }
-
-    // look for tree nodes that need to be processed
+    // First look for branch nodes that need to be processed.
+    // These get priority over skycells because they are from runs
+    // that are already in progress and there are fewer of them.
+    // When we looked for skycells first we got starved.
 
     // first find incomplete magicRuns
-    query = pxDataGet("magictool_toprocess_runs.sql");
+    psString query = pxDataGet("magictool_toprocess_runs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -904 +855 @@
     }
 
-    // we limit the query even though it is cheap (only magic_id is selected)
-
-    // XXX: if the first 1000 unfinished magicRuns have no ready nodes
-    // that haven't faulted, later runs won't get returned even though
-    // they have work to do. When we used a limit of 100 we actually ran
-    // into this problem. Since we're using labels a limit of 1000 will
-    // probably be ok.
+    // Find outstanding magicRuns in new state.
+    // We limit the query, but this is problematic. In practice do
+    // we need to?
+    // XXX: If the first 1000 magicRuns have no branch nodes ready
+    // but higher runs do we they won't be noticed.
+    // Perhaps have this limit be an argument.
     {
         psString limitString = psDBGenerateLimitSQL( 1000 );
@@ -940 +890 @@
         return false;
     }
+
     if (!psArrayLength(magicRuns)) {
+        // no pending magicRuns nothing to do
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicRuns);
         return true;
     }
-
-    query = pxDataGet("magictool_toprocess_tree.sql");
+    psArray *output = psArrayAllocEmpty(100);
+
+    {
+        query = pxDataGet("magictool_toprocess_tree.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
+            if (limit && (psArrayLength(output) >= limit)) {
+                break;
+            }
+            bool status;
+            psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
+            if (!status) {
+                psAbort("failed to lookup value for magic_id column");
+            }
+
+            whereString = NULL;
+            psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
+            if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(whereString);
+                psFree(query);
+                return false;
+            }
+            psFree(whereString);
+            psArray *magicTree = p_psDBFetchResult(config->dbh);
+            if (!magicTree) {
+                psErrorCode err = psErrorCodeLast();
+                switch (err) {
+                    case PS_ERR_DB_CLIENT:
+                        psError(PXTOOLS_ERR_SYS, false, "database error");
+                    case PS_ERR_DB_SERVER:
+                        psError(PXTOOLS_ERR_PROG, false, "database error");
+                    default:
+                        psError(PXTOOLS_ERR_PROG, false, "unknown error");
+                }
+
+                return false;
+            }
+            psS64 length = psArrayLength(magicTree);
+            if (!length) {
+                psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
+                psFree(magicTree);
+                continue;
+            }
+
+            psHash *forest = psHashAlloc(length);
+
+            // convert the array of metadata into a pxTree structure
+            for (long i = 0; i < length; i++) {
+                bool status;
+                psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
+                if (!status) {
+                    psAbort("failed to lookup value for node column");
+                }
+
+                psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
+                if (!status) {
+                    psAbort("failed to lookup value for dep column");
+                }
+
+                pxTreeBuilder(forest, node, dep, magicTree->data[i]);
+
+            }
+
+            // find the root of the tree
+            pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
+            psFree(forest);
+            //    pxTreePrint(stdout, root);
+
+            // crawl through the tree and looking for nodes with children that are all
+            // "done"
+            pxTreeCrawl(root, findReadyNodes, output);
+            psFree(root);
+            psFree(magicTree);
+        }
+
+        int len = psArrayLength(output);
+        if (len) {
+            if (limit) {
+                if (limit < psArrayLength(output)) {
+                    // truncate the array
+                    long arrayLength = psArrayLength(output);
+                    for (long i = arrayLength - 1; i >= limit; i--) {
+                        psArrayRemoveIndex(output, i);
+                    }
+                    // negative simple so the default is true
+                    if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
+                        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+                        psFree(output);
+                        return false;
+                    }
+                    psFree(output);
+                    return true;
+                } else {
+                    limit -= len;
+                }
+            }
+        }
+    }
+
+    // look for "inputs" (skycells) that need to processed
+    query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -952 +1007 @@
     }
 
-    for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
-        if (limit && (psArrayLength(output) >= limit)) {
-            break;
-        }
-        bool status;
-        psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
-        if (!status) {
-            psAbort("failed to lookup value for magic_id column");
-        }
-
-        whereString = NULL;
-        psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
-        if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(whereString);
-            psFree(query);
-            return false;
-        }
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(whereString);
-        psArray *magicTree = p_psDBFetchResult(config->dbh);
-        if (!magicTree) {
-            psErrorCode err = psErrorCodeLast();
-            switch (err) {
-                case PS_ERR_DB_CLIENT:
-                    psError(PXTOOLS_ERR_SYS, false, "database error");
-                case PS_ERR_DB_SERVER:
-                    psError(PXTOOLS_ERR_PROG, false, "database error");
-                default:
-                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
-            }
-
-            return false;
-        }
-        psS64 length = psArrayLength(magicTree);
-        if (!length) {
-            psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
-            psFree(magicTree);
-            continue;
-        }
-
-        psHash *forest = psHashAlloc(length);
-
-        // convert the array of metadata into a pxTree structure
-        for (long i = 0; i < length; i++) {
-            bool status;
-            psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
-            if (!status) {
-                psAbort("failed to lookup value for node column");
-            }
-
-            psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
-            if (!status) {
-                psAbort("failed to lookup value for dep column");
-            }
-
-            pxTreeBuilder(forest, node, dep, magicTree->data[i]);
-
-        }
-
-        // find the root of the tree
-        pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
-        psFree(forest);
-        //    pxTreePrint(stdout, root);
-
-        // crawl through the tree and looking for nodes with children that are all
-        // "done"
-        pxTreeCrawl(root, findReadyNodes, output);
-        psFree(root);
-        psFree(magicTree);
-    }
-
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *skycellOutput = p_psDBFetchResult(config->dbh);
+    if (!skycellOutput) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    // merge the arrays so that we can print them all at once
+    if (psArrayLength(skycellOutput)) {
+        int len = psArrayLength(skycellOutput);
+        for (int i=0; i < len; i++) {
+            psArrayAdd(output, 0, skycellOutput->data[i]);
+        }
+    }
+    psFree(skycellOutput);
     if (psArrayLength(output)) {
-        if (limit && (limit < psArrayLength(output))) {
-            // truncate the array
-            long arrayLength = psArrayLength(output);
-            for (long i = arrayLength - 1; i >= limit; i--) {
-                psArrayRemoveIndex(output, i);
-            }
-        }
-        // negative simple so the default is true
         if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
@@ -1037 +1052 @@
             return false;
         }
-    }
-
+    } else {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+    }
     psFree(output);
-
     return true;
 }
@@ -1622 +1637 @@
     return true;
 }
+static bool setgotocleanedMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "magicRun.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "magicRun.label", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "magicRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+
+    psString query = psStringCopy("UPDATE magicRun SET workdir_state = 'goto_cleaned'\n");
+    if (set_label) {
+        psStringAppend(&query, ", label = '%s'", set_label);
+    }
+    // This mode doubles as a revert function for cleanup errors
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
+    if (!clearfault) {
+        psStringAppend(&query, "WHERE workdir_state = 'dirty'");
+    } else {
+        psStringAppend(&query, "WHERE workdir_state = 'error_cleaned'");
+    }
+    psStringAppend(&query, "\nAND (magicRun.state = 'full' OR magicRun.state = 'drop')");
+
+    // Require search parameters unless we're just clearing faults
+    if (psListLength(where->list)) {
+        psString clause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nAND %s", clause);
+        psFree(clause);
+        psFree(where);
+    } else if ( !clearfault) {
+        psError(PS_ERR_UNKNOWN, false, "search parameters are required");
+        psFree(where);
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool setworkdirstateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
+    PXOPT_LOOKUP_STR(workdir_state, config->args, "-set_workdir_state", true, false);
+
+    if (strcmp(workdir_state, "cleaned") && strcmp(workdir_state, "error_cleaned")) {
+        psError(PS_ERR_UNKNOWN, true, "%s is not a valid value for workdir_state", workdir_state);
+        return false;
+    }
+    
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicRun SET workdir_state = '%s' WHERE magic_id = %" PRId64, workdir_state, magic_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+static bool tocleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-data_group", "magicRun.data_group", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_tocleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}