Changeset 27838 for branches/tap_branches/ippTools/src/magictool.c

Timestamp:

May 3, 2010, 8:41:49 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/tap_branches

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/magictool.c (modified) (19 diffs)

branches/tap_branches/ippTools/src

@@ -16 +16 @@
 difftool
 disttool
+dqstatstool
 faketool
 flatcorr

@@ -16 +16 @@
 difftool
 disttool
+dqstatstool
 faketool
 flatcorr

branches/tap_branches/ippTools/src/magictool.c

@@ -114 +114 @@
     // Required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);
 
     // Optional
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false);
     PXOPT_LOOKUP_STR(note, config->args, "-note", false, false);
@@ -135 +135 @@
         psString query = pxDataGet("magictool_definebyquery_select.sql");
         if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
             return false;
         }
@@ -151 +151 @@
         // what if no skycells for the diff run completed?
 
-        if (!rerun) {
-            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        psString rerunWhereStr = NULL;
+        if (rerun) {
+            psStringAppend(&rerunWhereStr, "\n WHERE magicRun.label = '%s'", label);
         }
 
@@ -176 +178 @@
             queryWhereStr = psStringCopy("");
         }
-
-        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
+        if (!rerunWhereStr) {
+            rerunWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, rerunWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(diffWhereStr);
             psFree(queryWhereStr);
+            psFree(rerunWhereStr);
             psFree(query);
             return false;
@@ -186 +192 @@
         psFree(diffWhereStr);
         psFree(queryWhereStr);
+        psFree(rerunWhereStr);
         psFree(query);
     }
@@ -427 +434 @@
     psString query = pxDataGet("magictool_inputskyfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -500 +507 @@
     psString query = pxDataGet("magictool_totree.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -573 +580 @@
 
     if (fault > 0) {
-        char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64;
+        char *query = "UPDATE magicRun SET fault = %d WHERE magic_id = %" PRId64;
         if (!p_psDBRunQueryF(config->dbh, query, fault, magic_id)) {
             psError(PS_ERR_UNKNOWN, false,
@@ -612 +619 @@
         return false;
     }
+    psS32 numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic runs", numUpdated);
+
     return true;
 }
@@ -635 +645 @@
     psString query = pxDataGet("magictool_inputs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -805 +815 @@
     psString query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -867 +877 @@
 
     // look for tree nodes that need to be processed
+
+    // first find incomplete magicRuns
     query = pxDataGet("magictool_toprocess_runs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // we limit the query even though it is cheap (only magic_id is selected)
+
+    // XXX: if the first 1000 unfinished magicRuns have no ready nodes
+    // that haven't faulted, later runs won't get returned even though
+    // they have work to do. When we used a limit of 100 we actually ran
+    // into this problem. Since we're using labels a limit of 1000 will
+    // probably be ok.
     {
-        psString limitString = psDBGenerateLimitSQL( 100 );
+        psString limitString = psDBGenerateLimitSQL( 1000 );
         psStringAppend(&query, " %s", limitString);
         psFree(limitString);
@@ -910 +929 @@
     query = pxDataGet("magictool_toprocess_tree.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -923 +942 @@
             psAbort("failed to lookup value for magic_id column");
         }
-        
+
         whereString = NULL;
         psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
@@ -1041 +1060 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     psString query = pxDataGet("magictool_revertnode.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1067 +1086 @@
 
     psS32 numUpdated = psDBAffectedRows(config->dbh);
-    psLogMsg("magictool", PS_LOG_INFO, "Updated %d magic nodes", numUpdated);
+    psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic nodes", numUpdated);
 
     return true;
@@ -1083 +1102 @@
     psString query = pxDataGet("magictool_tomask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1171 +1190 @@
     psString query = pxDataGet("magictool_addmask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         if (!psDBRollback(config->dbh)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1278 +1297 @@
     psString query = pxDataGet("magictool_mask.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }

@@ -16 +16 @@
 difftool
 disttool
+dqstatstool
 faketool
 flatcorr