Changeset 38586 for trunk

Timestamp:

Jul 14, 2015, 1:54:16 PM (11 years ago)

Author:

bills

Message:

Add a task to queue cleanup of images that have been updated for postage stamp requests.
It runs twice a day and discovers whether or not there are any unfinished postage stamp
dependents. If not queues all of the chip, warp, and diff runs in gpc1 database with label like ps_ud%
to be cleaned. If there are outstanding dependents it does not queue the updates since we don't
want to cause problems with the postage stamp jobs that are depending on them.
This should be much better than the formerly manual process which got done whenever Bill happened to think of it.

Location:

trunk

Files:

• ippTasks/pstamp.pro (modified) (2 diffs)
• ippTools/share/pstamptool_pendingdependent.sql (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (2 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/pstamp_queue_update_cleanup.pl (added)

trunk/ippTasks/pstamp.pro

@@ -18 +18 @@
 $pstampCleanup_DB = 0
 $pstampStopFaulted_DB = 0
+$pstampQCleanup_DB = 0
 
 # give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT
@@ -1212 +1213 @@
     end
 end
+
+task pstamp.queue.update.cleanup
+    host        local
+    trange      22:00:00 23:59:59 -nmax 2
+    periods     -poll $RUNPOLL
+    periods     -exec 3600
+    periods     -timeout 120
+    npending    1
+
+    task.exec
+        stdout NULL
+        stderr $LOGSUBDIR/pstamp.queue.update.cleanup.log
+
+        $DBNAME = $DB:$pstampQCleanup_DB
+        $pstampQCleanup_DB++
+        if ($pstampQCleanup_DB >= $DB:n) set pstampQCleanup_DB = 0
+
+        $run = pstamp_queue_update_cleanup.pl  -$PS_DBSERVER --imagedb gpc1 --label ps_ud%
+        add_standard_args run
+        echo $run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # echo nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end

trunk/ippTools/share/pstamptool_pendingdependent.sql

@@ -10 +10 @@
 LEFT JOIN Label ON pstampRequest.label = Label.label
 WHERE pstampDependent.state = 'new'
-    AND pstampDependent.fault = 0
     AND pstampJob.state = 'run'
     AND pstampRequest.state = 'run'

trunk/ippTools/src/pstamptool.c

@@ -1435 +1435 @@
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(includeFaulted, config->args, "-includefaulted", false);
 
     psString query = pxDataGet("pstamptool_pendingdependent.sql");
@@ -1442 +1443 @@
     }
 
+    if (!includeFaulted) {
+        psStringAppend(&query, "    AND (pstampDependent.fault = 0)\n");
+    }
+
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&query, "\n    AND %s", whereClause);
         psFree(whereClause);
     }
     psFree(where);
 
-    psStringAppend(&query, " GROUP BY dep_id ORDER BY priority DESC, MIN(req_id), dep_id");
+    psStringAppend(&query, "\nGROUP BY dep_id ORDER BY priority DESC, MIN(req_id), dep_id");
 
     // treat limit == 0 as "no limit"

trunk/ippTools/src/pstamptoolConfig.c

@@ -253 +253 @@
     psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0,   "define label for dependent", NULL);
     psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0,  "define imagedb for depenent", NULL);
+    psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-includefaulted", 0,   "include faulted dependents in the output", false);
     psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit",  0,    "limit result set to N items", 0);
     psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0,   "use the simple output format", false);

trunk/pstamp/scripts/Makefile.am

@@ -13 +13 @@
         pstamp_queue_cleanup.pl \
         pstamp_queue_requests.pl \
+        pstamp_queue_update_cleanup.pl \
         pstamp_request_file \
         pstamp_results_file.pl \