Changeset 26015 for trunk/ippTasks

Timestamp:

Nov 3, 2009, 1:02:08 PM (17 years ago)

Author:

bills

Message:

Spread building of distribution bundles over a number of hosts (currently 5)

Location:

trunk/ippTasks

Files:

• dist.pro (modified) (5 diffs)
• ipphosts.mhpcc.config (modified) (1 diff)
• rcserver.pro (modified) (1 diff)

trunk/ippTasks/dist.pro

@@ -70 +70 @@
     active false
   end
+end
+
+macro set.dist.workdir.by.component
+    # component template default varname
+    if ($0 != 5)
+        echo "USAGE: set.workdir.by.component (componentID) (template) (default) (varname)"
+        break
+    end
+    local host component default template varname length start count component_id
+    $component = $1
+    $template = $2
+    $default = $3
+    $varname = $4
+    if ("$template" == "NULL")
+        echo "ERROR: WORKDIR template not set."
+        break
+    end
+    book getword ipphosts distribution count -var count
+    if ("$count" == "NULL")
+        echo "ERROR: disthosts is empty"
+        break
+    end
+    # take last two letters of component id
+    # treat it as an integer and use modulous of length of distribution hosts
+    # to compute a host
+    strlen $component length
+    $start = $length - 2
+    substr $component $start 2 index
+    $component_id = $index % $count
+
+    book getword ipphosts distribution $component_id -var host
+    if ("$host" == "NULL")
+        echo "ERROR: failed to find host for $component"
+        $host = $default
+    end
+
+    strsub $template @HOST@ $host -var $varname
 end
 
@@ -165 +202 @@
     book getword distToProcess $pageName no_magic -var NO_MAGIC
     book getword distToProcess $pageName magicked -var MAGICKED
-    book getword distToProcess $pageName outdir -var OUTDIR
+    book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToProcess $pageName dbname -var DBNAME
 
@@ -187 +224 @@
 #    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
 #    host anyhost
+
+    set.dist.workdir.by.component $COMPONENT $OUTDIR_TEMPLATE ipp049 OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
+    end
 
     substr $COMPONENT 0 3 COMP_HEAD
@@ -294 +337 @@
     book getword distToAdvance $pageName stage   -var STAGE
     book getword distToAdvance $pageName stage_id -var STAGE_ID
-    book getword distToAdvance $pageName outdir -var OUTDIR
+    book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToAdvance $pageName clean -var CLEAN
     book getword distToAdvance $pageName dbname -var DBNAME
@@ -304 +347 @@
     host anyhost
 
+    # using $DIST_ID as the "component" works fine here since we only look
+    # at the last two digits
+    set.dist.workdir.by.component $DIST_ID $OUTDIR_TEMPLATE ipp049 OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $DIST_ID
+        break
+    end
+
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
 #    stdout $logfile

trunk/ippTasks/ipphosts.mhpcc.config

@@ -214 +214 @@
   ota76  STR  ipp025
 END
+
+ipphosts METADATA
+  camera STR distribution
+  count  S32  5
+  0      STR ipp048
+  1      STR ipp049
+  2      STR ipp050
+  3      STR ipp051
+  4      STR ipp052
+END

trunk/ippTasks/rcserver.pro

@@ -139 +139 @@
     book getword rcPendingFS $pageName dbname -var DBNAME
 
-#    set.host.for.camera $CAMERA $MAGIC_ID
-#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     host anyhost