Changeset 27838 for branches/tap_branches/ippTasks/dist.pro

Timestamp:

May 3, 2010, 8:41:49 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/tap_branches

Files:

• . (modified) (1 prop)
• ippTasks/dist.pro (modified) (10 diffs)

branches/tap_branches/ippTasks/dist.pro

@@ -15 +15 @@
 $distToAdvance_DB = 0
 $distQueue_DB = 0
+$dist_revert_DB = 0
 
 ### list of stages
@@ -26 +27 @@
 list DIST_STAGE -add "diff"
 list DIST_STAGE -add "stack"
+list DIST_STAGE -add "SSdiff"
 
 $currentStage = 0
@@ -55 +57 @@
     active true
   end
+  task dist.revert
+    active false
+  end
 end
 
@@ -70 +75 @@
     active false
   end
-end
+  task dist.revert
+    active false
+  end
+end
+
+macro dist.revert.on
+  task dist.revert
+    active true
+  end
+end
+
+macro dist.revert.off
+  task dist.revert
+    active false
+  end
+end
+
+macro get.host.for.component
+    if ($0 != 3)
+        echo "USAGE: get.host.for.component (componentID) (varname)"
+        break
+    end
+    local component host varname
+    $component = $1
+    $varname = $2
+    substr $component 0 3 COMP_HEAD
+    if ("$COMP_HEAD" == "sky")
+        book getword ipphosts dist_skycell count -var count
+        local j myValue skyhash
+        list word -splitbychar . $component
+        $skyhash = 0
+        for j 0 $word:n
+            inthash $word:$j $count -var myValue
+            $skyhash = $skyhash + $myValue
+        end
+        inthash $skyhash $count -var skyhash
+        sprintf skyname "sky%02d" $skyhash
+        book getword ipphosts dist_skycell $skyname -var host
+    else
+        book getword ipphosts dist_chip $component -var host
+    end
+    $$varname = $host
+end
+
+macro set.dist.workdir.by.component
+    if ($0 != 5)
+        echo "USAGE: set.workdir.by.component (stage_id) (componentID) (template) (varname)"
+        break
+    end
+    local host stage_id component default template varname length start count selector component_id random_number
+
+    $stage_id = $1
+    $component = $2
+    $template = $3
+    $varname = $4
+    if ("$template" == "NULL")
+        echo "ERROR: WORKDIR template not set."
+        break
+    end
+    book getword ipphosts distribution count -var count
+    if ("$count" == "NULL")
+        echo "ERROR: distributin hosts list is empty"
+        break
+    end
+    $host = "foo"
+    if ("$component" == "exposure") 
+        # take last two letters of selector
+        # treat it as an integer and use modulous of length of distribution hosts
+        # to compute an index into the host table
+        strlen $stage_id length
+        $start = $length - 2
+        substr $stage_id $start 2 index
+        $component_id = $index % $count
+        book getword ipphosts distribution $component_id -var myhost
+    else
+        get.host.for.component $component myhost
+    end
+
+    if ("$myhost" == "NULL")
+        echo "ERROR: failed to find host for $component"
+        break
+    end
+
+    strsub $template @HOST@ $myhost -var $varname
+
+    #echo template is $template
+    #echo outdir is $$varname
+end
+
 
 task           dist.process.load
@@ -165 +258 @@
     book getword distToProcess $pageName no_magic -var NO_MAGIC
     book getword distToProcess $pageName magicked -var MAGICKED
-    book getword distToProcess $pageName outdir -var OUTDIR
+    book getword distToProcess $pageName alt_path_base -var ALT_PATH_BASE
+    book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToProcess $pageName dbname -var DBNAME
 
@@ -183 +277 @@
         $EXTRA_ARGS = $EXTRA_ARGS --poor_quality
     end
-
-#    set.host.for.camera $CAMERA $MAGIC_ID
-#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
-#    host anyhost
+    if ("$ALT_PATH_BASE" != "")
+        $EXTRA_ARGS = $EXTRA_ARGS --alt_path_base $ALT_PATH_BASE
+    end
+
+
+    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
+    end
 
     substr $COMPONENT 0 3 COMP_HEAD
@@ -283 +383 @@
   task.exec
     book npages distToAdvance -var N
-    if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($N == 0) 
+        periods -exec 10 
+        break
+    end
+    periods -exec .05
     
     # look for new components to process (pantaskState == INIT)
@@ -294 +398 @@
     book getword distToAdvance $pageName stage   -var STAGE
     book getword distToAdvance $pageName stage_id -var STAGE_ID
-    book getword distToAdvance $pageName outdir -var OUTDIR
+    book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToAdvance $pageName clean -var CLEAN
     book getword distToAdvance $pageName dbname -var DBNAME
@@ -304 +408 @@
     host anyhost
 
+    # using $DIST_ID as the "component" works fine here since we only look
+    # at the last two digits
+    set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $DIST_ID
+        break
+    end
+
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
 #    stdout $logfile
@@ -332 +444 @@
   end
 end
+
+task dist.revert
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 120.0
+  npending     1
+  active false
+  
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = disttool -revertcomponent
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$dist_revert_DB
+      $run = $run -dbname $DB:$dist_revert_DB
+      $dist_revert_DB ++
+      if ($dist_revert_DB >= $DB:n) set dist_revert_DB = 0
+    end
+    add_poll_labels run
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end